diff options
Diffstat (limited to 'llvm/lib/Transforms/Instrumentation/MemProfUse.cpp')
| -rw-r--r-- | llvm/lib/Transforms/Instrumentation/MemProfUse.cpp | 739 |
1 files changed, 739 insertions, 0 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp new file mode 100644 index 000000000000..e5b357fc1bfb --- /dev/null +++ b/llvm/lib/Transforms/Instrumentation/MemProfUse.cpp @@ -0,0 +1,739 @@ +//===- MemProfUse.cpp - memory allocation profile use pass --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the MemProfUsePass which reads memory profiling data +// and uses it to add metadata to instructions to guide optimization. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/MemProfUse.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/MemoryProfileInfo.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Module.h" +#include "llvm/ProfileData/InstrProf.h" +#include "llvm/ProfileData/InstrProfReader.h" +#include "llvm/ProfileData/MemProfCommon.h" +#include "llvm/Support/BLAKE3.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/HashBuilder.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Transforms/Utils/LongestCommonSequence.h" +#include <map> +#include <set> + +using namespace llvm; +using namespace llvm::memprof; + +#define DEBUG_TYPE "memprof" + +namespace llvm { +extern cl::opt<bool> PGOWarnMissing; +extern cl::opt<bool> NoPGOWarnMismatch; +extern cl::opt<bool> NoPGOWarnMismatchComdatWeak; +} // namespace llvm + +// By default disable matching of allocation profiles onto operator new that +// already explicitly pass a hot/cold hint, since we don't currently +// override these hints anyway. +static cl::opt<bool> ClMemProfMatchHotColdNew( + "memprof-match-hot-cold-new", + cl::desc( + "Match allocation profiles onto existing hot/cold operator new calls"), + cl::Hidden, cl::init(false)); + +static cl::opt<bool> + ClPrintMemProfMatchInfo("memprof-print-match-info", + cl::desc("Print matching stats for each allocation " + "context in this module's profiles"), + cl::Hidden, cl::init(false)); + +static cl::opt<bool> + SalvageStaleProfile("memprof-salvage-stale-profile", + cl::desc("Salvage stale MemProf profile"), + cl::init(false), cl::Hidden); + +static cl::opt<bool> ClMemProfAttachCalleeGuids( + "memprof-attach-calleeguids", + cl::desc( + "Attach calleeguids as value profile metadata for indirect calls."), + cl::init(true), cl::Hidden); + +static cl::opt<unsigned> MinMatchedColdBytePercent( + "memprof-matching-cold-threshold", cl::init(100), cl::Hidden, + cl::desc("Min percent of cold bytes matched to hint allocation cold")); + +// Matching statistics +STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile."); +STATISTIC(NumOfMemProfMismatch, + "Number of functions having mismatched memory profile hash."); +STATISTIC(NumOfMemProfFunc, "Number of functions having valid memory profile."); +STATISTIC(NumOfMemProfAllocContextProfiles, + "Number of alloc contexts in memory profile."); +STATISTIC(NumOfMemProfCallSiteProfiles, + "Number of callsites in memory profile."); +STATISTIC(NumOfMemProfMatchedAllocContexts, + "Number of matched memory profile alloc contexts."); +STATISTIC(NumOfMemProfMatchedAllocs, + "Number of matched memory profile allocs."); +STATISTIC(NumOfMemProfMatchedCallSites, + "Number of matched memory profile callsites."); + +static void addCallsiteMetadata(Instruction &I, + ArrayRef<uint64_t> InlinedCallStack, + LLVMContext &Ctx) { + I.setMetadata(LLVMContext::MD_callsite, + buildCallstackMetadata(InlinedCallStack, Ctx)); +} + +static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset, + uint32_t Column) { + llvm::HashBuilder<llvm::TruncatedBLAKE3<8>, llvm::endianness::little> + HashBuilder; + HashBuilder.add(Function, LineOffset, Column); + llvm::BLAKE3Result<8> Hash = HashBuilder.final(); + uint64_t Id; + std::memcpy(&Id, Hash.data(), sizeof(Hash)); + return Id; +} + +static uint64_t computeStackId(const memprof::Frame &Frame) { + return computeStackId(Frame.Function, Frame.LineOffset, Frame.Column); +} + +static AllocationType addCallStack(CallStackTrie &AllocTrie, + const AllocationInfo *AllocInfo, + uint64_t FullStackId) { + SmallVector<uint64_t> StackIds; + for (const auto &StackFrame : AllocInfo->CallStack) + StackIds.push_back(computeStackId(StackFrame)); + auto AllocType = getAllocType(AllocInfo->Info.getTotalLifetimeAccessDensity(), + AllocInfo->Info.getAllocCount(), + AllocInfo->Info.getTotalLifetime()); + std::vector<ContextTotalSize> ContextSizeInfo; + if (recordContextSizeInfoForAnalysis()) { + auto TotalSize = AllocInfo->Info.getTotalSize(); + assert(TotalSize); + assert(FullStackId != 0); + ContextSizeInfo.push_back({FullStackId, TotalSize}); + } + AllocTrie.addCallStack(AllocType, StackIds, std::move(ContextSizeInfo)); + return AllocType; +} + +// Return true if InlinedCallStack, computed from a call instruction's debug +// info, is a prefix of ProfileCallStack, a list of Frames from profile data +// (either the allocation data or a callsite). +static bool +stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack, + ArrayRef<uint64_t> InlinedCallStack) { + return ProfileCallStack.size() >= InlinedCallStack.size() && + llvm::equal(ProfileCallStack.take_front(InlinedCallStack.size()), + InlinedCallStack, [](const Frame &F, uint64_t StackId) { + return computeStackId(F) == StackId; + }); +} + +static bool isAllocationWithHotColdVariant(const Function *Callee, + const TargetLibraryInfo &TLI) { + if (!Callee) + return false; + LibFunc Func; + if (!TLI.getLibFunc(*Callee, Func)) + return false; + switch (Func) { + case LibFunc_Znwm: + case LibFunc_ZnwmRKSt9nothrow_t: + case LibFunc_ZnwmSt11align_val_t: + case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t: + case LibFunc_Znam: + case LibFunc_ZnamRKSt9nothrow_t: + case LibFunc_ZnamSt11align_val_t: + case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t: + case LibFunc_size_returning_new: + case LibFunc_size_returning_new_aligned: + return true; + case LibFunc_Znwm12__hot_cold_t: + case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t: + case LibFunc_ZnwmSt11align_val_t12__hot_cold_t: + case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t: + case LibFunc_Znam12__hot_cold_t: + case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t: + case LibFunc_ZnamSt11align_val_t12__hot_cold_t: + case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t: + case LibFunc_size_returning_new_hot_cold: + case LibFunc_size_returning_new_aligned_hot_cold: + return ClMemProfMatchHotColdNew; + default: + return false; + } +} + +struct AllocMatchInfo { + uint64_t TotalSize = 0; + AllocationType AllocType = AllocationType::None; +}; + +DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>> +memprof::extractCallsFromIR(Module &M, const TargetLibraryInfo &TLI, + function_ref<bool(uint64_t)> IsPresentInProfile) { + DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>> Calls; + + auto GetOffset = [](const DILocation *DIL) { + return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) & + 0xffff; + }; + + for (Function &F : M) { + if (F.isDeclaration()) + continue; + + for (auto &BB : F) { + for (auto &I : BB) { + if (!isa<CallBase>(&I) || isa<IntrinsicInst>(&I)) + continue; + + auto *CB = dyn_cast<CallBase>(&I); + auto *CalledFunction = CB->getCalledFunction(); + // Disregard indirect calls and intrinsics. + if (!CalledFunction || CalledFunction->isIntrinsic()) + continue; + + StringRef CalleeName = CalledFunction->getName(); + // True if we are calling a heap allocation function that supports + // hot/cold variants. + bool IsAlloc = isAllocationWithHotColdVariant(CalledFunction, TLI); + // True for the first iteration below, indicating that we are looking at + // a leaf node. + bool IsLeaf = true; + for (const DILocation *DIL = I.getDebugLoc(); DIL; + DIL = DIL->getInlinedAt()) { + StringRef CallerName = DIL->getSubprogramLinkageName(); + assert(!CallerName.empty() && + "Be sure to enable -fdebug-info-for-profiling"); + uint64_t CallerGUID = memprof::getGUID(CallerName); + uint64_t CalleeGUID = memprof::getGUID(CalleeName); + // Pretend that we are calling a function with GUID == 0 if we are + // in the inline stack leading to a heap allocation function. + if (IsAlloc) { + if (IsLeaf) { + // For leaf nodes, set CalleeGUID to 0 without consulting + // IsPresentInProfile. + CalleeGUID = 0; + } else if (!IsPresentInProfile(CalleeGUID)) { + // In addition to the leaf case above, continue to set CalleeGUID + // to 0 as long as we don't see CalleeGUID in the profile. + CalleeGUID = 0; + } else { + // Once we encounter a callee that exists in the profile, stop + // setting CalleeGUID to 0. + IsAlloc = false; + } + } + + LineLocation Loc = {GetOffset(DIL), DIL->getColumn()}; + Calls[CallerGUID].emplace_back(Loc, CalleeGUID); + CalleeName = CallerName; + IsLeaf = false; + } + } + } + } + + // Sort each call list by the source location. + for (auto &[CallerGUID, CallList] : Calls) { + llvm::sort(CallList); + CallList.erase(llvm::unique(CallList), CallList.end()); + } + + return Calls; +} + +DenseMap<uint64_t, LocToLocMap> +memprof::computeUndriftMap(Module &M, IndexedInstrProfReader *MemProfReader, + const TargetLibraryInfo &TLI) { + DenseMap<uint64_t, LocToLocMap> UndriftMaps; + + DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>> CallsFromProfile = + MemProfReader->getMemProfCallerCalleePairs(); + DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>> CallsFromIR = + extractCallsFromIR(M, TLI, [&](uint64_t GUID) { + return CallsFromProfile.contains(GUID); + }); + + // Compute an undrift map for each CallerGUID. + for (const auto &[CallerGUID, IRAnchors] : CallsFromIR) { + auto It = CallsFromProfile.find(CallerGUID); + if (It == CallsFromProfile.end()) + continue; + const auto &ProfileAnchors = It->second; + + LocToLocMap Matchings; + longestCommonSequence<LineLocation, GlobalValue::GUID>( + ProfileAnchors, IRAnchors, std::equal_to<GlobalValue::GUID>(), + [&](LineLocation A, LineLocation B) { Matchings.try_emplace(A, B); }); + [[maybe_unused]] bool Inserted = + UndriftMaps.try_emplace(CallerGUID, std::move(Matchings)).second; + + // The insertion must succeed because we visit each GUID exactly once. + assert(Inserted); + } + + return UndriftMaps; +} + +// Given a MemProfRecord, undrift all the source locations present in the +// record in place. +static void +undriftMemProfRecord(const DenseMap<uint64_t, LocToLocMap> &UndriftMaps, + memprof::MemProfRecord &MemProfRec) { + // Undrift a call stack in place. + auto UndriftCallStack = [&](std::vector<Frame> &CallStack) { + for (auto &F : CallStack) { + auto I = UndriftMaps.find(F.Function); + if (I == UndriftMaps.end()) + continue; + auto J = I->second.find(LineLocation(F.LineOffset, F.Column)); + if (J == I->second.end()) + continue; + auto &NewLoc = J->second; + F.LineOffset = NewLoc.LineOffset; + F.Column = NewLoc.Column; + } + }; + + for (auto &AS : MemProfRec.AllocSites) + UndriftCallStack(AS.CallStack); + + for (auto &CS : MemProfRec.CallSites) + UndriftCallStack(CS.Frames); +} + +// Helper function to process CalleeGuids and create value profile metadata +static void addVPMetadata(Module &M, Instruction &I, + ArrayRef<GlobalValue::GUID> CalleeGuids) { + if (!ClMemProfAttachCalleeGuids || CalleeGuids.empty()) + return; + + if (I.getMetadata(LLVMContext::MD_prof)) { + uint64_t Unused; + // TODO: When merging is implemented, increase this to a typical ICP value + // (e.g., 3-6) For now, we only need to check if existing data exists, so 1 + // is sufficient + auto ExistingVD = getValueProfDataFromInst(I, IPVK_IndirectCallTarget, + /*MaxNumValueData=*/1, Unused); + // We don't know how to merge value profile data yet. + if (!ExistingVD.empty()) { + return; + } + } + + SmallVector<InstrProfValueData, 4> VDs; + uint64_t TotalCount = 0; + + for (const GlobalValue::GUID CalleeGUID : CalleeGuids) { + InstrProfValueData VD; + VD.Value = CalleeGUID; + // For MemProf, we don't have actual call counts, so we assign + // a weight of 1 to each potential target. + // TODO: Consider making this weight configurable or increasing it to + // improve effectiveness for ICP. + VD.Count = 1; + VDs.push_back(VD); + TotalCount += VD.Count; + } + + if (!VDs.empty()) { + annotateValueSite(M, I, VDs, TotalCount, IPVK_IndirectCallTarget, + VDs.size()); + } +} + +static void readMemprof(Module &M, Function &F, + IndexedInstrProfReader *MemProfReader, + const TargetLibraryInfo &TLI, + std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo> + &FullStackIdToAllocMatchInfo, + std::set<std::vector<uint64_t>> &MatchedCallSites, + DenseMap<uint64_t, LocToLocMap> &UndriftMaps, + OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize) { + auto &Ctx = M.getContext(); + // Previously we used getIRPGOFuncName() here. If F is local linkage, + // getIRPGOFuncName() returns FuncName with prefix 'FileName;'. But + // llvm-profdata uses FuncName in dwarf to create GUID which doesn't + // contain FileName's prefix. It caused local linkage function can't + // find MemProfRecord. So we use getName() now. + // 'unique-internal-linkage-names' can make MemProf work better for local + // linkage function. + auto FuncName = F.getName(); + auto FuncGUID = Function::getGUIDAssumingExternalLinkage(FuncName); + std::optional<memprof::MemProfRecord> MemProfRec; + auto Err = MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec); + if (Err) { + handleAllErrors(std::move(Err), [&](const InstrProfError &IPE) { + auto Err = IPE.get(); + bool SkipWarning = false; + LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncName + << ": "); + if (Err == instrprof_error::unknown_function) { + NumOfMemProfMissing++; + SkipWarning = !PGOWarnMissing; + LLVM_DEBUG(dbgs() << "unknown function"); + } else if (Err == instrprof_error::hash_mismatch) { + NumOfMemProfMismatch++; + SkipWarning = + NoPGOWarnMismatch || + (NoPGOWarnMismatchComdatWeak && + (F.hasComdat() || + F.getLinkage() == GlobalValue::AvailableExternallyLinkage)); + LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")"); + } + + if (SkipWarning) + return; + + std::string Msg = (IPE.message() + Twine(" ") + F.getName().str() + + Twine(" Hash = ") + std::to_string(FuncGUID)) + .str(); + + Ctx.diagnose( + DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning)); + }); + return; + } + + NumOfMemProfFunc++; + + // If requested, undrfit MemProfRecord so that the source locations in it + // match those in the IR. + if (SalvageStaleProfile) + undriftMemProfRecord(UndriftMaps, *MemProfRec); + + // Detect if there are non-zero column numbers in the profile. If not, + // treat all column numbers as 0 when matching (i.e. ignore any non-zero + // columns in the IR). The profiled binary might have been built with + // column numbers disabled, for example. + bool ProfileHasColumns = false; + + // Build maps of the location hash to all profile data with that leaf location + // (allocation info and the callsites). + std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo; + + // Helper struct for maintaining refs to callsite data. As an alternative we + // could store a pointer to the CallSiteInfo struct but we also need the frame + // index. Using ArrayRefs instead makes it a little easier to read. + struct CallSiteEntry { + // Subset of frames for the corresponding CallSiteInfo. + ArrayRef<Frame> Frames; + // Potential targets for indirect calls. + ArrayRef<GlobalValue::GUID> CalleeGuids; + + // Only compare Frame contents. + // Use pointer-based equality instead of ArrayRef's operator== which does + // element-wise comparison. We want to check if it's the same slice of the + // underlying array, not just equivalent content. + bool operator==(const CallSiteEntry &Other) const { + return Frames.data() == Other.Frames.data() && + Frames.size() == Other.Frames.size(); + } + }; + + struct CallSiteEntryHash { + size_t operator()(const CallSiteEntry &Entry) const { + return computeFullStackId(Entry.Frames); + } + }; + + // For the callsites we need to record slices of the frame array (see comments + // below where the map entries are added) along with their CalleeGuids. + std::map<uint64_t, std::unordered_set<CallSiteEntry, CallSiteEntryHash>> + LocHashToCallSites; + for (auto &AI : MemProfRec->AllocSites) { + NumOfMemProfAllocContextProfiles++; + // Associate the allocation info with the leaf frame. The later matching + // code will match any inlined call sequences in the IR with a longer prefix + // of call stack frames. + uint64_t StackId = computeStackId(AI.CallStack[0]); + LocHashToAllocInfo[StackId].insert(&AI); + ProfileHasColumns |= AI.CallStack[0].Column; + } + for (auto &CS : MemProfRec->CallSites) { + NumOfMemProfCallSiteProfiles++; + // Need to record all frames from leaf up to and including this function, + // as any of these may or may not have been inlined at this point. + unsigned Idx = 0; + for (auto &StackFrame : CS.Frames) { + uint64_t StackId = computeStackId(StackFrame); + ArrayRef<Frame> FrameSlice = ArrayRef<Frame>(CS.Frames).drop_front(Idx++); + ArrayRef<GlobalValue::GUID> CalleeGuids(CS.CalleeGuids); + LocHashToCallSites[StackId].insert({FrameSlice, CalleeGuids}); + + ProfileHasColumns |= StackFrame.Column; + // Once we find this function, we can stop recording. + if (StackFrame.Function == FuncGUID) + break; + } + assert(Idx <= CS.Frames.size() && CS.Frames[Idx - 1].Function == FuncGUID); + } + + auto GetOffset = [](const DILocation *DIL) { + return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) & + 0xffff; + }; + + // Now walk the instructions, looking up the associated profile data using + // debug locations. + for (auto &BB : F) { + for (auto &I : BB) { + if (I.isDebugOrPseudoInst()) + continue; + // We are only interested in calls (allocation or interior call stack + // context calls). + auto *CI = dyn_cast<CallBase>(&I); + if (!CI) + continue; + auto *CalledFunction = CI->getCalledFunction(); + if (CalledFunction && CalledFunction->isIntrinsic()) + continue; + // List of call stack ids computed from the location hashes on debug + // locations (leaf to inlined at root). + SmallVector<uint64_t, 8> InlinedCallStack; + // Was the leaf location found in one of the profile maps? + bool LeafFound = false; + // If leaf was found in a map, iterators pointing to its location in both + // of the maps. It might exist in neither, one, or both (the latter case + // can happen because we don't currently have discriminators to + // distinguish the case when a single line/col maps to both an allocation + // and another callsite). + auto AllocInfoIter = LocHashToAllocInfo.end(); + auto CallSitesIter = LocHashToCallSites.end(); + for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr; + DIL = DIL->getInlinedAt()) { + // Use C++ linkage name if possible. Need to compile with + // -fdebug-info-for-profiling to get linkage name. + StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); + if (Name.empty()) + Name = DIL->getScope()->getSubprogram()->getName(); + auto CalleeGUID = Function::getGUIDAssumingExternalLinkage(Name); + auto StackId = computeStackId(CalleeGUID, GetOffset(DIL), + ProfileHasColumns ? DIL->getColumn() : 0); + // Check if we have found the profile's leaf frame. If yes, collect + // the rest of the call's inlined context starting here. If not, see if + // we find a match further up the inlined context (in case the profile + // was missing debug frames at the leaf). + if (!LeafFound) { + AllocInfoIter = LocHashToAllocInfo.find(StackId); + CallSitesIter = LocHashToCallSites.find(StackId); + if (AllocInfoIter != LocHashToAllocInfo.end() || + CallSitesIter != LocHashToCallSites.end()) + LeafFound = true; + } + if (LeafFound) + InlinedCallStack.push_back(StackId); + } + // If leaf not in either of the maps, skip inst. + if (!LeafFound) + continue; + + // First add !memprof metadata from allocation info, if we found the + // instruction's leaf location in that map, and if the rest of the + // instruction's locations match the prefix Frame locations on an + // allocation context with the same leaf. + if (AllocInfoIter != LocHashToAllocInfo.end() && + // Only consider allocations which support hinting. + isAllocationWithHotColdVariant(CI->getCalledFunction(), TLI)) { + // We may match this instruction's location list to multiple MIB + // contexts. Add them to a Trie specialized for trimming the contexts to + // the minimal needed to disambiguate contexts with unique behavior. + CallStackTrie AllocTrie(&ORE, MaxColdSize); + uint64_t TotalSize = 0; + uint64_t TotalColdSize = 0; + for (auto *AllocInfo : AllocInfoIter->second) { + // Check the full inlined call stack against this one. + // If we found and thus matched all frames on the call, include + // this MIB. + if (stackFrameIncludesInlinedCallStack(AllocInfo->CallStack, + InlinedCallStack)) { + NumOfMemProfMatchedAllocContexts++; + uint64_t FullStackId = 0; + if (ClPrintMemProfMatchInfo || recordContextSizeInfoForAnalysis()) + FullStackId = computeFullStackId(AllocInfo->CallStack); + auto AllocType = addCallStack(AllocTrie, AllocInfo, FullStackId); + TotalSize += AllocInfo->Info.getTotalSize(); + if (AllocType == AllocationType::Cold) + TotalColdSize += AllocInfo->Info.getTotalSize(); + // Record information about the allocation if match info printing + // was requested. + if (ClPrintMemProfMatchInfo) { + assert(FullStackId != 0); + FullStackIdToAllocMatchInfo[std::make_pair( + FullStackId, InlinedCallStack.size())] = { + AllocInfo->Info.getTotalSize(), AllocType}; + } + } + } + // If the threshold for the percent of cold bytes is less than 100%, + // and not all bytes are cold, see if we should still hint this + // allocation as cold without context sensitivity. + if (TotalColdSize < TotalSize && MinMatchedColdBytePercent < 100 && + TotalColdSize * 100 >= MinMatchedColdBytePercent * TotalSize) { + AllocTrie.addSingleAllocTypeAttribute(CI, AllocationType::Cold, + "dominant"); + continue; + } + + // We might not have matched any to the full inlined call stack. + // But if we did, create and attach metadata, or a function attribute if + // all contexts have identical profiled behavior. + if (!AllocTrie.empty()) { + NumOfMemProfMatchedAllocs++; + // MemprofMDAttached will be false if a function attribute was + // attached. + bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata(CI); + assert(MemprofMDAttached == I.hasMetadata(LLVMContext::MD_memprof)); + if (MemprofMDAttached) { + // Add callsite metadata for the instruction's location list so that + // it simpler later on to identify which part of the MIB contexts + // are from this particular instruction (including during inlining, + // when the callsite metadata will be updated appropriately). + // FIXME: can this be changed to strip out the matching stack + // context ids from the MIB contexts and not add any callsite + // metadata here to save space? + addCallsiteMetadata(I, InlinedCallStack, Ctx); + } + } + continue; + } + + if (CallSitesIter == LocHashToCallSites.end()) + continue; + + // Otherwise, add callsite metadata. If we reach here then we found the + // instruction's leaf location in the callsites map and not the allocation + // map. + for (const auto &CallSiteEntry : CallSitesIter->second) { + // If we found and thus matched all frames on the call, create and + // attach call stack metadata. + if (stackFrameIncludesInlinedCallStack(CallSiteEntry.Frames, + InlinedCallStack)) { + NumOfMemProfMatchedCallSites++; + addCallsiteMetadata(I, InlinedCallStack, Ctx); + + // Try to attach indirect call metadata if possible. + if (!CalledFunction) + addVPMetadata(M, I, CallSiteEntry.CalleeGuids); + + // Only need to find one with a matching call stack and add a single + // callsite metadata. + + // Accumulate call site matching information upon request. + if (ClPrintMemProfMatchInfo) { + std::vector<uint64_t> CallStack; + append_range(CallStack, InlinedCallStack); + MatchedCallSites.insert(std::move(CallStack)); + } + break; + } + } + } + } +} + +MemProfUsePass::MemProfUsePass(std::string MemoryProfileFile, + IntrusiveRefCntPtr<vfs::FileSystem> FS) + : MemoryProfileFileName(MemoryProfileFile), FS(FS) { + if (!FS) + this->FS = vfs::getRealFileSystem(); +} + +PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) { + // Return immediately if the module doesn't contain any function. + if (M.empty()) + return PreservedAnalyses::all(); + + LLVM_DEBUG(dbgs() << "Read in memory profile:"); + auto &Ctx = M.getContext(); + auto ReaderOrErr = IndexedInstrProfReader::create(MemoryProfileFileName, *FS); + if (Error E = ReaderOrErr.takeError()) { + handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) { + Ctx.diagnose( + DiagnosticInfoPGOProfile(MemoryProfileFileName.data(), EI.message())); + }); + return PreservedAnalyses::all(); + } + + std::unique_ptr<IndexedInstrProfReader> MemProfReader = + std::move(ReaderOrErr.get()); + if (!MemProfReader) { + Ctx.diagnose(DiagnosticInfoPGOProfile( + MemoryProfileFileName.data(), StringRef("Cannot get MemProfReader"))); + return PreservedAnalyses::all(); + } + + if (!MemProfReader->hasMemoryProfile()) { + Ctx.diagnose(DiagnosticInfoPGOProfile(MemoryProfileFileName.data(), + "Not a memory profile")); + return PreservedAnalyses::all(); + } + + auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); + + TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(*M.begin()); + DenseMap<uint64_t, LocToLocMap> UndriftMaps; + if (SalvageStaleProfile) + UndriftMaps = computeUndriftMap(M, MemProfReader.get(), TLI); + + // Map from the stack hash and matched frame count of each allocation context + // in the function profiles to the total profiled size (bytes) and allocation + // type. + std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo> + FullStackIdToAllocMatchInfo; + + // Set of the matched call sites, each expressed as a sequence of an inline + // call stack. + std::set<std::vector<uint64_t>> MatchedCallSites; + + uint64_t MaxColdSize = 0; + if (auto *MemProfSum = MemProfReader->getMemProfSummary()) + MaxColdSize = MemProfSum->getMaxColdTotalSize(); + + for (auto &F : M) { + if (F.isDeclaration()) + continue; + + const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F); + auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F); + readMemprof(M, F, MemProfReader.get(), TLI, FullStackIdToAllocMatchInfo, + MatchedCallSites, UndriftMaps, ORE, MaxColdSize); + } + + if (ClPrintMemProfMatchInfo) { + for (const auto &[IdLengthPair, Info] : FullStackIdToAllocMatchInfo) { + auto [Id, Length] = IdLengthPair; + errs() << "MemProf " << getAllocTypeAttributeString(Info.AllocType) + << " context with id " << Id << " has total profiled size " + << Info.TotalSize << " is matched with " << Length << " frames\n"; + } + + for (const auto &CallStack : MatchedCallSites) { + errs() << "MemProf callsite match for inline call stack"; + for (uint64_t StackId : CallStack) + errs() << " " << StackId; + errs() << "\n"; + } + } + + return PreservedAnalyses::none(); +} |