90#include <system_error>
97#define DEBUG_TYPE "sample-profile"
98#define CSINLINE_DEBUG DEBUG_TYPE "-inline"
101 "Number of functions inlined with context sensitive profile");
103 "Number of functions not inlined with context sensitive profile");
105 "Number of functions with CFG mismatched profile");
106STATISTIC(NumMatchedProfile,
"Number of functions with CFG matched profile");
108 "Number of inlined callsites with a partial distribution factor");
111 "Number of functions with FDO inline stopped due to min size limit");
113 "Number of functions with FDO inline stopped due to max size limit");
115 NumCSInlinedHitGrowthLimit,
116 "Number of functions with FDO inline stopped due to growth size limit");
122 cl::desc(
"Sort profiled recursion by edge weights."));
126 cl::desc(
"The size growth ratio limit for priority-based sample profile "
127 "loader inlining."));
131 cl::desc(
"The lower bound of size growth limit for "
132 "priority-based sample profile loader inlining."));
136 cl::desc(
"The upper bound of size growth limit for "
137 "priority-based sample profile loader inlining."));
141 cl::desc(
"Hot callsite threshold for priority-based sample profile loader "
146 cl::desc(
"Threshold for inlining cold callsites"));
152 "In ThinLTO pre-link, add import hints for in-module targets of "
153 "deferred indirect calls so they can be imported with the caller."));
163using Edge = std::pair<const BasicBlock *, const BasicBlock *>;
167class GUIDToFuncNameMapper {
172 CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
176 for (
const auto &
F : CurrentModule) {
178 CurrentGUIDToFuncNameMap.insert(
189 if (CanonName != OrigName)
190 CurrentGUIDToFuncNameMap.insert(
195 SetGUIDToFuncNameMapForAll(&CurrentGUIDToFuncNameMap);
198 ~GUIDToFuncNameMapper() {
202 CurrentGUIDToFuncNameMap.clear();
206 SetGUIDToFuncNameMapForAll(
nullptr);
211 std::queue<FunctionSamples *> FSToUpdate;
213 FSToUpdate.push(&IFS.second);
216 while (!FSToUpdate.empty()) {
219 FS->GUIDToFuncNameMap = Map;
220 for (
const auto &ICS : FS->getCallsiteSamples()) {
222 for (
const auto &IFS : FSMap) {
224 FSToUpdate.push(&FS);
236struct InlineCandidate {
238 const FunctionSamples *CalleeSamples;
246 float CallsiteDistribution;
250struct CandidateComparer {
251 bool operator()(
const InlineCandidate &
LHS,
const InlineCandidate &
RHS) {
252 if (
LHS.CallsiteCount !=
RHS.CallsiteCount)
253 return LHS.CallsiteCount <
RHS.CallsiteCount;
255 const FunctionSamples *LCS =
LHS.CalleeSamples;
256 const FunctionSamples *RCS =
RHS.CalleeSamples;
271using CandidateQueue =
283 const IPOOptions &Opts, StringRef Name, StringRef RemapName,
285 std::function<AssumptionCache &(
Function &)> GetAssumptionCache,
286 std::function<TargetTransformInfo &(
Function &)> GetTargetTransformInfo,
287 std::function<
const TargetLibraryInfo &(
Function &)> GetTLI,
288 LazyCallGraph &CG,
bool DisableSampleProfileInlining,
289 bool UseFlattenedProfile)
292 Opts(Opts), ProfileMergeInlinee(Opts.sample_profile_merge_inlinee),
293 ProfileSizeInline(
valueOr(Opts.sample_profile_inline_size,
false)),
294 CallsitePrioritizedInline(
295 valueOr(Opts.sample_profile_prioritized_inline,
false)),
296 AllowRecursiveInline(
298 UsePreInlinerDecision(
300 SalvageStaleProfile(
valueOr(Opts.salvage_stale_profile,
false)),
301 SalvageUnusedProfile(
valueOr(Opts.salvage_unused_profile,
false)),
302 GetAC(std::
move(GetAssumptionCache)),
303 GetTTI(std::
move(GetTargetTransformInfo)), GetTLI(std::
move(GetTLI)),
304 CG(CG), LTOPhase(LTOPhase),
305 AnnotatedPassName(Opts.annotate_sample_profile_inline_phase
309 DisableSampleProfileInlining(DisableSampleProfileInlining),
310 UseFlattenedProfile(UseFlattenedProfile) {}
314 ProfileSummaryInfo *_PSI);
317 const IPOOptions &Opts;
319 bool ProfileMergeInlinee;
320 bool ProfileSizeInline;
321 bool CallsitePrioritizedInline;
322 bool AllowRecursiveInline;
323 bool UsePreInlinerDecision;
324 bool SalvageStaleProfile;
325 bool SalvageUnusedProfile;
329 ErrorOr<uint64_t> getInstWeight(
const Instruction &
I)
override;
330 const FunctionSamples *findCalleeFunctionSamples(
const CallBase &
I)
const;
331 const FunctionSamples *
332 findFunctionSamples(
const Instruction &
I)
const override;
333 std::vector<const FunctionSamples *>
334 findIndirectCallFunctionSamples(
const Instruction &
I,
uint64_t &Sum)
const;
335 void findExternalInlineCandidate(CallBase *CB,
const FunctionSamples *Samples,
336 DenseSet<GlobalValue::GUID> &InlinedGUIDs,
339 bool tryPromoteAndInlineCandidate(
344 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
345 std::optional<InlineCost> getExternalInlineAdvisorCost(CallBase &CB);
346 bool getExternalInlineAdvisorShouldInline(CallBase &CB);
347 InlineCost shouldInlineCandidate(InlineCandidate &Candidate);
348 bool getInlineCandidate(InlineCandidate *NewCandidate, CallBase *CB);
350 tryInlineCandidate(InlineCandidate &Candidate,
353 inlineHotFunctionsWithPriority(
Function &
F,
354 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
356 bool shouldInlineColdCallee(CallBase &CallInst);
357 void emitOptimizationRemarksForInlineCandidates(
358 const SmallVectorImpl<CallBase *> &Candidates,
const Function &
F,
360 void promoteMergeNotInlinedContextSamples(
361 MapVector<CallBase *, const FunctionSamples *> NonInlinedCallSites,
363 std::vector<Function *> buildFunctionOrder(
Module &M, LazyCallGraph &CG);
364 std::unique_ptr<ProfiledCallGraph> buildProfiledCallGraph(
Module &M);
365 void generateMDProfMetadata(
Function &
F);
366 bool rejectHighStalenessProfile(
Module &M, ProfileSummaryInfo *PSI,
367 const SampleProfileMap &Profiles);
368 void removePseudoProbeInstsDiscriminator(
Module &M);
374 HashKeyMap<DenseMap, FunctionId, Function *> SymbolMap;
378 HashKeyMap<DenseMap, FunctionId, FunctionId> FuncNameToProfNameMap;
380 std::function<AssumptionCache &(
Function &)> GetAC;
381 std::function<TargetTransformInfo &(
Function &)> GetTTI;
382 std::function<
const TargetLibraryInfo &(
Function &)> GetTLI;
386 std::unique_ptr<SampleContextTracker> ContextTracker;
394 const std::string AnnotatedPassName;
398 std::shared_ptr<ProfileSymbolList> PSL;
403 struct NotInlinedProfileInfo {
406 DenseMap<Function *, NotInlinedProfileInfo> notInlinedCallInfo;
410 DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
417 bool ProfAccForSymsInList;
419 bool DisableSampleProfileInlining;
421 bool UseFlattenedProfile;
424 std::unique_ptr<InlineAdvisor> ExternalInlineAdvisor;
427 std::unique_ptr<SampleProfileMatcher> MatchingManager;
430 const char *getAnnotatedRemarkPassName()
const {
431 return AnnotatedPassName.c_str();
438inline bool SampleProfileInference<Function>::isExit(
const BasicBlock *BB) {
443inline void SampleProfileInference<Function>::findUnlikelyJumps(
444 const std::vector<const BasicBlockT *> &BasicBlocks,
446 for (
auto &Jump :
Func.Jumps) {
447 const auto *BB = BasicBlocks[Jump.Source];
448 const auto *Succ = BasicBlocks[Jump.Target];
452 const auto &Succs = Successors[BB];
453 if (Succs.size() == 2 && Succs.back() == Succ) {
455 Jump.IsUnlikely =
true;
462 Jump.IsUnlikely =
true;
483 return getProbeWeight(Inst);
487 return std::error_code();
493 return std::error_code();
503 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
506 return getInstWeightImpl(Inst);
521const FunctionSamples *
522SampleProfileLoader::findCalleeFunctionSamples(
const CallBase &Inst)
const {
528 StringRef CalleeName;
530 CalleeName =
Callee->getName();
533 return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName);
535 const FunctionSamples *
FS = findFunctionSamples(Inst);
540 CalleeName, Reader->getRemapper(),
541 &FuncNameToProfNameMap);
547std::vector<const FunctionSamples *>
548SampleProfileLoader::findIndirectCallFunctionSamples(
549 const Instruction &Inst,
uint64_t &Sum)
const {
551 std::vector<const FunctionSamples *>
R;
557 auto FSCompare = [](
const FunctionSamples *
L,
const FunctionSamples *
R) {
558 assert(L && R &&
"Expect non-null FunctionSamples");
559 if (
L->getHeadSamplesEstimate() !=
R->getHeadSamplesEstimate())
560 return L->getHeadSamplesEstimate() >
R->getHeadSamplesEstimate();
561 return L->getGUID() <
R->getGUID();
566 ContextTracker->getIndirectCalleeContextSamplesFor(DIL);
567 if (CalleeSamples.empty())
573 for (
const auto *
const FS : CalleeSamples) {
574 Sum +=
FS->getHeadSamplesEstimate();
581 const FunctionSamples *
FS = findFunctionSamples(Inst);
587 if (
auto T =
FS->findCallTargetMapAt(CallSite))
588 for (
const auto &T_C : *
T)
593 for (
const auto &NameFS : *M) {
594 Sum += NameFS.second.getHeadSamplesEstimate();
595 R.push_back(&NameFS.second);
602const FunctionSamples *
603SampleProfileLoader::findFunctionSamples(
const Instruction &Inst)
const {
614 auto it = DILocation2SampleMap.try_emplace(DIL,
nullptr);
617 it.first->second = ContextTracker->getContextSamplesFor(DIL);
619 it.first->second = Samples->findFunctionSamples(
620 DIL, Reader->getRemapper(), &FuncNameToProfNameMap);
622 return it.first->second;
636 Opts.sample_profile_icp_max_prom,
640 if (ValueData.empty())
643 unsigned NumPromoted = 0;
644 for (
const auto &
V : ValueData) {
655 if (NumPromoted == Opts.sample_profile_icp_max_prom)
675 if (Opts.sample_profile_icp_max_prom == 0)
681 Opts.sample_profile_icp_max_prom, OldSum,
true);
687 "If sum is 0, assume only one element in CallTargets "
688 "with count being NOMORE_ICP_MAGICNUM");
690 for (
const auto &
V : ValueData)
691 ValueCountMap[
V.Value] =
V.Count;
697 OldSum -= Pair.first->second;
704 for (
const auto &
V : ValueData) {
706 ValueCountMap[
V.Value] =
V.Count;
709 for (
const auto &
Data : CallTargets) {
716 assert(Sum >=
Data.Count &&
"Sum should never be less than Data.Count");
722 for (
const auto &ValueCount : ValueCountMap) {
724 InstrProfValueData{ValueCount.first, ValueCount.second});
728 [](
const InstrProfValueData &L,
const InstrProfValueData &R) {
729 return std::tie(L.Count, L.Value) > std::tie(R.Count, R.Value);
733 std::min(NewCallTargets.
size(),
734 static_cast<size_t>(Opts.sample_profile_icp_max_prom));
736 NewCallTargets, Sum, IPVK_IndirectCallTarget, MaxMDCount);
749bool SampleProfileLoader::tryPromoteAndInlineCandidate(
753 if (DisableSampleProfileInlining)
758 if (Opts.sample_profile_icp_max_prom == 0)
760 auto CalleeFunctionName = Candidate.CalleeSamples->
getFunction();
761 auto R = SymbolMap.find(CalleeFunctionName);
762 if (R == SymbolMap.end() || !
R->second)
765 auto &CI = *Candidate.CallInstr;
769 const char *Reason =
"Callee function not available";
776 if (!
R->second->isDeclaration() &&
R->second->getSubprogram() &&
777 R->second->hasFnAttribute(
"use-sample-profile") &&
782 Function::getGUIDAssumingExternalLinkage(
R->second->getName()),
787 CI,
R->second, Candidate.CallsiteCount, Sum,
false, ORE);
789 Sum -= Candidate.CallsiteCount;
802 Candidate.CallInstr = DI;
804 bool Inlined = tryInlineCandidate(Candidate, InlinedCallSite);
809 *DI,
static_cast<float>(Candidate.CallsiteCount) / SumOrigin);
817 Candidate.CallInstr->
getName())<<
" because "
823bool SampleProfileLoader::shouldInlineColdCallee(CallBase &CallInst) {
824 if (!ProfileSizeInline)
832 GetTTI(*CallInst.
getCaller()), GetAC, GetTLI);
843void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
844 const SmallVectorImpl<CallBase *> &Candidates,
const Function &
F,
846 for (
auto *
I : Candidates) {
847 Function *CalledFunction =
I->getCalledFunction();
848 if (CalledFunction) {
849 ORE->emit(OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(),
850 "InlineAttempt",
I->getDebugLoc(),
852 <<
"previous inlining reattempted for "
853 << (
Hot ?
"hotness: '" :
"size: '")
854 <<
ore::NV(
"Callee", CalledFunction) <<
"' into '"
860void SampleProfileLoader::findExternalInlineCandidate(
861 CallBase *CB,
const FunctionSamples *Samples,
862 DenseSet<GlobalValue::GUID> &InlinedGUIDs,
uint64_t Threshold) {
867 bool IncludeLocalTargets =
872 if (CB && getExternalInlineAdvisorShouldInline(*CB)) {
876 InlinedGUIDs.
insert(Function::getGUIDAssumingExternalLinkage(
896 if (UsePreInlinerDecision)
899 IncludeLocalTargets);
903 ContextTrieNode *
Caller = ContextTracker->getContextNodeForProfile(Samples);
904 std::queue<ContextTrieNode *> CalleeList;
905 CalleeList.push(Caller);
906 while (!CalleeList.empty()) {
907 ContextTrieNode *
Node = CalleeList.front();
909 FunctionSamples *CalleeSample =
Node->getFunctionSamples();
918 UsePreInlinerDecision &&
926 InlinedGUIDs, SymbolMap, CalleeSample->
getFunction(),
932 for (
const auto &TS : BS.second.getCallTargets())
933 if (TS.second > Threshold)
935 IncludeLocalTargets);
941 for (
auto &Child :
Node->getAllChildContext()) {
942 ContextTrieNode *CalleeNode = &Child.second;
943 CalleeList.push(CalleeNode);
970bool SampleProfileLoader::inlineHotFunctions(
971 Function &
F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
974 assert((!ProfAccForSymsInList ||
975 (!Opts.profile_sample_accurate &&
976 !
F.hasFnAttribute(
"profile-sample-accurate"))) &&
977 "ProfAccForSymsInList should be false when profile-sample-accurate "
980 MapVector<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites;
982 bool LocalChanged =
true;
983 while (LocalChanged) {
984 LocalChanged =
false;
991 const FunctionSamples *
FS =
nullptr;
994 if ((FS = findCalleeFunctionSamples(*CB))) {
996 "GUIDToFuncNameMap has to be populated");
998 if (
FS->getHeadSamplesEstimate() > 0 ||
1000 LocalNotInlinedCallSites.
insert({CB,
FS});
1003 else if (shouldInlineColdCallee(*CB))
1005 }
else if (getExternalInlineAdvisorShouldInline(*CB)) {
1011 if (
Hot || ExternalInlineAdvisor) {
1013 emitOptimizationRemarksForInlineCandidates(AllCandidates,
F,
true);
1016 emitOptimizationRemarksForInlineCandidates(ColdCandidates,
F,
false);
1019 for (CallBase *
I : CIS) {
1020 Function *CalledFunction =
I->getCalledFunction();
1021 InlineCandidate Candidate = {
I, LocalNotInlinedCallSites.
lookup(
I),
1025 if (CalledFunction == &
F)
1027 if (
I->isIndirectCall()) {
1029 for (
const auto *FS : findIndirectCallFunctionSamples(*
I, Sum)) {
1031 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1032 findExternalInlineCandidate(
I, FS, InlinedGUIDs,
1033 PSI->getOrCompHotCountThreshold());
1039 Candidate = {
I,
FS,
FS->getHeadSamplesEstimate(), 1.0};
1040 if (tryPromoteAndInlineCandidate(
F, Candidate, SumOrigin, Sum)) {
1041 LocalNotInlinedCallSites.
erase(
I);
1042 LocalChanged =
true;
1045 }
else if (CalledFunction && CalledFunction->
getSubprogram() &&
1047 if (tryInlineCandidate(Candidate)) {
1048 LocalNotInlinedCallSites.
erase(
I);
1049 LocalChanged =
true;
1051 }
else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1052 findExternalInlineCandidate(
I, findCalleeFunctionSamples(*
I),
1054 PSI->getOrCompHotCountThreshold());
1063 promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites,
F);
1067bool SampleProfileLoader::tryInlineCandidate(
1071 if (DisableSampleProfileInlining)
1074 CallBase &CB = *Candidate.CallInstr;
1076 assert(CalledFunction &&
"Expect a callee with definition");
1080 InlineCost
Cost = shouldInlineCandidate(Candidate);
1081 if (
Cost.isNever()) {
1082 ORE->emit(OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(),
1083 "InlineFail", DLoc, BB)
1084 <<
"incompatible inlining");
1091 InlineFunctionInfo IFI(GetAC);
1092 IFI.UpdateProfile =
false;
1095 if (!
IR.isSuccess())
1100 Cost,
true, getAnnotatedRemarkPassName());
1103 if (InlinedCallSites) {
1104 InlinedCallSites->
clear();
1109 ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples);
1119 if (Candidate.CallsiteDistribution < 1) {
1120 for (
auto &
I : IFI.InlinedCallSites) {
1123 Candidate.CallsiteDistribution);
1125 NumDuplicatedInlinesite++;
1131bool SampleProfileLoader::getInlineCandidate(InlineCandidate *NewCandidate,
1133 assert(CB &&
"Expect non-null call instruction");
1139 const FunctionSamples *CalleeSamples = findCalleeFunctionSamples(*CB);
1142 if (!CalleeSamples && !getExternalInlineAdvisorShouldInline(*CB))
1146 if (std::optional<PseudoProbe> Probe =
extractProbe(*CB))
1147 Factor = Probe->Factor;
1151 *NewCandidate = {CB, CalleeSamples, CallsiteCount, Factor};
1155std::optional<InlineCost>
1156SampleProfileLoader::getExternalInlineAdvisorCost(CallBase &CB) {
1157 std::unique_ptr<InlineAdvice> Advice =
nullptr;
1158 if (ExternalInlineAdvisor) {
1159 Advice = ExternalInlineAdvisor->getAdvice(CB);
1161 if (!Advice->isInliningRecommended()) {
1162 Advice->recordUnattemptedInlining();
1165 Advice->recordInlining();
1173bool SampleProfileLoader::getExternalInlineAdvisorShouldInline(CallBase &CB) {
1174 std::optional<InlineCost>
Cost = getExternalInlineAdvisorCost(CB);
1179SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
1180 if (std::optional<InlineCost> ReplayCost =
1181 getExternalInlineAdvisorCost(*Candidate.CallInstr))
1186 if (CallsitePrioritizedInline) {
1187 if (Candidate.CallsiteCount > PSI->getHotCountThreshold())
1189 else if (!ProfileSizeInline)
1195 assert(Callee &&
"Expect a definition for inline candidate of direct call");
1208 GetTTI(*Caller), GetAC, GetTLI);
1211 if (
Cost.isNever() ||
Cost.isAlways())
1223 if (UsePreInlinerDecision && Candidate.CalleeSamples) {
1236 if (!CallsitePrioritizedInline) {
1245bool SampleProfileLoader::inlineHotFunctionsWithPriority(
1246 Function &
F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
1249 assert((!ProfAccForSymsInList ||
1250 (!Opts.profile_sample_accurate &&
1251 !
F.hasFnAttribute(
"profile-sample-accurate"))) &&
1252 "ProfAccForSymsInList should be false when profile-sample-accurate "
1257 CandidateQueue CQueue;
1258 InlineCandidate NewCandidate;
1259 for (
auto &BB :
F) {
1261 if (getInlineCandidate(&NewCandidate, &CB))
1262 CQueue.push(NewCandidate);
1271 "Max inline size limit should not be smaller than min inline size "
1276 if (ExternalInlineAdvisor)
1277 SizeLimit = std::numeric_limits<unsigned>::max();
1279 MapVector<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites;
1283 while (!CQueue.empty() &&
F.getInstructionCount() <
SizeLimit) {
1284 InlineCandidate Candidate = CQueue.top();
1286 CallBase *
I = Candidate.CallInstr;
1287 Function *CalledFunction =
I->getCalledFunction();
1289 if (CalledFunction == &
F)
1291 if (
I->isIndirectCall()) {
1293 auto CalleeSamples = findIndirectCallFunctionSamples(*
I, Sum);
1295 Sum *= Candidate.CallsiteDistribution;
1296 unsigned ICPCount = 0;
1297 for (
const auto *FS : CalleeSamples) {
1299 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1300 findExternalInlineCandidate(
I, FS, InlinedGUIDs,
1301 PSI->getOrCompHotCountThreshold());
1305 FS->getHeadSamplesEstimate() * Candidate.CallsiteDistribution;
1310 if (ICPCount >= Opts.sample_profile_icp_relative_hotness_skip &&
1311 EntryCountDistributed * 100 <
1312 SumOrigin * Opts.sample_profile_icp_relative_hotness)
1321 if (!PSI->isHotCount(EntryCountDistributed))
1326 Candidate = {
I,
FS, EntryCountDistributed,
1327 Candidate.CallsiteDistribution};
1328 if (tryPromoteAndInlineCandidate(
F, Candidate, SumOrigin, Sum,
1329 &InlinedCallSites)) {
1330 for (
auto *CB : InlinedCallSites) {
1331 if (getInlineCandidate(&NewCandidate, CB))
1332 CQueue.emplace(NewCandidate);
1336 }
else if (!ContextTracker) {
1337 LocalNotInlinedCallSites.
insert({
I,
FS});
1340 }
else if (CalledFunction && CalledFunction->
getSubprogram() &&
1343 if (tryInlineCandidate(Candidate, &InlinedCallSites)) {
1344 for (
auto *CB : InlinedCallSites) {
1345 if (getInlineCandidate(&NewCandidate, CB))
1346 CQueue.emplace(NewCandidate);
1349 }
else if (!ContextTracker) {
1350 LocalNotInlinedCallSites.
insert({
I, Candidate.CalleeSamples});
1352 }
else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1353 findExternalInlineCandidate(
I, findCalleeFunctionSamples(*
I),
1355 PSI->getOrCompHotCountThreshold());
1359 if (!CQueue.empty()) {
1361 ++NumCSInlinedHitMaxLimit;
1363 ++NumCSInlinedHitMinLimit;
1365 ++NumCSInlinedHitGrowthLimit;
1371 promoteMergeNotInlinedContextSamples(LocalNotInlinedCallSites,
F);
1375void SampleProfileLoader::promoteMergeNotInlinedContextSamples(
1376 MapVector<CallBase *, const FunctionSamples *> NonInlinedCallSites,
1379 for (
const auto &Pair : NonInlinedCallSites) {
1380 CallBase *
I = Pair.first;
1382 if (!Callee ||
Callee->isDeclaration())
1386 OptimizationRemarkAnalysis(getAnnotatedRemarkPassName(),
"NotInline",
1387 I->getDebugLoc(),
I->getParent())
1388 <<
"previous inlining not repeated: '" <<
ore::NV(
"Callee", Callee)
1389 <<
"' into '" <<
ore::NV(
"Caller", &
F) <<
"'");
1392 const FunctionSamples *
FS = Pair.second;
1393 if (
FS->getTotalSamples() == 0 &&
FS->getHeadSamplesEstimate() == 0) {
1401 if (ProfileMergeInlinee) {
1407 if (
FS->getHeadSamples() == 0) {
1410 const_cast<FunctionSamples *
>(
FS)->addHeadSamples(
1411 FS->getHeadSamplesEstimate());
1416 FunctionSamples *OutlineFS = Reader->getSamplesFor(*Callee);
1420 OutlineFS = &OutlineFunctionSamples[
1422 OutlineFS->
merge(*FS, 1);
1428 notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
1429 pair.first->second.entryCount +=
FS->getHeadSamplesEstimate();
1440 InstrProfValueData{
I.first.getHashCode(),
I.second});
1447void SampleProfileLoader::generateMDProfMetadata(
Function &
F) {
1450 LLVM_DEBUG(
dbgs() <<
"\nPropagation complete. Setting branch weights\n");
1451 LLVMContext &Ctx =
F.getContext();
1453 for (
auto &BI :
F) {
1456 if (BlockWeights[BB]) {
1457 for (
auto &
I : *BB) {
1464 const DILocation *DIL = DLoc;
1465 const FunctionSamples *
FS = findFunctionSamples(
I);
1469 ErrorOr<SampleRecord::CallTargetMap>
T =
1470 FS->findCallTargetMapAt(CallSite);
1471 if (!
T ||
T.get().empty())
1478 if (Probe->Factor < 1)
1485 for (
const auto &
C :
T.get())
1492 FS->findFunctionSamplesMapAt(CallSite)) {
1493 for (
const auto &NameFS : *M)
1494 Sum += NameFS.second.getHeadSamplesEstimate();
1499 else if (Opts.overwrite_existing_weights)
1500 I.setMetadata(LLVMContext::MD_prof,
nullptr);
1503 I, ArrayRef<uint32_t>{
static_cast<uint32_t
>(BlockWeights[BB])},
1507 }
else if (Opts.overwrite_existing_weights ||
1508 Opts.profile_sample_block_accurate) {
1511 for (
auto &
I : *BB) {
1514 I.setMetadata(LLVMContext::MD_prof,
nullptr);
1532 << ((BranchLoc) ? Twine(BranchLoc.
getLine())
1533 : Twine(
"<UNKNOWN LOCATION>"))
1535 SmallVector<uint32_t, 4> Weights;
1536 uint32_t MaxWeight = 0;
1541 DenseMap<const BasicBlock *, uint64_t> EdgeMultiplicity;
1542 std::vector<uint64_t> EdgeIndex;
1547 EdgeIndex[
I] = EdgeMultiplicity[Succ];
1548 EdgeMultiplicity[Succ]++;
1553 Edge E = std::make_pair(BB, Succ);
1559 if (Weight > std::numeric_limits<uint32_t>::max()) {
1561 Weight = std::numeric_limits<uint32_t>::max();
1566 Weights.
push_back(
static_cast<uint32_t
>(
1567 Weight == std::numeric_limits<uint32_t>::max() ? Weight
1572 uint64_t W = Weight / EdgeMultiplicity[Succ];
1574 if (EdgeIndex[
I] < Weight % EdgeMultiplicity[Succ])
1576 Weights.
push_back(
static_cast<uint32_t
>(W));
1579 if (Weight > MaxWeight) {
1581 MaxDestInst = &*Succ->getFirstNonPHIOrLifetime();
1598 Opts.overwrite_existing_weights)) {
1602 return OptimizationRemark(
DEBUG_TYPE,
"PopularDest", MaxDestInst)
1603 <<
"most popular destination for conditional branches at "
1604 <<
ore::NV(
"CondBranchesLoc", BranchLoc);
1607 if (Opts.overwrite_existing_weights) {
1623bool SampleProfileLoader::emitAnnotations(
Function &
F) {
1628 if (!ProbeManager->getDesc(
F))
1629 dbgs() <<
"Probe descriptor missing for Function " <<
F.getName()
1633 if (ProbeManager->profileIsValid(
F, *Samples)) {
1634 ++NumMatchedProfile;
1636 ++NumMismatchedProfile;
1638 dbgs() <<
"Profile is invalid due to CFG mismatch for Function "
1639 <<
F.getName() <<
"\n");
1640 if (!SalvageStaleProfile)
1644 if (getFunctionLoc(
F) == 0)
1648 <<
F.getName() <<
": " << getFunctionLoc(
F) <<
"\n");
1651 DenseSet<GlobalValue::GUID> InlinedGUIDs;
1652 if (CallsitePrioritizedInline)
1653 Changed |= inlineHotFunctionsWithPriority(
F, InlinedGUIDs);
1655 Changed |= inlineHotFunctions(
F, InlinedGUIDs);
1657 Changed |= computeAndPropagateWeights(
F, InlinedGUIDs);
1660 generateMDProfMetadata(
F);
1662 emitCoverageRemarks(
F);
1666std::unique_ptr<ProfiledCallGraph>
1667SampleProfileLoader::buildProfiledCallGraph(
Module &M) {
1668 std::unique_ptr<ProfiledCallGraph> ProfiledCG;
1670 ProfiledCG = std::make_unique<ProfiledCallGraph>(*ContextTracker);
1672 ProfiledCG = std::make_unique<ProfiledCallGraph>(Reader->getProfiles());
1680 ProfiledCG->addProfiledFunction(
1687std::vector<Function *>
1688SampleProfileLoader::buildFunctionOrder(
Module &M, LazyCallGraph &CG) {
1689 std::vector<Function *> FunctionOrderList;
1690 FunctionOrderList.reserve(
M.size());
1692 if (!Opts.sample_profile_top_down_load && Opts.use_profiled_call_graph)
1693 errs() <<
"WARNING: -use-profiled-call-graph ignored, should be used "
1694 "together with -sample-profile-top-down-load.\n";
1696 if (!Opts.sample_profile_top_down_load) {
1697 if (ProfileMergeInlinee) {
1703 ProfileMergeInlinee =
false;
1708 FunctionOrderList.push_back(&
F);
1709 return FunctionOrderList;
1712 if (Opts.use_profiled_call_graph) {
1761 std::unique_ptr<ProfiledCallGraph> ProfiledCG = buildProfiledCallGraph(M);
1762 scc_iterator<ProfiledCallGraph *> CGI =
scc_begin(ProfiledCG.get());
1767 scc_member_iterator<ProfiledCallGraph *>
SI(*CGI);
1770 for (
auto *Node :
Range) {
1773 FunctionOrderList.push_back(
F);
1777 std::reverse(FunctionOrderList.begin(), FunctionOrderList.end());
1782 dbgs() <<
"Function processing order:\n";
1783 for (
auto F : FunctionOrderList) {
1784 dbgs() <<
F->getName() <<
"\n";
1788 return FunctionOrderList;
1791bool SampleProfileLoader::doInitialization(
Module &M,
1793 auto &Ctx =
M.getContext();
1796 Filename, Ctx, *FS, FSDiscriminatorPass::Base, RemappingFilename);
1797 if (std::error_code EC = ReaderOrErr.getError()) {
1798 std::string
Msg =
"Could not open profile: " +
EC.message();
1802 Reader = std::move(ReaderOrErr.get());
1803 Reader->setSkipFlatProf(LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink);
1806 Reader->setModule(&M);
1807 if (std::error_code EC = Reader->read()) {
1808 std::string
Msg =
"profile reading failed: " +
EC.message();
1813 PSL = Reader->getProfileSymbolList();
1815 DisableSampleProfileInlining =
1816 !
valueOr(Opts.sample_loader_inlining, !DisableSampleProfileInlining);
1818 if (UseFlattenedProfile)
1820 Reader->profileIsCS());
1823 ProfAccForSymsInList = Opts.profile_accurate_for_symsinlist && PSL &&
1824 !Opts.profile_sample_accurate;
1825 if (ProfAccForSymsInList)
1826 CoverageTracker.setProfAccForSymsInList(
true);
1828 if (
FAM && !Opts.sample_profile_inline_replay.
empty()) {
1830 M, *
FAM, Ctx,
nullptr,
1831 ReplayInlinerSettings{Opts.sample_profile_inline_replay,
1832 Opts.sample_profile_inline_replay_scope,
1833 Opts.sample_profile_inline_replay_fallback,
1834 {Opts.sample_profile_inline_replay_format}},
1836 InlineContext{LTOPhase, InlinePass::ReplaySampleProfileInliner});
1840 if (Reader->profileIsCS() || Reader->profileIsPreInlined() ||
1841 Reader->profileIsProbeBased()) {
1849 ProfileSizeInline =
valueOr(Opts.sample_profile_inline_size,
true);
1850 CallsitePrioritizedInline =
1851 valueOr(Opts.sample_profile_prioritized_inline,
true);
1853 AllowRecursiveInline =
valueOr(Opts.sample_profile_recursive_inline,
true);
1855 if (Reader->profileIsPreInlined())
1856 UsePreInlinerDecision =
valueOr(Opts.sample_profile_use_preinliner,
true);
1863 if (Reader->profileIsProbeBased()) {
1864 SalvageStaleProfile =
valueOr(Opts.salvage_stale_profile,
true);
1865 SalvageUnusedProfile =
valueOr(Opts.salvage_unused_profile,
true);
1868 if (!Reader->profileIsCS()) {
1880 if (Reader->profileIsCS()) {
1882 ContextTracker = std::make_unique<SampleContextTracker>(
1883 Reader->getProfiles(), &GUIDToFuncNameMap);
1887 if (Reader->profileIsProbeBased()) {
1888 ProbeManager = std::make_unique<PseudoProbeManager>(M);
1889 if (!ProbeManager->moduleIsProbed(M)) {
1891 "Pseudo-probe-based profile requires SampleProfileProbePass";
1892 Ctx.
diagnose(DiagnosticInfoSampleProfile(
M.getModuleIdentifier(),
Msg,
1898 if (Opts.report_profile_staleness || Opts.persist_profile_staleness ||
1899 SalvageStaleProfile) {
1900 MatchingManager = std::make_unique<SampleProfileMatcher>(
1901 Opts, M, *Reader, CG, ProbeManager.get(), LTOPhase, SymbolMap, PSL,
1902 FuncNameToProfNameMap, SalvageStaleProfile, SalvageUnusedProfile);
1918bool SampleProfileLoader::rejectHighStalenessProfile(
1919 Module &M, ProfileSummaryInfo *PSI,
const SampleProfileMap &Profiles) {
1921 "Only support for probe-based profile");
1924 for (
const auto &
I : Profiles) {
1925 const auto &
FS =
I.second;
1926 const auto *FuncDesc = ProbeManager->getDesc(
FS.getGUID());
1932 FS.getTotalSamples()))
1936 if (ProbeManager->profileIsHashMismatched(*FuncDesc, FS) &&
1937 !ProbeManager->probeFromWeakSymbol(
FS.getGUID()))
1938 NumMismatchedFunc++;
1942 if (TotalHotFunc < Opts.min_functions_for_staleness_error)
1946 if (NumMismatchedFunc * 100 >=
1947 TotalHotFunc * Opts.percent_mismatch_for_staleness_error) {
1948 auto &Ctx =
M.getContext();
1950 "The input profile significantly mismatches current source code. "
1951 "Please recollect profile to avoid performance regression.";
1952 Ctx.
diagnose(DiagnosticInfoSampleProfile(
M.getModuleIdentifier(),
Msg));
1958void SampleProfileLoader::removePseudoProbeInstsDiscriminator(
Module &M) {
1960 std::vector<Instruction *> InstsToDel;
1961 for (
auto &BB :
F) {
1962 for (
auto &
I : BB) {
1964 InstsToDel.push_back(&
I);
1966 if (
const DILocation *DIL =
I.getDebugLoc().get()) {
1970 std::optional<uint32_t> DwarfDiscriminator =
1979 for (
auto *
I : InstsToDel)
1980 I->eraseFromParent();
1985 ProfileSummaryInfo *_PSI) {
1986 GUIDToFuncNameMapper
Mapper(M, *Reader, GUIDToFuncNameMap);
1989 if (
M.getProfileSummary(
false) ==
nullptr) {
1990 M.setProfileSummary(Reader->getSummary().getMD(
M.getContext()),
1996 rejectHighStalenessProfile(M, PSI, Reader->getProfiles()))
1999 auto Remapper = Reader->getRemapper();
2001 for (
const auto &N_F :
M.getValueSymbolTable()) {
2002 StringRef OrigName = N_F.getKey();
2004 if (
F ==
nullptr || OrigName.
empty())
2006 SymbolMap[FunctionId(OrigName)] =
F;
2008 if (OrigName != NewName && !NewName.
empty()) {
2009 auto r = SymbolMap.emplace(FunctionId(NewName),
F);
2015 r.first->second =
nullptr;
2020 if (
auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
2021 if (*MapName != OrigName && !MapName->empty())
2022 SymbolMap.emplace(FunctionId(*MapName),
F);
2028 if (Opts.report_profile_staleness || Opts.persist_profile_staleness ||
2029 SalvageStaleProfile) {
2030 MatchingManager->runOnModule();
2031 MatchingManager->clearMatchingData();
2033 assert(SymbolMap.count(FunctionId()) == 0 &&
2034 "No empty StringRef should be added in SymbolMap");
2035 assert((SalvageUnusedProfile || FuncNameToProfNameMap.empty()) &&
2036 "FuncNameToProfNameMap is not empty when --salvage-unused-profile is "
2039 bool retval =
false;
2040 for (
auto *
F : buildFunctionOrder(M, CG)) {
2042 clearFunctionData();
2048 for (
const auto &[Fn, Info] : notInlinedCallInfo)
2051 if (Opts.sample_profile_remove_probe &&
2053 removePseudoProbeInstsDiscriminator(M);
2055 M.eraseNamedMetadata(FuncInfo);
2061bool SampleProfileLoader::runOnFunction(
Function &
F,
2063 LLVM_DEBUG(
dbgs() <<
"\n\nProcessing Function " <<
F.getName() <<
"\n");
2064 DILocation2SampleMap.clear();
2071 ProfAccForSymsInList = Opts.profile_accurate_for_symsinlist && PSL;
2072 if (Opts.profile_sample_accurate ||
2073 F.hasFnAttribute(
"profile-sample-accurate")) {
2076 initialEntryCount = 0;
2079 ProfAccForSymsInList =
false;
2081 CoverageTracker.setProfAccForSymsInList(ProfAccForSymsInList);
2087 if (ProfAccForSymsInList) {
2089 if (PSL->contains(
F.getName()))
2090 initialEntryCount = 0;
2105 Function::getGUIDAssumingExternalLinkage(CanonName))
2106 : Reader->contains(CanonName))
2107 initialEntryCount = -1;
2112 if (!
F.getEntryCount())
2113 F.setEntryCount(initialEntryCount);
2119 Samples = ContextTracker->getBaseSamplesFor(
F);
2121 Samples = Reader->getSamplesFor(
F);
2126 auto It = OutlineFunctionSamples.find(FunctionId(CanonName));
2127 if (It != OutlineFunctionSamples.end()) {
2128 Samples = &It->second;
2129 }
else if (
auto Remapper = Reader->getRemapper()) {
2130 if (
auto RemppedName = Remapper->lookUpNameInProfile(CanonName)) {
2131 It = OutlineFunctionSamples.find(FunctionId(*RemppedName));
2132 if (It != OutlineFunctionSamples.end())
2133 Samples = &It->second;
2139 if (Samples && !Samples->
empty())
2140 return emitAnnotations(
F);
2150 bool UseFlattenedProfile)
2151 : ProfileFileName(File), ProfileRemappingFileName(RemappingFile),
2152 LTOPhase(LTOPhase), FS(
std::
move(FS)),
2153 DisableSampleProfileInlining(DisableSampleProfileInlining),
2154 UseFlattenedProfile(UseFlattenedProfile) {}
2176 SampleProfileLoader SampleLoader(
2178 ProfileFileName.empty() ? Opts.sample_profile_file
2180 ProfileRemappingFileName.
empty() ? Opts.sample_profile_remapping_file
2182 LTOPhase, FS, GetAssumptionCache, GetTTI, GetTLI, CG,
2183 DisableSampleProfileInlining, UseFlattenedProfile);
2184 if (!SampleLoader.doInitialization(M, &
FAM))
2188 if (!SampleLoader.runOnModule(M, AM, PSI))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Function Alias Analysis false
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
static bool runOnFunction(Function &F, bool PostInlining)
Provides ErrorOr<T> smart pointer.
static cl::opt< unsigned > SizeLimit("eif-limit", cl::init(6), cl::Hidden, cl::desc("Size limit in Hexagon early if-conversion"))
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
static LVReader * CurrentReader
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
static const Function * getCalledFunction(const Value *V)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
static constexpr StringLiteral Filename
FunctionAnalysisManager FAM
This file defines the PriorityQueue class.
This file contains the declarations for profiling metadata utility functions.
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
std::pair< BasicBlock *, BasicBlock * > Edge
This file provides the interface for context-sensitive profile tracker used by CSSPGO.
This file provides the interface for the sampled PGO profile loader base implementation.
This file provides the utility functions for the sampled PGO loader base implementation.
This file provides the interface for SampleProfileMatcher.
This file provides the interface for the pseudo probe implementation for AutoFDO.
static cl::opt< bool > PreserveDeferredInlines("sample-profile-preserve-deferred-inlines", cl::Hidden, cl::init(true), cl::desc("In ThinLTO pre-link, add import hints for in-module targets of " "deferred indirect calls so they can be imported with the caller."))
static void updateIDTMetaData(const IPOOptions &Opts, Instruction &Inst, const SmallVectorImpl< InstrProfValueData > &CallTargets, uint64_t Sum)
Update indirect call target profile metadata for Inst.
static bool doesHistoryAllowICP(const IPOOptions &Opts, const Instruction &Inst, StringRef Candidate)
Check whether the indirect call promotion history of Inst allows the promotion for Candidate.
static SmallVector< InstrProfValueData, 2 > GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M)
Returns the sorted CallTargetMap M by count in descending order.
This file provides the interface for the sampled PGO loader pass.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Defines the virtual file system interface vfs::FileSystem.
bool empty() const
Returns true if the analysis manager has an empty results cache.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
static bool isPseudoProbeDiscriminator(unsigned Discriminator)
const DILocation * cloneWithDiscriminator(unsigned Discriminator) const
Returns a new DILocation with updated Discriminator.
LLVM_ABI unsigned getLine() const
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Represents either an error or a value T.
DISubprogram * getSubprogram() const
Get the attached subprogram.
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
static InlineCost getNever(const char *Reason, std::optional< CostBenefitPair > CostBenefit=std::nullopt)
static InlineCost getAlways(const char *Reason, std::optional< CostBenefitPair > CostBenefit=std::nullopt)
static InlineCost get(int Cost, int Threshold, int StaticBonus=0)
LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI bool extractProfTotalWeight(uint64_t &TotalVal) const
Retrieve total raw weight values of a branch.
LLVM_ABI BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
This is an important class for using LLVM in a threaded context.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
An analysis pass which computes the call graph for a module.
A lazily constructed view of the call graph of a module.
ValueT lookup(const KeyT &Key) const
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
A Module instance is used to store all the information related to an LLVM module.
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
LLVM_ABI void refresh(std::unique_ptr< ProfileSummary > &&Other=nullptr)
If a summary is provided as argument, use that.
LLVM_ABI bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered hot with regard to a given hot percentile cutoff value.
void computeDominanceAndLoopInfo(FunctionT &F)
PostDominatorTreePtrT PDT
static LLVM_ABI StringRef getSampleProfileFile(const LLVMContext &C)
Return -sample-profile-file.
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
LLVM_ABI SampleProfileLoaderPass(std::string File="", std::string RemappingFile="", ThinOrFullLTOPhase LTOPhase=ThinOrFullLTOPhase::None, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr, bool DisableSampleProfileInlining=false, bool UseFlattenedProfile=false)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
constexpr bool empty() const
Check if the string is empty.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVM Value Representation.
LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
int getNumOccurrences() const
std::pair< iterator, bool > insert(const ValueT &V)
const ParentTy * getParent() const
Representation of the samples collected for a function.
void findInlinedFunctions(DenseSet< GlobalValue::GUID > &S, const HashKeyMap< DenseMap, FunctionId, Function * > &SymbolMap, uint64_t Threshold, bool IncludeLocalTargets=false) const
Recursively collect import GUIDs from contexts with total samples greater than Threshold,...
static LLVM_ABI std::atomic< bool > UseMD5
Whether the profile uses MD5 to represent string.
FunctionId getFunction() const
Return the function name.
SampleContext & getContext() const LLVM_LIFETIME_BOUND
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
static void addImportGUID(DenseSet< GlobalValue::GUID > &S, const HashKeyMap< DenseMap, FunctionId, Function * > &SymbolMap, FunctionId Name, bool IncludeLocal)
Add the import GUID of profiled function Name to S.
void setContextSynthetic()
static LLVM_ABI std::atomic< bool > ProfileIsProbeBased
const BodySampleMap & getBodySamples() const LLVM_LIFETIME_BOUND
Return all the samples collected in the body of the function.
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight=1)
Merge the samples in Other into this one.
static LLVM_ABI std::atomic< bool > ProfileIsCS
static LLVM_ABI LineLocation getCallSiteIdentifier(const DILocation *DIL, bool ProfileIsFS=false)
Returns a unique call site identifier for a given debug location of a call instruction.
uint64_t getHeadSamplesEstimate() const
Return an estimate of the sample count of the function entry basic block.
uint64_t getGUID() const
Return the GUID of the context's name.
static void flattenProfile(SampleProfileMap &ProfileMap, bool ProfileIsCS=false)
bool hasAttribute(ContextAttributeMask A)
static LLVM_ABI ErrorOr< std::unique_ptr< SampleProfileReader > > create(StringRef Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, StringRef RemapFilename="")
Create a sample profile reader appropriate to the file format.
static SortedCallTargetSet sortCallTargets(const CallTargetMap &Targets)
Sort call targets in descending order of call frequency.
static const CallTargetMap adjustCallTargets(const CallTargetMap &Targets, float DistributionFactor)
Prorate call targets by a distribution factor.
SortedVectorMap< FunctionId, uint64_t, 0 > CallTargetMap
bool isAtEnd() const
Direct loop termination test which is more efficient than comparison with end().
@ BasicBlock
Various leaf nodes.
initializer< Ty > init(const Ty &Val)
LLVM_ABI void checkExpectAnnotations(const Instruction &I, ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)
checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...
DiagnosticInfoOptimizationBase::Argument NV
LLVM_ABI CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)
NodeAddr< NodeBase * > Node
NodeAddr< FuncNode * > Func
static FunctionId getRepInFormat(StringRef Name)
Get the proper representation of a string according to whether the current Format uses MD5 to represe...
@ ContextDuplicatedIntoBase
std::map< FunctionId, FunctionSamples > FunctionSamplesMap
LLVM_ABI bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI, bool ProfAccForSymsInList)
Return true if the given callsite is hot wrt to hot cutoff threshold.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
LLVM_ABI IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
static bool isIndirectCall(const MachineInstr &MI)
LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
bool succ_empty(const Instruction *I)
LLVM_ABI InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CallerTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr, function_ref< EphemeralValuesCache &(Function &)> GetEphValuesCache=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
LLVM_ABI InlineParams getInlineParams(const LLVMContext &C)
Generate the parameters to tune the inline cost analysis based only on the commandline options.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
LLVM_ABI cl::opt< int > SampleHotCallSiteThreshold
static void buildTopDownFuncOrder(LazyCallGraph &CG, std::vector< Function * > &FunctionOrderList)
LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, bool TrackInlineHistory=false, Function *ForwardVarArgsTo=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
This function inlines the called function into the basic block of the caller.
LLVM_ABI void setProbeDistributionFactor(Instruction &Inst, float Factor)
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
LLVM_ABI cl::opt< int > ProfileInlineGrowthLimit
auto make_isa_range(RangeT &&Range)
Return a range over Range containing only elements for which isa<T> holds, casting each of them to T.
LLVM_ABI std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
LLVM_ABI cl::opt< bool > SampleProfileUseProfi
const IPOOptions & getIPOOptions(const LLVMContext &C)
LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI llvm::cl::opt< bool > UseIterativeBFIInference
LLVM_ABI std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
LLVM_ABI cl::opt< int > ProfileInlineLimitMax
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI void updateProfileCallee(Function *Callee, int64_t EntryDelta, const ValueMap< const Value *, WeakTrackingVH > *VMap=nullptr)
Updates profile information by adjusting the entry count by adding EntryDelta then scaling callsite i...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
LLVM_ABI cl::opt< int > SampleColdCallSiteThreshold
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
static bool skipProfileForFunction(const Function &F)
LLVM_ABI cl::opt< bool > SortProfiledSCC
constexpr bool valueOr(BoolOrDefault X, bool Default)
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
cl::opt< bool > EnableExtTspBlockPlacement
const uint64_t NOMORE_ICP_MAGICNUM
Magic number in the value profile metadata showing a target has been promoted for the instruction and...
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
LLVM_ABI cl::opt< int > ProfileInlineLimitMin
constexpr const char * PseudoProbeDescMetadataName
Implement std::hash so that hash_code can be used in STL containers.
A wrapper of binary function with basic blocks and jumps.
std::optional< bool > AllowRecursiveCall
Indicate whether we allow inlining for recursive call.
std::optional< bool > ComputeFullInlineCost
Compute inline cost even when the cost has exceeded the threshold.
static std::optional< uint32_t > extractDwarfBaseDiscriminator(uint32_t Value)