LLVM 17.0.0git
ProfileSummaryInfo.cpp
Go to the documentation of this file.
1//===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that provides access to the global profile summary
10// information.
11//
12//===----------------------------------------------------------------------===//
13
16#include "llvm/IR/BasicBlock.h"
18#include "llvm/IR/Module.h"
23#include <optional>
24using namespace llvm;
25
26// Knobs for profile summary based thresholds.
27namespace llvm {
34} // namespace llvm
35
37 "partial-profile", cl::Hidden, cl::init(false),
38 cl::desc("Specify the current profile is used as a partial profile."));
39
40// TODO: Remove this support completely after ensuring that disabling by
41// default has no unexpected effects. This causes the global number of basic
42// blocks to be recorded in the ThinLTO summary, which breaks caching in the
43// distributed ThinLTO case.
45 "scale-partial-sample-profile-working-set-size", cl::Hidden,
46 cl::init(false),
48 "If true, scale the working set size of the partial sample profile "
49 "by the partial profile ratio to reflect the size of the program "
50 "being compiled."));
51
53 "partial-sample-profile-working-set-size-scale-factor", cl::Hidden,
54 cl::init(0.008),
55 cl::desc("The scale factor used to scale the working set size of the "
56 "partial sample profile along with the partial profile ratio. "
57 "This includes the factor of the profile counter per block "
58 "and the factor to scale the working set size to use the same "
59 "shared thresholds as PGO."));
60
61// The profile summary metadata may be attached either by the frontend or by
62// any backend passes (IR level instrumentation, for example). This method
63// checks if the Summary is null and if so checks if the summary metadata is now
64// available in the module and parses it to get the Summary object.
67 return;
68 // First try to get context sensitive ProfileSummary.
69 auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);
70 if (SummaryMD)
71 Summary.reset(ProfileSummary::getFromMD(SummaryMD));
72
73 if (!hasProfileSummary()) {
74 // This will actually return PSK_Instr or PSK_Sample summary.
75 SummaryMD = M->getProfileSummary(/* IsCS */ false);
76 if (SummaryMD)
77 Summary.reset(ProfileSummary::getFromMD(SummaryMD));
78 }
79 if (!hasProfileSummary())
80 return;
81 computeThresholds();
82}
83
85 const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const {
86 assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) &&
87 "We can only get profile count for call/invoke instruction.");
88 if (hasSampleProfile()) {
89 // In sample PGO mode, check if there is a profile metadata on the
90 // instruction. If it is present, determine hotness solely based on that,
91 // since the sampled entry count may not be accurate. If there is no
92 // annotated on the instruction, return std::nullopt.
93 uint64_t TotalCount;
94 if (Call.extractProfTotalWeight(TotalCount))
95 return TotalCount;
96 return std::nullopt;
97 }
98 if (BFI)
99 return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic);
100 return std::nullopt;
101}
102
103/// Returns true if the function's entry is hot. If it returns false, it
104/// either means it is not hot or it is unknown whether it is hot or not (for
105/// example, no profile data is available).
107 if (!F || !hasProfileSummary())
108 return false;
109 auto FunctionCount = F->getEntryCount();
110 // FIXME: The heuristic used below for determining hotness is based on
111 // preliminary SPEC tuning for inliner. This will eventually be a
112 // convenience method that calls isHotCount.
113 return FunctionCount && isHotCount(FunctionCount->getCount());
114}
115
116/// Returns true if the function contains hot code. This can include a hot
117/// function entry count, hot basic block, or (in the case of Sample PGO)
118/// hot total call edge count.
119/// If it returns false, it either means it is not hot or it is unknown
120/// (for example, no profile data is available).
122 const Function *F, BlockFrequencyInfo &BFI) const {
123 if (!F || !hasProfileSummary())
124 return false;
125 if (auto FunctionCount = F->getEntryCount())
126 if (isHotCount(FunctionCount->getCount()))
127 return true;
128
129 if (hasSampleProfile()) {
130 uint64_t TotalCallCount = 0;
131 for (const auto &BB : *F)
132 for (const auto &I : BB)
133 if (isa<CallInst>(I) || isa<InvokeInst>(I))
134 if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr))
135 TotalCallCount += *CallCount;
136 if (isHotCount(TotalCallCount))
137 return true;
138 }
139 for (const auto &BB : *F)
140 if (isHotBlock(&BB, &BFI))
141 return true;
142 return false;
143}
144
145/// Returns true if the function only contains cold code. This means that
146/// the function entry and blocks are all cold, and (in the case of Sample PGO)
147/// the total call edge count is cold.
148/// If it returns false, it either means it is not cold or it is unknown
149/// (for example, no profile data is available).
151 const Function *F, BlockFrequencyInfo &BFI) const {
152 if (!F || !hasProfileSummary())
153 return false;
154 if (auto FunctionCount = F->getEntryCount())
155 if (!isColdCount(FunctionCount->getCount()))
156 return false;
157
158 if (hasSampleProfile()) {
159 uint64_t TotalCallCount = 0;
160 for (const auto &BB : *F)
161 for (const auto &I : BB)
162 if (isa<CallInst>(I) || isa<InvokeInst>(I))
163 if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr))
164 TotalCallCount += *CallCount;
165 if (!isColdCount(TotalCallCount))
166 return false;
167 }
168 for (const auto &BB : *F)
169 if (!isColdBlock(&BB, &BFI))
170 return false;
171 return true;
172}
173
175 assert(hasPartialSampleProfile() && "Expect partial sample profile");
176 return !F.getEntryCount();
177}
178
179template <bool isHot>
180bool ProfileSummaryInfo::isFunctionHotOrColdInCallGraphNthPercentile(
181 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const {
182 if (!F || !hasProfileSummary())
183 return false;
184 if (auto FunctionCount = F->getEntryCount()) {
185 if (isHot &&
186 isHotCountNthPercentile(PercentileCutoff, FunctionCount->getCount()))
187 return true;
188 if (!isHot &&
189 !isColdCountNthPercentile(PercentileCutoff, FunctionCount->getCount()))
190 return false;
191 }
192 if (hasSampleProfile()) {
193 uint64_t TotalCallCount = 0;
194 for (const auto &BB : *F)
195 for (const auto &I : BB)
196 if (isa<CallInst>(I) || isa<InvokeInst>(I))
197 if (auto CallCount = getProfileCount(cast<CallBase>(I), nullptr))
198 TotalCallCount += *CallCount;
199 if (isHot && isHotCountNthPercentile(PercentileCutoff, TotalCallCount))
200 return true;
201 if (!isHot && !isColdCountNthPercentile(PercentileCutoff, TotalCallCount))
202 return false;
203 }
204 for (const auto &BB : *F) {
205 if (isHot && isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI))
206 return true;
207 if (!isHot && !isColdBlockNthPercentile(PercentileCutoff, &BB, &BFI))
208 return false;
209 }
210 return !isHot;
211}
212
213// Like isFunctionHotInCallGraph but for a given cutoff.
215 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const {
216 return isFunctionHotOrColdInCallGraphNthPercentile<true>(
217 PercentileCutoff, F, BFI);
218}
219
221 int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const {
222 return isFunctionHotOrColdInCallGraphNthPercentile<false>(
223 PercentileCutoff, F, BFI);
224}
225
226/// Returns true if the function's entry is a cold. If it returns false, it
227/// either means it is not cold or it is unknown whether it is cold or not (for
228/// example, no profile data is available).
230 if (!F)
231 return false;
232 if (F->hasFnAttribute(Attribute::Cold))
233 return true;
234 if (!hasProfileSummary())
235 return false;
236 auto FunctionCount = F->getEntryCount();
237 // FIXME: The heuristic used below for determining coldness is based on
238 // preliminary SPEC tuning for inliner. This will eventually be a
239 // convenience method that calls isHotCount.
240 return FunctionCount && isColdCount(FunctionCount->getCount());
241}
242
243/// Compute the hot and cold thresholds.
244void ProfileSummaryInfo::computeThresholds() {
245 auto &DetailedSummary = Summary->getDetailedSummary();
247 DetailedSummary, ProfileSummaryCutoffHot);
248 HotCountThreshold =
250 ColdCountThreshold =
252 assert(ColdCountThreshold <= HotCountThreshold &&
253 "Cold count threshold cannot exceed hot count threshold!");
255 HasHugeWorkingSetSize =
257 HasLargeWorkingSetSize =
259 } else {
260 // Scale the working set size of the partial sample profile to reflect the
261 // size of the program being compiled.
262 double PartialProfileRatio = Summary->getPartialProfileRatio();
263 uint64_t ScaledHotEntryNumCounts =
264 static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio *
266 HasHugeWorkingSetSize =
267 ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
268 HasLargeWorkingSetSize =
269 ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
270 }
271}
272
273std::optional<uint64_t>
274ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {
275 if (!hasProfileSummary())
276 return std::nullopt;
277 auto iter = ThresholdCache.find(PercentileCutoff);
278 if (iter != ThresholdCache.end()) {
279 return iter->second;
280 }
281 auto &DetailedSummary = Summary->getDetailedSummary();
282 auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,
284 uint64_t CountThreshold = Entry.MinCount;
285 ThresholdCache[PercentileCutoff] = CountThreshold;
286 return CountThreshold;
287}
288
290 return HasHugeWorkingSetSize && *HasHugeWorkingSetSize;
291}
292
294 return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;
295}
296
298 return HotCountThreshold && C >= *HotCountThreshold;
299}
300
302 return ColdCountThreshold && C <= *ColdCountThreshold;
303}
304
305template <bool isHot>
306bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff,
307 uint64_t C) const {
308 auto CountThreshold = computeThreshold(PercentileCutoff);
309 if (isHot)
310 return CountThreshold && C >= *CountThreshold;
311 else
312 return CountThreshold && C <= *CountThreshold;
313}
314
316 uint64_t C) const {
317 return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C);
318}
319
321 uint64_t C) const {
322 return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C);
323}
324
326 return HotCountThreshold.value_or(UINT64_MAX);
327}
328
330 return ColdCountThreshold.value_or(0);
331}
332
334 BlockFrequencyInfo *BFI) const {
335 auto Count = BFI->getBlockProfileCount(BB);
336 return Count && isHotCount(*Count);
337}
338
340 BlockFrequencyInfo *BFI) const {
341 auto Count = BFI->getBlockProfileCount(BB);
342 return Count && isColdCount(*Count);
343}
344
345template <bool isHot>
346bool ProfileSummaryInfo::isHotOrColdBlockNthPercentile(
347 int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const {
348 auto Count = BFI->getBlockProfileCount(BB);
349 if (isHot)
350 return Count && isHotCountNthPercentile(PercentileCutoff, *Count);
351 else
352 return Count && isColdCountNthPercentile(PercentileCutoff, *Count);
353}
354
356 int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const {
357 return isHotOrColdBlockNthPercentile<true>(PercentileCutoff, BB, BFI);
358}
359
361 int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const {
362 return isHotOrColdBlockNthPercentile<false>(PercentileCutoff, BB, BFI);
363}
364
366 BlockFrequencyInfo *BFI) const {
367 auto C = getProfileCount(CB, BFI);
368 return C && isHotCount(*C);
369}
370
372 BlockFrequencyInfo *BFI) const {
373 auto C = getProfileCount(CB, BFI);
374 if (C)
375 return isColdCount(*C);
376
377 // In SamplePGO, if the caller has been sampled, and there is no profile
378 // annotated on the callsite, we consider the callsite as cold.
379 return hasSampleProfile() && CB.getCaller()->hasProfileData();
380}
381
383 return hasProfileSummary() &&
384 Summary->getKind() == ProfileSummary::PSK_Sample &&
385 (PartialProfile || Summary->isPartialProfile());
386}
387
389 "Profile summary info", false, true)
390
392 : ImmutablePass(ID) {
394}
395
397 PSI.reset(new ProfileSummaryInfo(M));
398 return false;
399}
400
402 PSI.reset();
403 return false;
404}
405
406AnalysisKey ProfileSummaryAnalysis::Key;
409 return ProfileSummaryInfo(M);
410}
411
415
416 OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
417 for (auto &F : M) {
418 OS << F.getName();
419 if (PSI.isFunctionEntryHot(&F))
420 OS << " :hot entry ";
421 else if (PSI.isFunctionEntryCold(&F))
422 OS << " :cold entry ";
423 OS << "\n";
424 }
425 return PreservedAnalyses::all();
426}
427
static cl::opt< unsigned > CountThreshold("hexagon-cext-threshold", cl::init(3), cl::Hidden, cl::desc("Minimum number of extenders to trigger replacement"))
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
static cl::opt< unsigned > PercentileCutoff("mfs-psi-cutoff", cl::desc("Percentile profile summary cutoff used to " "determine cold blocks. Unused if set to zero."), cl::init(999950), cl::Hidden)
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize
Module.h This file contains the declarations for the Module class.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize("scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(false), cl::desc("If true, scale the working set size of the partial sample profile " "by the partial profile ratio to reflect the size of the program " "being compiled."))
static cl::opt< bool > PartialProfile("partial-profile", cl::Hidden, cl::init(false), cl::desc("Specify the current profile is used as a partial profile."))
static cl::opt< double > PartialSampleProfileWorkingSetSizeScaleFactor("partial-sample-profile-working-set-size-scale-factor", cl::Hidden, cl::init(0.008), cl::desc("The scale factor used to scale the working set size of the " "partial sample profile along with the partial profile ratio. " "This includes the factor of the profile counter per block " "and the factor to scale the working set size to use the same " "shared thresholds as PGO."))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1190
Function * getCaller()
Helper to get the caller (the parent function).
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
bool hasProfileData(bool IncludeSynthetic=false) const
Return true if the function is annotated with profile data.
Definition: Function.h:289
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition: Pass.h:282
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Metadata * getProfileSummary(bool IsCS) const
Returns profile summary metadata.
Definition: Module.cpp:643
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Result run(Module &M, ModuleAnalysisManager &)
static const ProfileSummaryEntry & getEntryForPercentile(const SummaryEntryVector &DS, uint64_t Percentile)
Find the summary entry for a desired percentile of counts.
static uint64_t getHotCountThreshold(const SummaryEntryVector &DS)
static uint64_t getColdCountThreshold(const SummaryEntryVector &DS)
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool doInitialization(Module &M) override
doInitialization - Virtual method overridden by subclasses to do any necessary initialization before ...
Analysis providing profile information.
bool isFunctionEntryHot(const Function *F) const
Returns true if F has hot function entry.
bool isHotBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered hot with regard to a given hot percentile cutoff value.
uint64_t getOrCompColdCountThreshold() const
Returns ColdCountThreshold if set.
bool hasProfileSummary() const
Returns true if profile summary is available.
bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered hot.
bool isFunctionHotnessUnknown(const Function &F) const
Returns true if the hotness of F is unknown.
void refresh()
If no summary is present, attempt to refresh.
bool isColdBlockNthPercentile(int PercentileCutoff, const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered cold with regard to a given cold percentile cutoff value.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
bool hasSampleProfile() const
Returns true if module M has sample profile.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool isColdCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered cold with regard to a given cold percentile cutoff value.
bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C) const
Returns true if count C is considered hot with regard to a given hot percentile cutoff value.
bool isFunctionColdInCallGraphNthPercentile(int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains cold code with regard to a given cold percentile cutoff value.
bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
bool hasLargeWorkingSetSize() const
Returns true if the working set size of the code is considered large.
bool isColdCallSite(const CallBase &CB, BlockFrequencyInfo *BFI) const
Returns true if call site CB is considered cold.
bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains hot code with regard to a given hot percentile cutoff value.
bool isFunctionHotInCallGraph(const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains hot code.
bool isHotCallSite(const CallBase &CB, BlockFrequencyInfo *BFI) const
Returns true if the call site CB is considered hot.
bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) const
Returns true if BasicBlock BB is considered cold.
bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI) const
Returns true if F contains only cold code.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
bool hasHugeWorkingSetSize() const
Returns true if the working set size of the code is considered huge.
uint64_t getOrCompHotCountThreshold() const
Returns HotCountThreshold if set.
bool isFunctionEntryCold(const Function *F) const
Returns true if F has cold function entry.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ProfileSummary * getFromMD(Metadata *MD)
Construct profile summary from metdata.
#define UINT64_MAX
Definition: DataTypes.h:77
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
cl::opt< int > ProfileSummaryHotCount
cl::opt< int > ProfileSummaryColdCount
cl::opt< int > ProfileSummaryCutoffCold
cl::opt< unsigned > ProfileSummaryLargeWorkingSetSizeThreshold
void initializeProfileSummaryInfoWrapperPassPass(PassRegistry &)
cl::opt< int > ProfileSummaryCutoffHot
cl::opt< unsigned > ProfileSummaryHugeWorkingSetSizeThreshold
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:69