Line data Source code
1 : //===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file contains a pass that provides access to the global profile summary
11 : // information.
12 : //
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "llvm/Analysis/ProfileSummaryInfo.h"
16 : #include "llvm/Analysis/BlockFrequencyInfo.h"
17 : #include "llvm/IR/BasicBlock.h"
18 : #include "llvm/IR/CallSite.h"
19 : #include "llvm/IR/Metadata.h"
20 : #include "llvm/IR/Module.h"
21 : #include "llvm/IR/ProfileSummary.h"
22 : using namespace llvm;
23 :
24 : // The following two parameters determine the threshold for a count to be
25 : // considered hot/cold. These two parameters are percentile values (multiplied
26 : // by 10000). If the counts are sorted in descending order, the minimum count to
27 : // reach ProfileSummaryCutoffHot gives the threshold to determine a hot count.
28 : // Similarly, the minimum count to reach ProfileSummaryCutoffCold gives the
29 : // threshold for determining cold count (everything <= this threshold is
30 : // considered cold).
31 :
32 : static cl::opt<int> ProfileSummaryCutoffHot(
33 : "profile-summary-cutoff-hot", cl::Hidden, cl::init(990000), cl::ZeroOrMore,
34 : cl::desc("A count is hot if it exceeds the minimum count to"
35 : " reach this percentile of total counts."));
36 :
37 : static cl::opt<int> ProfileSummaryCutoffCold(
38 : "profile-summary-cutoff-cold", cl::Hidden, cl::init(999999), cl::ZeroOrMore,
39 : cl::desc("A count is cold if it is below the minimum count"
40 : " to reach this percentile of total counts."));
41 :
42 : static cl::opt<bool> ProfileSampleAccurate(
43 : "profile-sample-accurate", cl::Hidden, cl::init(false),
44 : cl::desc("If the sample profile is accurate, we will mark all un-sampled "
45 : "callsite as cold. Otherwise, treat un-sampled callsites as if "
46 : "we have no profile."));
47 : static cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold(
48 : "profile-summary-huge-working-set-size-threshold", cl::Hidden,
49 : cl::init(15000), cl::ZeroOrMore,
50 : cl::desc("The code working set size is considered huge if the number of"
51 : " blocks required to reach the -profile-summary-cutoff-hot"
52 : " percentile exceeds this count."));
53 :
54 : // Find the summary entry for a desired percentile of counts.
55 168 : static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS,
56 : uint64_t Percentile) {
57 : auto Compare = [](const ProfileSummaryEntry &Entry, uint64_t Percentile) {
58 568 : return Entry.Cutoff < Percentile;
59 : };
60 : auto It = std::lower_bound(DS.begin(), DS.end(), Percentile, Compare);
61 : // The required percentile has to be <= one of the percentiles in the
62 : // detailed summary.
63 168 : if (It == DS.end())
64 0 : report_fatal_error("Desired percentile exceeds the maximum cutoff");
65 168 : return *It;
66 : }
67 :
68 : // The profile summary metadata may be attached either by the frontend or by
69 : // any backend passes (IR level instrumentation, for example). This method
70 : // checks if the Summary is null and if so checks if the summary metadata is now
71 : // available in the module and parses it to get the Summary object. Returns true
72 : // if a valid Summary is available.
73 1742431 : bool ProfileSummaryInfo::computeSummary() {
74 1742431 : if (Summary)
75 : return true;
76 1741923 : auto *SummaryMD = M.getProfileSummary();
77 1741923 : if (!SummaryMD)
78 : return false;
79 93 : Summary.reset(ProfileSummary::getFromMD(SummaryMD));
80 : return true;
81 : }
82 :
83 : Optional<uint64_t>
84 493 : ProfileSummaryInfo::getProfileCount(const Instruction *Inst,
85 : BlockFrequencyInfo *BFI) {
86 493 : if (!Inst)
87 : return None;
88 : assert((isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) &&
89 : "We can only get profile count for call/invoke instruction.");
90 : if (hasSampleProfile()) {
91 : // In sample PGO mode, check if there is a profile metadata on the
92 : // instruction. If it is present, determine hotness solely based on that,
93 : // since the sampled entry count may not be accurate. If there is no
94 : // annotated on the instruction, return None.
95 : uint64_t TotalCount;
96 77 : if (Inst->extractProfTotalWeight(TotalCount))
97 : return TotalCount;
98 : return None;
99 : }
100 416 : if (BFI)
101 348 : return BFI->getBlockProfileCount(Inst->getParent());
102 : return None;
103 : }
104 :
105 : /// Returns true if the function's entry is hot. If it returns false, it
106 : /// either means it is not hot or it is unknown whether it is hot or not (for
107 : /// example, no profile data is available).
108 287928 : bool ProfileSummaryInfo::isFunctionEntryHot(const Function *F) {
109 287928 : if (!F || !computeSummary())
110 287865 : return false;
111 63 : auto FunctionCount = F->getEntryCount();
112 : // FIXME: The heuristic used below for determining hotness is based on
113 : // preliminary SPEC tuning for inliner. This will eventually be a
114 : // convenience method that calls isHotCount.
115 63 : return FunctionCount && isHotCount(FunctionCount.getCount());
116 : }
117 :
118 : /// Returns true if the function contains hot code. This can include a hot
119 : /// function entry count, hot basic block, or (in the case of Sample PGO)
120 : /// hot total call edge count.
121 : /// If it returns false, it either means it is not hot or it is unknown
122 : /// (for example, no profile data is available).
123 198528 : bool ProfileSummaryInfo::isFunctionHotInCallGraph(const Function *F,
124 : BlockFrequencyInfo &BFI) {
125 198528 : if (!F || !computeSummary())
126 198480 : return false;
127 48 : if (auto FunctionCount = F->getEntryCount())
128 18 : if (isHotCount(FunctionCount.getCount()))
129 4 : return true;
130 :
131 : if (hasSampleProfile()) {
132 : uint64_t TotalCallCount = 0;
133 46 : for (const auto &BB : *F)
134 90 : for (const auto &I : BB)
135 59 : if (isa<CallInst>(I) || isa<InvokeInst>(I))
136 17 : if (auto CallCount = getProfileCount(&I, nullptr))
137 9 : TotalCallCount += CallCount.getValue();
138 15 : if (isHotCount(TotalCallCount))
139 : return true;
140 : }
141 84 : for (const auto &BB : *F)
142 49 : if (isHotBB(&BB, &BFI))
143 : return true;
144 : return false;
145 : }
146 :
147 : /// Returns true if the function only contains cold code. This means that
148 : /// the function entry and blocks are all cold, and (in the case of Sample PGO)
149 : /// the total call edge count is cold.
150 : /// If it returns false, it either means it is not cold or it is unknown
151 : /// (for example, no profile data is available).
152 198515 : bool ProfileSummaryInfo::isFunctionColdInCallGraph(const Function *F,
153 : BlockFrequencyInfo &BFI) {
154 198515 : if (!F || !computeSummary())
155 198480 : return false;
156 35 : if (auto FunctionCount = F->getEntryCount())
157 7 : if (!isColdCount(FunctionCount.getCount()))
158 0 : return false;
159 :
160 : if (hasSampleProfile()) {
161 : uint64_t TotalCallCount = 0;
162 19 : for (const auto &BB : *F)
163 36 : for (const auto &I : BB)
164 24 : if (isa<CallInst>(I) || isa<InvokeInst>(I))
165 8 : if (auto CallCount = getProfileCount(&I, nullptr))
166 4 : TotalCallCount += CallCount.getValue();
167 7 : if (!isColdCount(TotalCallCount))
168 : return false;
169 : }
170 45 : for (const auto &BB : *F)
171 39 : if (!isColdBB(&BB, &BFI))
172 : return false;
173 : return true;
174 : }
175 :
176 : /// Returns true if the function's entry is a cold. If it returns false, it
177 : /// either means it is not cold or it is unknown whether it is cold or not (for
178 : /// example, no profile data is available).
179 288073 : bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) {
180 288073 : if (!F)
181 : return false;
182 288073 : if (F->hasFnAttribute(Attribute::Cold))
183 : return true;
184 288056 : if (!computeSummary())
185 : return false;
186 17 : auto FunctionCount = F->getEntryCount();
187 : // FIXME: The heuristic used below for determining coldness is based on
188 : // preliminary SPEC tuning for inliner. This will eventually be a
189 : // convenience method that calls isHotCount.
190 17 : return FunctionCount && isColdCount(FunctionCount.getCount());
191 : }
192 :
193 : /// Compute the hot and cold thresholds.
194 192789 : void ProfileSummaryInfo::computeThresholds() {
195 192789 : if (!computeSummary())
196 : return;
197 : auto &DetailedSummary = Summary->getDetailedSummary();
198 : auto &HotEntry =
199 84 : getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffHot);
200 84 : HotCountThreshold = HotEntry.MinCount;
201 : auto &ColdEntry =
202 84 : getEntryForPercentile(DetailedSummary, ProfileSummaryCutoffCold);
203 84 : ColdCountThreshold = ColdEntry.MinCount;
204 : HasHugeWorkingSetSize =
205 168 : HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
206 : }
207 :
208 192724 : bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
209 192724 : if (!HasHugeWorkingSetSize)
210 192689 : computeThresholds();
211 192724 : return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue();
212 : }
213 :
214 281 : bool ProfileSummaryInfo::isHotCount(uint64_t C) {
215 281 : if (!HotCountThreshold)
216 85 : computeThresholds();
217 281 : return HotCountThreshold && C >= HotCountThreshold.getValue();
218 : }
219 :
220 131 : bool ProfileSummaryInfo::isColdCount(uint64_t C) {
221 131 : if (!ColdCountThreshold)
222 15 : computeThresholds();
223 131 : return ColdCountThreshold && C <= ColdCountThreshold.getValue();
224 : }
225 :
226 8 : uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() {
227 8 : if (!HotCountThreshold)
228 0 : computeThresholds();
229 8 : return HotCountThreshold ? HotCountThreshold.getValue() : UINT64_MAX;
230 : }
231 :
232 0 : uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() {
233 0 : if (!ColdCountThreshold)
234 0 : computeThresholds();
235 0 : return ColdCountThreshold ? ColdCountThreshold.getValue() : 0;
236 : }
237 :
238 58 : bool ProfileSummaryInfo::isHotBB(const BasicBlock *B, BlockFrequencyInfo *BFI) {
239 58 : auto Count = BFI->getBlockProfileCount(B);
240 58 : return Count && isHotCount(*Count);
241 : }
242 :
243 124 : bool ProfileSummaryInfo::isColdBB(const BasicBlock *B,
244 : BlockFrequencyInfo *BFI) {
245 124 : auto Count = BFI->getBlockProfileCount(B);
246 124 : return Count && isColdCount(*Count);
247 : }
248 :
249 43 : bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS,
250 : BlockFrequencyInfo *BFI) {
251 43 : auto C = getProfileCount(CS.getInstruction(), BFI);
252 43 : return C && isHotCount(*C);
253 : }
254 :
255 25 : bool ProfileSummaryInfo::isColdCallSite(const CallSite &CS,
256 : BlockFrequencyInfo *BFI) {
257 25 : auto C = getProfileCount(CS.getInstruction(), BFI);
258 25 : if (C)
259 11 : return isColdCount(*C);
260 :
261 : // In SamplePGO, if the caller has been sampled, and there is no profile
262 : // annotated on the callsite, we consider the callsite as cold.
263 : // If there is no profile for the caller, and we know the profile is
264 : // accurate, we consider the callsite as cold.
265 8 : return (hasSampleProfile() &&
266 3 : (CS.getCaller()->hasProfileData() || ProfileSampleAccurate ||
267 3 : CS.getCaller()->hasFnAttribute("profile-sample-accurate")));
268 : }
269 :
270 349851 : INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",
271 : "Profile summary info", false, true)
272 :
273 36257 : ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()
274 36257 : : ImmutablePass(ID) {
275 36257 : initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
276 36257 : }
277 :
278 36253 : bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {
279 36253 : PSI.reset(new ProfileSummaryInfo(M));
280 36253 : return false;
281 : }
282 :
283 36107 : bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) {
284 : PSI.reset();
285 36107 : return false;
286 : }
287 :
288 : AnalysisKey ProfileSummaryAnalysis::Key;
289 182 : ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M,
290 : ModuleAnalysisManager &) {
291 182 : return ProfileSummaryInfo(M);
292 : }
293 :
294 1 : PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M,
295 : ModuleAnalysisManager &AM) {
296 : ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
297 :
298 1 : OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
299 4 : for (auto &F : M) {
300 3 : OS << F.getName();
301 3 : if (PSI.isFunctionEntryHot(&F))
302 2 : OS << " :hot entry ";
303 1 : else if (PSI.isFunctionEntryCold(&F))
304 1 : OS << " :cold entry ";
305 3 : OS << "\n";
306 : }
307 1 : return PreservedAnalyses::all();
308 : }
309 :
310 : char ProfileSummaryInfoWrapperPass::ID = 0;
|
