Bug Summary

File:llvm/lib/Transforms/IPO/SampleProfile.cpp
Warning:line 1329, column 7
Assigned value is garbage or undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SampleProfile.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Transforms/IPO -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Transforms/IPO -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../x86_64-linux-gnu/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/build-llvm/lib/Transforms/IPO -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-04-05-202135-9119-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp
1//===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SampleProfileLoader transformation. This pass
10// reads a profile file generated by a sampling profiler (e.g. Linux Perf -
11// http://perf.wiki.kernel.org/) and generates IR metadata to reflect the
12// profile information in the given profile.
13//
14// This pass generates branch weight annotations on the IR:
15//
16// - prof: Represents branch weights. This annotation is added to branches
17// to indicate the weights of each edge coming out of the branch.
18// The weight of each edge is the weight of the target block for
19// that edge. The weight of a block B is computed as the maximum
20// number of samples found in B.
21//
22//===----------------------------------------------------------------------===//
23
24#include "llvm/Transforms/IPO/SampleProfile.h"
25#include "llvm/ADT/ArrayRef.h"
26#include "llvm/ADT/DenseMap.h"
27#include "llvm/ADT/DenseSet.h"
28#include "llvm/ADT/None.h"
29#include "llvm/ADT/PriorityQueue.h"
30#include "llvm/ADT/SCCIterator.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include "llvm/ADT/SmallSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/ADT/StringMap.h"
36#include "llvm/ADT/StringRef.h"
37#include "llvm/ADT/Twine.h"
38#include "llvm/Analysis/AssumptionCache.h"
39#include "llvm/Analysis/CallGraph.h"
40#include "llvm/Analysis/CallGraphSCCPass.h"
41#include "llvm/Analysis/InlineAdvisor.h"
42#include "llvm/Analysis/InlineCost.h"
43#include "llvm/Analysis/LoopInfo.h"
44#include "llvm/Analysis/OptimizationRemarkEmitter.h"
45#include "llvm/Analysis/PostDominators.h"
46#include "llvm/Analysis/ProfileSummaryInfo.h"
47#include "llvm/Analysis/ReplayInlineAdvisor.h"
48#include "llvm/Analysis/TargetLibraryInfo.h"
49#include "llvm/Analysis/TargetTransformInfo.h"
50#include "llvm/IR/BasicBlock.h"
51#include "llvm/IR/CFG.h"
52#include "llvm/IR/DebugInfoMetadata.h"
53#include "llvm/IR/DebugLoc.h"
54#include "llvm/IR/DiagnosticInfo.h"
55#include "llvm/IR/Dominators.h"
56#include "llvm/IR/Function.h"
57#include "llvm/IR/GlobalValue.h"
58#include "llvm/IR/InstrTypes.h"
59#include "llvm/IR/Instruction.h"
60#include "llvm/IR/Instructions.h"
61#include "llvm/IR/IntrinsicInst.h"
62#include "llvm/IR/LLVMContext.h"
63#include "llvm/IR/MDBuilder.h"
64#include "llvm/IR/Module.h"
65#include "llvm/IR/PassManager.h"
66#include "llvm/IR/ValueSymbolTable.h"
67#include "llvm/InitializePasses.h"
68#include "llvm/Pass.h"
69#include "llvm/ProfileData/InstrProf.h"
70#include "llvm/ProfileData/SampleProf.h"
71#include "llvm/ProfileData/SampleProfReader.h"
72#include "llvm/Support/Casting.h"
73#include "llvm/Support/CommandLine.h"
74#include "llvm/Support/Debug.h"
75#include "llvm/Support/ErrorHandling.h"
76#include "llvm/Support/ErrorOr.h"
77#include "llvm/Support/GenericDomTree.h"
78#include "llvm/Support/raw_ostream.h"
79#include "llvm/Transforms/IPO.h"
80#include "llvm/Transforms/IPO/ProfiledCallGraph.h"
81#include "llvm/Transforms/IPO/SampleContextTracker.h"
82#include "llvm/Transforms/IPO/SampleProfileProbe.h"
83#include "llvm/Transforms/Instrumentation.h"
84#include "llvm/Transforms/Utils/CallPromotionUtils.h"
85#include "llvm/Transforms/Utils/Cloning.h"
86#include "llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h"
87#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
88#include <algorithm>
89#include <cassert>
90#include <cstdint>
91#include <functional>
92#include <limits>
93#include <map>
94#include <memory>
95#include <queue>
96#include <string>
97#include <system_error>
98#include <utility>
99#include <vector>
100
101using namespace llvm;
102using namespace sampleprof;
103using namespace llvm::sampleprofutil;
104using ProfileCount = Function::ProfileCount;
105#define DEBUG_TYPE"sample-profile" "sample-profile"
106#define CSINLINE_DEBUG"sample-profile" "-inline" DEBUG_TYPE"sample-profile" "-inline"
107
108STATISTIC(NumCSInlined,static llvm::Statistic NumCSInlined = {"sample-profile", "NumCSInlined"
, "Number of functions inlined with context sensitive profile"
}
109 "Number of functions inlined with context sensitive profile")static llvm::Statistic NumCSInlined = {"sample-profile", "NumCSInlined"
, "Number of functions inlined with context sensitive profile"
}
;
110STATISTIC(NumCSNotInlined,static llvm::Statistic NumCSNotInlined = {"sample-profile", "NumCSNotInlined"
, "Number of functions not inlined with context sensitive profile"
}
111 "Number of functions not inlined with context sensitive profile")static llvm::Statistic NumCSNotInlined = {"sample-profile", "NumCSNotInlined"
, "Number of functions not inlined with context sensitive profile"
}
;
112STATISTIC(NumMismatchedProfile,static llvm::Statistic NumMismatchedProfile = {"sample-profile"
, "NumMismatchedProfile", "Number of functions with CFG mismatched profile"
}
113 "Number of functions with CFG mismatched profile")static llvm::Statistic NumMismatchedProfile = {"sample-profile"
, "NumMismatchedProfile", "Number of functions with CFG mismatched profile"
}
;
114STATISTIC(NumMatchedProfile, "Number of functions with CFG matched profile")static llvm::Statistic NumMatchedProfile = {"sample-profile",
"NumMatchedProfile", "Number of functions with CFG matched profile"
}
;
115STATISTIC(NumDuplicatedInlinesite,static llvm::Statistic NumDuplicatedInlinesite = {"sample-profile"
, "NumDuplicatedInlinesite", "Number of inlined callsites with a partial distribution factor"
}
116 "Number of inlined callsites with a partial distribution factor")static llvm::Statistic NumDuplicatedInlinesite = {"sample-profile"
, "NumDuplicatedInlinesite", "Number of inlined callsites with a partial distribution factor"
}
;
117
118STATISTIC(NumCSInlinedHitMinLimit,static llvm::Statistic NumCSInlinedHitMinLimit = {"sample-profile"
, "NumCSInlinedHitMinLimit", "Number of functions with FDO inline stopped due to min size limit"
}
119 "Number of functions with FDO inline stopped due to min size limit")static llvm::Statistic NumCSInlinedHitMinLimit = {"sample-profile"
, "NumCSInlinedHitMinLimit", "Number of functions with FDO inline stopped due to min size limit"
}
;
120STATISTIC(NumCSInlinedHitMaxLimit,static llvm::Statistic NumCSInlinedHitMaxLimit = {"sample-profile"
, "NumCSInlinedHitMaxLimit", "Number of functions with FDO inline stopped due to max size limit"
}
121 "Number of functions with FDO inline stopped due to max size limit")static llvm::Statistic NumCSInlinedHitMaxLimit = {"sample-profile"
, "NumCSInlinedHitMaxLimit", "Number of functions with FDO inline stopped due to max size limit"
}
;
122STATISTIC(static llvm::Statistic NumCSInlinedHitGrowthLimit = {"sample-profile"
, "NumCSInlinedHitGrowthLimit", "Number of functions with FDO inline stopped due to growth size limit"
}
123 NumCSInlinedHitGrowthLimit,static llvm::Statistic NumCSInlinedHitGrowthLimit = {"sample-profile"
, "NumCSInlinedHitGrowthLimit", "Number of functions with FDO inline stopped due to growth size limit"
}
124 "Number of functions with FDO inline stopped due to growth size limit")static llvm::Statistic NumCSInlinedHitGrowthLimit = {"sample-profile"
, "NumCSInlinedHitGrowthLimit", "Number of functions with FDO inline stopped due to growth size limit"
}
;
125
126// Command line option to specify the file to read samples from. This is
127// mainly used for debugging.
128static cl::opt<std::string> SampleProfileFile(
129 "sample-profile-file", cl::init(""), cl::value_desc("filename"),
130 cl::desc("Profile file loaded by -sample-profile"), cl::Hidden);
131
132// The named file contains a set of transformations that may have been applied
133// to the symbol names between the program from which the sample data was
134// collected and the current program's symbols.
135static cl::opt<std::string> SampleProfileRemappingFile(
136 "sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"),
137 cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden);
138
139static cl::opt<bool> ProfileSampleAccurate(
140 "profile-sample-accurate", cl::Hidden, cl::init(false),
141 cl::desc("If the sample profile is accurate, we will mark all un-sampled "
142 "callsite and function as having 0 samples. Otherwise, treat "
143 "un-sampled callsites and functions conservatively as unknown. "));
144
145static cl::opt<bool> ProfileAccurateForSymsInList(
146 "profile-accurate-for-symsinlist", cl::Hidden, cl::ZeroOrMore,
147 cl::init(true),
148 cl::desc("For symbols in profile symbol list, regard their profiles to "
149 "be accurate. It may be overriden by profile-sample-accurate. "));
150
151static cl::opt<bool> ProfileMergeInlinee(
152 "sample-profile-merge-inlinee", cl::Hidden, cl::init(true),
153 cl::desc("Merge past inlinee's profile to outline version if sample "
154 "profile loader decided not to inline a call site. It will "
155 "only be enabled when top-down order of profile loading is "
156 "enabled. "));
157
158static cl::opt<bool> ProfileTopDownLoad(
159 "sample-profile-top-down-load", cl::Hidden, cl::init(true),
160 cl::desc("Do profile annotation and inlining for functions in top-down "
161 "order of call graph during sample profile loading. It only "
162 "works for new pass manager. "));
163
164static cl::opt<bool>
165 UseProfiledCallGraph("use-profiled-call-graph", cl::init(true), cl::Hidden,
166 cl::desc("Process functions in a top-down order "
167 "defined by the profiled call graph when "
168 "-sample-profile-top-down-load is on."));
169
170static cl::opt<bool> ProfileSizeInline(
171 "sample-profile-inline-size", cl::Hidden, cl::init(false),
172 cl::desc("Inline cold call sites in profile loader if it's beneficial "
173 "for code size."));
174
175cl::opt<int> ProfileInlineGrowthLimit(
176 "sample-profile-inline-growth-limit", cl::Hidden, cl::init(12),
177 cl::desc("The size growth ratio limit for proirity-based sample profile "
178 "loader inlining."));
179
180cl::opt<int> ProfileInlineLimitMin(
181 "sample-profile-inline-limit-min", cl::Hidden, cl::init(100),
182 cl::desc("The lower bound of size growth limit for "
183 "proirity-based sample profile loader inlining."));
184
185cl::opt<int> ProfileInlineLimitMax(
186 "sample-profile-inline-limit-max", cl::Hidden, cl::init(10000),
187 cl::desc("The upper bound of size growth limit for "
188 "proirity-based sample profile loader inlining."));
189
190cl::opt<int> SampleHotCallSiteThreshold(
191 "sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000),
192 cl::desc("Hot callsite threshold for proirity-based sample profile loader "
193 "inlining."));
194
195cl::opt<int> SampleColdCallSiteThreshold(
196 "sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45),
197 cl::desc("Threshold for inlining cold callsites"));
198
199static cl::opt<int> ProfileICPThreshold(
200 "sample-profile-icp-threshold", cl::Hidden, cl::init(5),
201 cl::desc(
202 "Relative hotness threshold for indirect "
203 "call promotion in proirity-based sample profile loader inlining."));
204
205static cl::opt<bool> CallsitePrioritizedInline(
206 "sample-profile-prioritized-inline", cl::Hidden, cl::ZeroOrMore,
207 cl::init(false),
208 cl::desc("Use call site prioritized inlining for sample profile loader."
209 "Currently only CSSPGO is supported."));
210
211
212static cl::opt<std::string> ProfileInlineReplayFile(
213 "sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"),
214 cl::desc(
215 "Optimization remarks file containing inline remarks to be replayed "
216 "by inlining from sample profile loader."),
217 cl::Hidden);
218
219static cl::opt<unsigned>
220 MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden,
221 cl::ZeroOrMore,
222 cl::desc("Max number of promotions for a single indirect "
223 "call callsite in sample profile loader"));
224
225namespace {
226
227using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>;
228using EquivalenceClassMap = DenseMap<const BasicBlock *, const BasicBlock *>;
229using Edge = std::pair<const BasicBlock *, const BasicBlock *>;
230using EdgeWeightMap = DenseMap<Edge, uint64_t>;
231using BlockEdgeMap =
232 DenseMap<const BasicBlock *, SmallVector<const BasicBlock *, 8>>;
233
234class GUIDToFuncNameMapper {
235public:
236 GUIDToFuncNameMapper(Module &M, SampleProfileReader &Reader,
237 DenseMap<uint64_t, StringRef> &GUIDToFuncNameMap)
238 : CurrentReader(Reader), CurrentModule(M),
239 CurrentGUIDToFuncNameMap(GUIDToFuncNameMap) {
240 if (!CurrentReader.useMD5())
241 return;
242
243 for (const auto &F : CurrentModule) {
244 StringRef OrigName = F.getName();
245 CurrentGUIDToFuncNameMap.insert(
246 {Function::getGUID(OrigName), OrigName});
247
248 // Local to global var promotion used by optimization like thinlto
249 // will rename the var and add suffix like ".llvm.xxx" to the
250 // original local name. In sample profile, the suffixes of function
251 // names are all stripped. Since it is possible that the mapper is
252 // built in post-thin-link phase and var promotion has been done,
253 // we need to add the substring of function name without the suffix
254 // into the GUIDToFuncNameMap.
255 StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
256 if (CanonName != OrigName)
257 CurrentGUIDToFuncNameMap.insert(
258 {Function::getGUID(CanonName), CanonName});
259 }
260
261 // Update GUIDToFuncNameMap for each function including inlinees.
262 SetGUIDToFuncNameMapForAll(&CurrentGUIDToFuncNameMap);
263 }
264
265 ~GUIDToFuncNameMapper() {
266 if (!CurrentReader.useMD5())
267 return;
268
269 CurrentGUIDToFuncNameMap.clear();
270
271 // Reset GUIDToFuncNameMap for of each function as they're no
272 // longer valid at this point.
273 SetGUIDToFuncNameMapForAll(nullptr);
274 }
275
276private:
277 void SetGUIDToFuncNameMapForAll(DenseMap<uint64_t, StringRef> *Map) {
278 std::queue<FunctionSamples *> FSToUpdate;
279 for (auto &IFS : CurrentReader.getProfiles()) {
280 FSToUpdate.push(&IFS.second);
281 }
282
283 while (!FSToUpdate.empty()) {
284 FunctionSamples *FS = FSToUpdate.front();
285 FSToUpdate.pop();
286 FS->GUIDToFuncNameMap = Map;
287 for (const auto &ICS : FS->getCallsiteSamples()) {
288 const FunctionSamplesMap &FSMap = ICS.second;
289 for (auto &IFS : FSMap) {
290 FunctionSamples &FS = const_cast<FunctionSamples &>(IFS.second);
291 FSToUpdate.push(&FS);
292 }
293 }
294 }
295 }
296
297 SampleProfileReader &CurrentReader;
298 Module &CurrentModule;
299 DenseMap<uint64_t, StringRef> &CurrentGUIDToFuncNameMap;
300};
301
302// Inline candidate used by iterative callsite prioritized inliner
303struct InlineCandidate {
304 CallBase *CallInstr;
305 const FunctionSamples *CalleeSamples;
306 // Prorated callsite count, which will be used to guide inlining. For example,
307 // if a callsite is duplicated in LTO prelink, then in LTO postlink the two
308 // copies will get their own distribution factors and their prorated counts
309 // will be used to decide if they should be inlined independently.
310 uint64_t CallsiteCount;
311 // Call site distribution factor to prorate the profile samples for a
312 // duplicated callsite. Default value is 1.0.
313 float CallsiteDistribution;
314};
315
316// Inline candidate comparer using call site weight
317struct CandidateComparer {
318 bool operator()(const InlineCandidate &LHS, const InlineCandidate &RHS) {
319 if (LHS.CallsiteCount != RHS.CallsiteCount)
320 return LHS.CallsiteCount < RHS.CallsiteCount;
321
322 const FunctionSamples *LCS = LHS.CalleeSamples;
323 const FunctionSamples *RCS = RHS.CalleeSamples;
324 assert(LCS && RCS && "Expect non-null FunctionSamples")((LCS && RCS && "Expect non-null FunctionSamples"
) ? static_cast<void> (0) : __assert_fail ("LCS && RCS && \"Expect non-null FunctionSamples\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 324, __PRETTY_FUNCTION__))
;
325
326 // Tie breaker using number of samples try to favor smaller functions first
327 if (LCS->getBodySamples().size() != RCS->getBodySamples().size())
328 return LCS->getBodySamples().size() > RCS->getBodySamples().size();
329
330 // Tie breaker using GUID so we have stable/deterministic inlining order
331 return LCS->getGUID(LCS->getName()) < RCS->getGUID(RCS->getName());
332 }
333};
334
335using CandidateQueue =
336 PriorityQueue<InlineCandidate, std::vector<InlineCandidate>,
337 CandidateComparer>;
338
339/// Sample profile pass.
340///
341/// This pass reads profile data from the file specified by
342/// -sample-profile-file and annotates every affected function with the
343/// profile information found in that file.
344class SampleProfileLoader final
345 : public SampleProfileLoaderBaseImpl<BasicBlock> {
346public:
347 SampleProfileLoader(
348 StringRef Name, StringRef RemapName, ThinOrFullLTOPhase LTOPhase,
349 std::function<AssumptionCache &(Function &)> GetAssumptionCache,
350 std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo,
351 std::function<const TargetLibraryInfo &(Function &)> GetTLI)
352 : SampleProfileLoaderBaseImpl(std::string(Name)),
353 GetAC(std::move(GetAssumptionCache)),
354 GetTTI(std::move(GetTargetTransformInfo)), GetTLI(std::move(GetTLI)),
355 RemappingFilename(std::string(RemapName)), LTOPhase(LTOPhase) {}
356
357 bool doInitialization(Module &M, FunctionAnalysisManager *FAM = nullptr);
358 bool runOnModule(Module &M, ModuleAnalysisManager *AM,
359 ProfileSummaryInfo *_PSI, CallGraph *CG);
360
361protected:
362 bool runOnFunction(Function &F, ModuleAnalysisManager *AM);
363 bool emitAnnotations(Function &F);
364 ErrorOr<uint64_t> getInstWeight(const Instruction &I) override;
365 ErrorOr<uint64_t> getProbeWeight(const Instruction &I);
366 const FunctionSamples *findCalleeFunctionSamples(const CallBase &I) const;
367 const FunctionSamples *
368 findFunctionSamples(const Instruction &I) const override;
369 std::vector<const FunctionSamples *>
370 findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
371 void findExternalInlineCandidate(const FunctionSamples *Samples,
372 DenseSet<GlobalValue::GUID> &InlinedGUIDs,
373 const StringMap<Function *> &SymbolMap,
374 uint64_t Threshold);
375 // Attempt to promote indirect call and also inline the promoted call
376 bool tryPromoteAndInlineCandidate(
377 Function &F, InlineCandidate &Candidate, uint64_t SumOrigin,
378 uint64_t &Sum, SmallVector<CallBase *, 8> *InlinedCallSites = nullptr);
379 bool inlineHotFunctions(Function &F,
380 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
381 InlineCost shouldInlineCandidate(InlineCandidate &Candidate);
382 bool getInlineCandidate(InlineCandidate *NewCandidate, CallBase *CB);
383 bool
384 tryInlineCandidate(InlineCandidate &Candidate,
385 SmallVector<CallBase *, 8> *InlinedCallSites = nullptr);
386 bool
387 inlineHotFunctionsWithPriority(Function &F,
388 DenseSet<GlobalValue::GUID> &InlinedGUIDs);
389 // Inline cold/small functions in addition to hot ones
390 bool shouldInlineColdCallee(CallBase &CallInst);
391 void emitOptimizationRemarksForInlineCandidates(
392 const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
393 bool Hot);
394 std::vector<Function *> buildFunctionOrder(Module &M, CallGraph *CG);
395 std::unique_ptr<ProfiledCallGraph> buildProfiledCallGraph(CallGraph &CG);
396 void generateMDProfMetadata(Function &F);
397
398 /// Map from function name to Function *. Used to find the function from
399 /// the function name. If the function name contains suffix, additional
400 /// entry is added to map from the stripped name to the function if there
401 /// is one-to-one mapping.
402 StringMap<Function *> SymbolMap;
403
404 std::function<AssumptionCache &(Function &)> GetAC;
405 std::function<TargetTransformInfo &(Function &)> GetTTI;
406 std::function<const TargetLibraryInfo &(Function &)> GetTLI;
407
408 /// Profile tracker for different context.
409 std::unique_ptr<SampleContextTracker> ContextTracker;
410
411 /// Name of the profile remapping file to load.
412 std::string RemappingFilename;
413
414 /// Flag indicating whether the profile input loaded successfully.
415 bool ProfileIsValid = false;
416
417 /// Flag indicating whether input profile is context-sensitive
418 bool ProfileIsCS = false;
419
420 /// Flag indicating which LTO/ThinLTO phase the pass is invoked in.
421 ///
422 /// We need to know the LTO phase because for example in ThinLTOPrelink
423 /// phase, in annotation, we should not promote indirect calls. Instead,
424 /// we will mark GUIDs that needs to be annotated to the function.
425 ThinOrFullLTOPhase LTOPhase;
426
427 /// Profle Symbol list tells whether a function name appears in the binary
428 /// used to generate the current profile.
429 std::unique_ptr<ProfileSymbolList> PSL;
430
431 /// Total number of samples collected in this profile.
432 ///
433 /// This is the sum of all the samples collected in all the functions executed
434 /// at runtime.
435 uint64_t TotalCollectedSamples = 0;
436
437 // Information recorded when we declined to inline a call site
438 // because we have determined it is too cold is accumulated for
439 // each callee function. Initially this is just the entry count.
440 struct NotInlinedProfileInfo {
441 uint64_t entryCount;
442 };
443 DenseMap<Function *, NotInlinedProfileInfo> notInlinedCallInfo;
444
445 // GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
446 // all the function symbols defined or declared in current module.
447 DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
448
449 // All the Names used in FunctionSamples including outline function
450 // names, inline instance names and call target names.
451 StringSet<> NamesInProfile;
452
453 // For symbol in profile symbol list, whether to regard their profiles
454 // to be accurate. It is mainly decided by existance of profile symbol
455 // list and -profile-accurate-for-symsinlist flag, but it can be
456 // overriden by -profile-sample-accurate or profile-sample-accurate
457 // attribute.
458 bool ProfAccForSymsInList;
459
460 // External inline advisor used to replay inline decision from remarks.
461 std::unique_ptr<ReplayInlineAdvisor> ExternalInlineAdvisor;
462
463 // A pseudo probe helper to correlate the imported sample counts.
464 std::unique_ptr<PseudoProbeManager> ProbeManager;
465};
466
467class SampleProfileLoaderLegacyPass : public ModulePass {
468public:
469 // Class identification, replacement for typeinfo
470 static char ID;
471
472 SampleProfileLoaderLegacyPass(
473 StringRef Name = SampleProfileFile,
474 ThinOrFullLTOPhase LTOPhase = ThinOrFullLTOPhase::None)
475 : ModulePass(ID), SampleLoader(
476 Name, SampleProfileRemappingFile, LTOPhase,
477 [&](Function &F) -> AssumptionCache & {
478 return ACT->getAssumptionCache(F);
479 },
480 [&](Function &F) -> TargetTransformInfo & {
481 return TTIWP->getTTI(F);
482 },
483 [&](Function &F) -> TargetLibraryInfo & {
484 return TLIWP->getTLI(F);
485 }) {
486 initializeSampleProfileLoaderLegacyPassPass(
487 *PassRegistry::getPassRegistry());
488 }
489
490 void dump() { SampleLoader.dump(); }
491
492 bool doInitialization(Module &M) override {
493 return SampleLoader.doInitialization(M);
494 }
495
496 StringRef getPassName() const override { return "Sample profile pass"; }
497 bool runOnModule(Module &M) override;
498
499 void getAnalysisUsage(AnalysisUsage &AU) const override {
500 AU.addRequired<AssumptionCacheTracker>();
501 AU.addRequired<TargetTransformInfoWrapperPass>();
502 AU.addRequired<TargetLibraryInfoWrapperPass>();
503 AU.addRequired<ProfileSummaryInfoWrapperPass>();
504 }
505
506private:
507 SampleProfileLoader SampleLoader;
508 AssumptionCacheTracker *ACT = nullptr;
509 TargetTransformInfoWrapperPass *TTIWP = nullptr;
510 TargetLibraryInfoWrapperPass *TLIWP = nullptr;
511};
512
513} // end anonymous namespace
514
515ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
516 if (FunctionSamples::ProfileIsProbeBased)
517 return getProbeWeight(Inst);
518
519 const DebugLoc &DLoc = Inst.getDebugLoc();
520 if (!DLoc)
521 return std::error_code();
522
523 // Ignore all intrinsics, phinodes and branch instructions.
524 // Branch and phinodes instruction usually contains debug info from sources
525 // outside of the residing basic block, thus we ignore them during annotation.
526 if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst) || isa<PHINode>(Inst))
527 return std::error_code();
528
529 // For non-CS profile, if a direct call/invoke instruction is inlined in
530 // profile (findCalleeFunctionSamples returns non-empty result), but not
531 // inlined here, it means that the inlined callsite has no sample, thus the
532 // call instruction should have 0 count.
533 // For CS profile, the callsite count of previously inlined callees is
534 // populated with the entry count of the callees.
535 if (!ProfileIsCS)
536 if (const auto *CB = dyn_cast<CallBase>(&Inst))
537 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
538 return 0;
539
540 return getInstWeightImpl(Inst);
541}
542
543ErrorOr<uint64_t> SampleProfileLoader::getProbeWeight(const Instruction &Inst) {
544 assert(FunctionSamples::ProfileIsProbeBased &&((FunctionSamples::ProfileIsProbeBased && "Profile is not pseudo probe based"
) ? static_cast<void> (0) : __assert_fail ("FunctionSamples::ProfileIsProbeBased && \"Profile is not pseudo probe based\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 545, __PRETTY_FUNCTION__))
545 "Profile is not pseudo probe based")((FunctionSamples::ProfileIsProbeBased && "Profile is not pseudo probe based"
) ? static_cast<void> (0) : __assert_fail ("FunctionSamples::ProfileIsProbeBased && \"Profile is not pseudo probe based\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 545, __PRETTY_FUNCTION__))
;
546 Optional<PseudoProbe> Probe = extractProbe(Inst);
547 if (!Probe)
548 return std::error_code();
549
550 // Ignore danling probes since they are logically deleted and should not
551 // consume any profile samples.
552 if (Probe->isDangling())
553 return std::error_code();
554
555 const FunctionSamples *FS = findFunctionSamples(Inst);
556 if (!FS)
557 return std::error_code();
558
559 // For non-CS profile, If a direct call/invoke instruction is inlined in
560 // profile (findCalleeFunctionSamples returns non-empty result), but not
561 // inlined here, it means that the inlined callsite has no sample, thus the
562 // call instruction should have 0 count.
563 // For CS profile, the callsite count of previously inlined callees is
564 // populated with the entry count of the callees.
565 if (!ProfileIsCS)
566 if (const auto *CB = dyn_cast<CallBase>(&Inst))
567 if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
568 return 0;
569
570 const ErrorOr<uint64_t> &R = FS->findSamplesAt(Probe->Id, 0);
571 if (R) {
572 uint64_t Samples = R.get() * Probe->Factor;
573 bool FirstMark = CoverageTracker.markSamplesUsed(FS, Probe->Id, 0, Samples);
574 if (FirstMark) {
575 ORE->emit([&]() {
576 OptimizationRemarkAnalysis Remark(DEBUG_TYPE"sample-profile", "AppliedSamples", &Inst);
577 Remark << "Applied " << ore::NV("NumSamples", Samples);
578 Remark << " samples from profile (ProbeId=";
579 Remark << ore::NV("ProbeId", Probe->Id);
580 Remark << ", Factor=";
581 Remark << ore::NV("Factor", Probe->Factor);
582 Remark << ", OriginalSamples=";
583 Remark << ore::NV("OriginalSamples", R.get());
584 Remark << ")";
585 return Remark;
586 });
587 }
588 LLVM_DEBUG(dbgs() << " " << Probe->Id << ":" << Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << Probe->
Id << ":" << Inst << " - weight: " <<
R.get() << " - factor: " << format("%0.2f", Probe
->Factor) << ")\n"; } } while (false)
589 << " - weight: " << R.get() << " - factor: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << Probe->
Id << ":" << Inst << " - weight: " <<
R.get() << " - factor: " << format("%0.2f", Probe
->Factor) << ")\n"; } } while (false)
590 << format("%0.2f", Probe->Factor) << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " " << Probe->
Id << ":" << Inst << " - weight: " <<
R.get() << " - factor: " << format("%0.2f", Probe
->Factor) << ")\n"; } } while (false)
;
591 return Samples;
592 }
593 return R;
594}
595
596/// Get the FunctionSamples for a call instruction.
597///
598/// The FunctionSamples of a call/invoke instruction \p Inst is the inlined
599/// instance in which that call instruction is calling to. It contains
600/// all samples that resides in the inlined instance. We first find the
601/// inlined instance in which the call instruction is from, then we
602/// traverse its children to find the callsite with the matching
603/// location.
604///
605/// \param Inst Call/Invoke instruction to query.
606///
607/// \returns The FunctionSamples pointer to the inlined instance.
608const FunctionSamples *
609SampleProfileLoader::findCalleeFunctionSamples(const CallBase &Inst) const {
610 const DILocation *DIL = Inst.getDebugLoc();
611 if (!DIL) {
612 return nullptr;
613 }
614
615 StringRef CalleeName;
616 if (Function *Callee = Inst.getCalledFunction())
617 CalleeName = Callee->getName();
618
619 if (ProfileIsCS)
620 return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName);
621
622 const FunctionSamples *FS = findFunctionSamples(Inst);
623 if (FS == nullptr)
624 return nullptr;
625
626 return FS->findFunctionSamplesAt(FunctionSamples::getCallSiteIdentifier(DIL),
627 CalleeName, Reader->getRemapper());
628}
629
630/// Returns a vector of FunctionSamples that are the indirect call targets
631/// of \p Inst. The vector is sorted by the total number of samples. Stores
632/// the total call count of the indirect call in \p Sum.
633std::vector<const FunctionSamples *>
634SampleProfileLoader::findIndirectCallFunctionSamples(
635 const Instruction &Inst, uint64_t &Sum) const {
636 const DILocation *DIL = Inst.getDebugLoc();
637 std::vector<const FunctionSamples *> R;
638
639 if (!DIL) {
18
Assuming 'DIL' is null
19
Taking true branch
640 return R;
20
Returning without writing to 'Sum'
641 }
642
643 auto FSCompare = [](const FunctionSamples *L, const FunctionSamples *R) {
644 assert(L && R && "Expect non-null FunctionSamples")((L && R && "Expect non-null FunctionSamples"
) ? static_cast<void> (0) : __assert_fail ("L && R && \"Expect non-null FunctionSamples\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 644, __PRETTY_FUNCTION__))
;
645 if (L->getEntrySamples() != R->getEntrySamples())
646 return L->getEntrySamples() > R->getEntrySamples();
647 return FunctionSamples::getGUID(L->getName()) <
648 FunctionSamples::getGUID(R->getName());
649 };
650
651 if (ProfileIsCS) {
652 auto CalleeSamples =
653 ContextTracker->getIndirectCalleeContextSamplesFor(DIL);
654 if (CalleeSamples.empty())
655 return R;
656
657 // For CSSPGO, we only use target context profile's entry count
658 // as that already includes both inlined callee and non-inlined ones..
659 Sum = 0;
660 for (const auto *const FS : CalleeSamples) {
661 Sum += FS->getEntrySamples();
662 R.push_back(FS);
663 }
664 llvm::sort(R, FSCompare);
665 return R;
666 }
667
668 const FunctionSamples *FS = findFunctionSamples(Inst);
669 if (FS == nullptr)
670 return R;
671
672 auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL);
673 auto T = FS->findCallTargetMapAt(CallSite);
674 Sum = 0;
675 if (T)
676 for (const auto &T_C : T.get())
677 Sum += T_C.second;
678 if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) {
679 if (M->empty())
680 return R;
681 for (const auto &NameFS : *M) {
682 Sum += NameFS.second.getEntrySamples();
683 R.push_back(&NameFS.second);
684 }
685 llvm::sort(R, FSCompare);
686 }
687 return R;
688}
689
690const FunctionSamples *
691SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
692 if (FunctionSamples::ProfileIsProbeBased) {
693 Optional<PseudoProbe> Probe = extractProbe(Inst);
694 if (!Probe)
695 return nullptr;
696 }
697
698 const DILocation *DIL = Inst.getDebugLoc();
699 if (!DIL)
700 return Samples;
701
702 auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
703 if (it.second) {
704 if (ProfileIsCS)
705 it.first->second = ContextTracker->getContextSamplesFor(DIL);
706 else
707 it.first->second =
708 Samples->findFunctionSamples(DIL, Reader->getRemapper());
709 }
710 return it.first->second;
711}
712
713/// Check whether the indirect call promotion history of \p Inst allows
714/// the promotion for \p Candidate.
715/// If the profile count for the promotion candidate \p Candidate is
716/// NOMORE_ICP_MAGICNUM, it means \p Candidate has already been promoted
717/// for \p Inst. If we already have at least MaxNumPromotions
718/// NOMORE_ICP_MAGICNUM count values in the value profile of \p Inst, we
719/// cannot promote for \p Inst anymore.
720static bool doesHistoryAllowICP(const Instruction &Inst, StringRef Candidate) {
721 uint32_t NumVals = 0;
722 uint64_t TotalCount = 0;
723 std::unique_ptr<InstrProfValueData[]> ValueData =
724 std::make_unique<InstrProfValueData[]>(MaxNumPromotions);
725 bool Valid =
726 getValueProfDataFromInst(Inst, IPVK_IndirectCallTarget, MaxNumPromotions,
727 ValueData.get(), NumVals, TotalCount, true);
728 // No valid value profile so no promoted targets have been recorded
729 // before. Ok to do ICP.
730 if (!Valid)
731 return true;
732
733 unsigned NumPromoted = 0;
734 for (uint32_t I = 0; I < NumVals; I++) {
735 if (ValueData[I].Count != NOMORE_ICP_MAGICNUM)
736 continue;
737
738 // If the promotion candidate has NOMORE_ICP_MAGICNUM count in the
739 // metadata, it means the candidate has been promoted for this
740 // indirect call.
741 if (ValueData[I].Value == Function::getGUID(Candidate))
742 return false;
743 NumPromoted++;
744 // If already have MaxNumPromotions promotion, don't do it anymore.
745 if (NumPromoted == MaxNumPromotions)
746 return false;
747 }
748 return true;
749}
750
751/// Update indirect call target profile metadata for \p Inst.
752/// Usually \p Sum is the sum of counts of all the targets for \p Inst.
753/// If it is 0, it means updateIDTMetaData is used to mark a
754/// certain target to be promoted already. If it is not zero,
755/// we expect to use it to update the total count in the value profile.
756static void
757updateIDTMetaData(Instruction &Inst,
758 const SmallVectorImpl<InstrProfValueData> &CallTargets,
759 uint64_t Sum) {
760 uint32_t NumVals = 0;
761 // OldSum is the existing total count in the value profile data.
762 uint64_t OldSum = 0;
763 std::unique_ptr<InstrProfValueData[]> ValueData =
764 std::make_unique<InstrProfValueData[]>(MaxNumPromotions);
765 bool Valid =
766 getValueProfDataFromInst(Inst, IPVK_IndirectCallTarget, MaxNumPromotions,
767 ValueData.get(), NumVals, OldSum, true);
768
769 DenseMap<uint64_t, uint64_t> ValueCountMap;
770 if (Sum == 0) {
771 assert((CallTargets.size() == 1 &&(((CallTargets.size() == 1 && CallTargets[0].Count ==
NOMORE_ICP_MAGICNUM) && "If sum is 0, assume only one element in CallTargets "
"with count being NOMORE_ICP_MAGICNUM") ? static_cast<void
> (0) : __assert_fail ("(CallTargets.size() == 1 && CallTargets[0].Count == NOMORE_ICP_MAGICNUM) && \"If sum is 0, assume only one element in CallTargets \" \"with count being NOMORE_ICP_MAGICNUM\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 774, __PRETTY_FUNCTION__))
772 CallTargets[0].Count == NOMORE_ICP_MAGICNUM) &&(((CallTargets.size() == 1 && CallTargets[0].Count ==
NOMORE_ICP_MAGICNUM) && "If sum is 0, assume only one element in CallTargets "
"with count being NOMORE_ICP_MAGICNUM") ? static_cast<void
> (0) : __assert_fail ("(CallTargets.size() == 1 && CallTargets[0].Count == NOMORE_ICP_MAGICNUM) && \"If sum is 0, assume only one element in CallTargets \" \"with count being NOMORE_ICP_MAGICNUM\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 774, __PRETTY_FUNCTION__))
773 "If sum is 0, assume only one element in CallTargets "(((CallTargets.size() == 1 && CallTargets[0].Count ==
NOMORE_ICP_MAGICNUM) && "If sum is 0, assume only one element in CallTargets "
"with count being NOMORE_ICP_MAGICNUM") ? static_cast<void
> (0) : __assert_fail ("(CallTargets.size() == 1 && CallTargets[0].Count == NOMORE_ICP_MAGICNUM) && \"If sum is 0, assume only one element in CallTargets \" \"with count being NOMORE_ICP_MAGICNUM\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 774, __PRETTY_FUNCTION__))
774 "with count being NOMORE_ICP_MAGICNUM")(((CallTargets.size() == 1 && CallTargets[0].Count ==
NOMORE_ICP_MAGICNUM) && "If sum is 0, assume only one element in CallTargets "
"with count being NOMORE_ICP_MAGICNUM") ? static_cast<void
> (0) : __assert_fail ("(CallTargets.size() == 1 && CallTargets[0].Count == NOMORE_ICP_MAGICNUM) && \"If sum is 0, assume only one element in CallTargets \" \"with count being NOMORE_ICP_MAGICNUM\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 774, __PRETTY_FUNCTION__))
;
775 // Initialize ValueCountMap with existing value profile data.
776 if (Valid) {
777 for (uint32_t I = 0; I < NumVals; I++)
778 ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
779 }
780 auto Pair =
781 ValueCountMap.try_emplace(CallTargets[0].Value, CallTargets[0].Count);
782 // If the target already exists in value profile, decrease the total
783 // count OldSum and reset the target's count to NOMORE_ICP_MAGICNUM.
784 if (!Pair.second) {
785 OldSum -= Pair.first->second;
786 Pair.first->second = NOMORE_ICP_MAGICNUM;
787 }
788 Sum = OldSum;
789 } else {
790 // Initialize ValueCountMap with existing NOMORE_ICP_MAGICNUM
791 // counts in the value profile.
792 if (Valid) {
793 for (uint32_t I = 0; I < NumVals; I++) {
794 if (ValueData[I].Count == NOMORE_ICP_MAGICNUM)
795 ValueCountMap[ValueData[I].Value] = ValueData[I].Count;
796 }
797 }
798
799 for (const auto &Data : CallTargets) {
800 auto Pair = ValueCountMap.try_emplace(Data.Value, Data.Count);
801 if (Pair.second)
802 continue;
803 // The target represented by Data.Value has already been promoted.
804 // Keep the count as NOMORE_ICP_MAGICNUM in the profile and decrease
805 // Sum by Data.Count.
806 assert(Sum >= Data.Count && "Sum should never be less than Data.Count")((Sum >= Data.Count && "Sum should never be less than Data.Count"
) ? static_cast<void> (0) : __assert_fail ("Sum >= Data.Count && \"Sum should never be less than Data.Count\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 806, __PRETTY_FUNCTION__))
;
807 Sum -= Data.Count;
808 }
809 }
810
811 SmallVector<InstrProfValueData, 8> NewCallTargets;
812 for (const auto &ValueCount : ValueCountMap) {
813 NewCallTargets.emplace_back(
814 InstrProfValueData{ValueCount.first, ValueCount.second});
815 }
816
817 llvm::sort(NewCallTargets,
818 [](const InstrProfValueData &L, const InstrProfValueData &R) {
819 if (L.Count != R.Count)
820 return L.Count > R.Count;
821 return L.Value > R.Value;
822 });
823
824 uint32_t MaxMDCount =
825 std::min(NewCallTargets.size(), static_cast<size_t>(MaxNumPromotions));
826 annotateValueSite(*Inst.getParent()->getParent()->getParent(), Inst,
827 NewCallTargets, Sum, IPVK_IndirectCallTarget, MaxMDCount);
828}
829
830/// Attempt to promote indirect call and also inline the promoted call.
831///
832/// \param F Caller function.
833/// \param Candidate ICP and inline candidate.
834/// \param Sum Sum of target counts for indirect call.
835/// \param InlinedCallSite Output vector for new call sites exposed after
836/// inlining.
837bool SampleProfileLoader::tryPromoteAndInlineCandidate(
838 Function &F, InlineCandidate &Candidate, uint64_t SumOrigin, uint64_t &Sum,
839 SmallVector<CallBase *, 8> *InlinedCallSite) {
840 auto CalleeFunctionName = Candidate.CalleeSamples->getFuncName();
841 auto R = SymbolMap.find(CalleeFunctionName);
842 if (R == SymbolMap.end() || !R->getValue())
843 return false;
844
845 auto &CI = *Candidate.CallInstr;
846 if (!doesHistoryAllowICP(CI, R->getValue()->getName()))
847 return false;
848
849 const char *Reason = "Callee function not available";
850 // R->getValue() != &F is to prevent promoting a recursive call.
851 // If it is a recursive call, we do not inline it as it could bloat
852 // the code exponentially. There is way to better handle this, e.g.
853 // clone the caller first, and inline the cloned caller if it is
854 // recursive. As llvm does not inline recursive calls, we will
855 // simply ignore it instead of handling it explicitly.
856 if (!R->getValue()->isDeclaration() && R->getValue()->getSubprogram() &&
857 R->getValue()->hasFnAttribute("use-sample-profile") &&
858 R->getValue() != &F && isLegalToPromote(CI, R->getValue(), &Reason)) {
859 // For promoted target, set its value with NOMORE_ICP_MAGICNUM count
860 // in the value profile metadata so the target won't be promoted again.
861 SmallVector<InstrProfValueData, 1> SortedCallTargets = {InstrProfValueData{
862 Function::getGUID(R->getValue()->getName()), NOMORE_ICP_MAGICNUM}};
863 updateIDTMetaData(CI, SortedCallTargets, 0);
864
865 auto *DI = &pgo::promoteIndirectCall(
866 CI, R->getValue(), Candidate.CallsiteCount, Sum, false, ORE);
867 if (DI) {
868 Sum -= Candidate.CallsiteCount;
869 // Prorate the indirect callsite distribution.
870 // Do not update the promoted direct callsite distribution at this
871 // point since the original distribution combined with the callee
872 // profile will be used to prorate callsites from the callee if
873 // inlined. Once not inlined, the direct callsite distribution should
874 // be prorated so that the it will reflect the real callsite counts.
875 setProbeDistributionFactor(CI, Candidate.CallsiteDistribution * Sum /
876 SumOrigin);
877 Candidate.CallInstr = DI;
878 if (isa<CallInst>(DI) || isa<InvokeInst>(DI)) {
879 bool Inlined = tryInlineCandidate(Candidate, InlinedCallSite);
880 if (!Inlined) {
881 // Prorate the direct callsite distribution so that it reflects real
882 // callsite counts.
883 setProbeDistributionFactor(*DI, Candidate.CallsiteDistribution *
884 Candidate.CallsiteCount /
885 SumOrigin);
886 }
887 return Inlined;
888 }
889 }
890 } else {
891 LLVM_DEBUG(dbgs() << "\nFailed to promote indirect call to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nFailed to promote indirect call to "
<< Candidate.CalleeSamples->getFuncName() << " because "
<< Reason << "\n"; } } while (false)
892 << Candidate.CalleeSamples->getFuncName() << " because "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nFailed to promote indirect call to "
<< Candidate.CalleeSamples->getFuncName() << " because "
<< Reason << "\n"; } } while (false)
893 << Reason << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nFailed to promote indirect call to "
<< Candidate.CalleeSamples->getFuncName() << " because "
<< Reason << "\n"; } } while (false)
;
894 }
895 return false;
896}
897
898bool SampleProfileLoader::shouldInlineColdCallee(CallBase &CallInst) {
899 if (!ProfileSizeInline)
900 return false;
901
902 Function *Callee = CallInst.getCalledFunction();
903 if (Callee == nullptr)
904 return false;
905
906 InlineCost Cost = getInlineCost(CallInst, getInlineParams(), GetTTI(*Callee),
907 GetAC, GetTLI);
908
909 if (Cost.isNever())
910 return false;
911
912 if (Cost.isAlways())
913 return true;
914
915 return Cost.getCost() <= SampleColdCallSiteThreshold;
916}
917
918void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
919 const SmallVectorImpl<CallBase *> &Candidates, const Function &F,
920 bool Hot) {
921 for (auto I : Candidates) {
922 Function *CalledFunction = I->getCalledFunction();
923 if (CalledFunction) {
924 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG"sample-profile" "-inline", "InlineAttempt",
925 I->getDebugLoc(), I->getParent())
926 << "previous inlining reattempted for "
927 << (Hot ? "hotness: '" : "size: '")
928 << ore::NV("Callee", CalledFunction) << "' into '"
929 << ore::NV("Caller", &F) << "'");
930 }
931 }
932}
933
934void SampleProfileLoader::findExternalInlineCandidate(
935 const FunctionSamples *Samples, DenseSet<GlobalValue::GUID> &InlinedGUIDs,
936 const StringMap<Function *> &SymbolMap, uint64_t Threshold) {
937 assert(Samples && "expect non-null caller profile")((Samples && "expect non-null caller profile") ? static_cast
<void> (0) : __assert_fail ("Samples && \"expect non-null caller profile\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 937, __PRETTY_FUNCTION__))
;
938
939 // For AutoFDO profile, retrieve candidate profiles by walking over
940 // the nested inlinee profiles.
941 if (!ProfileIsCS) {
942 Samples->findInlinedFunctions(InlinedGUIDs, SymbolMap, Threshold);
943 return;
944 }
945
946 ContextTrieNode *Caller =
947 ContextTracker->getContextFor(Samples->getContext());
948 std::queue<ContextTrieNode *> CalleeList;
949 CalleeList.push(Caller);
950 while (!CalleeList.empty()) {
951 ContextTrieNode *Node = CalleeList.front();
952 CalleeList.pop();
953 FunctionSamples *CalleeSample = Node->getFunctionSamples();
954 // For CSSPGO profile, retrieve candidate profile by walking over the
955 // trie built for context profile. Note that also take call targets
956 // even if callee doesn't have a corresponding context profile.
957 if (!CalleeSample || CalleeSample->getEntrySamples() < Threshold)
958 continue;
959
960 StringRef Name = CalleeSample->getFuncName();
961 Function *Func = SymbolMap.lookup(Name);
962 // Add to the import list only when it's defined out of module.
963 if (!Func || Func->isDeclaration())
964 InlinedGUIDs.insert(FunctionSamples::getGUID(Name));
965
966 // Import hot CallTargets, which may not be available in IR because full
967 // profile annotation cannot be done until backend compilation in ThinLTO.
968 for (const auto &BS : CalleeSample->getBodySamples())
969 for (const auto &TS : BS.second.getCallTargets())
970 if (TS.getValue() > Threshold) {
971 StringRef CalleeName = CalleeSample->getFuncName(TS.getKey());
972 const Function *Callee = SymbolMap.lookup(CalleeName);
973 if (!Callee || Callee->isDeclaration())
974 InlinedGUIDs.insert(FunctionSamples::getGUID(CalleeName));
975 }
976
977 // Import hot child context profile associted with callees. Note that this
978 // may have some overlap with the call target loop above, but doing this
979 // based child context profile again effectively allow us to use the max of
980 // entry count and call target count to determine importing.
981 for (auto &Child : Node->getAllChildContext()) {
982 ContextTrieNode *CalleeNode = &Child.second;
983 CalleeList.push(CalleeNode);
984 }
985 }
986}
987
988/// Iteratively inline hot callsites of a function.
989///
990/// Iteratively traverse all callsites of the function \p F, and find if
991/// the corresponding inlined instance exists and is hot in profile. If
992/// it is hot enough, inline the callsites and adds new callsites of the
993/// callee into the caller. If the call is an indirect call, first promote
994/// it to direct call. Each indirect call is limited with a single target.
995///
996/// \param F function to perform iterative inlining.
997/// \param InlinedGUIDs a set to be updated to include all GUIDs that are
998/// inlined in the profiled binary.
999///
1000/// \returns True if there is any inline happened.
1001bool SampleProfileLoader::inlineHotFunctions(
1002 Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
1003 // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
1004 // Profile symbol list is ignored when profile-sample-accurate is on.
1005 assert((!ProfAccForSymsInList ||(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1009, __PRETTY_FUNCTION__))
1006 (!ProfileSampleAccurate &&(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1009, __PRETTY_FUNCTION__))
1007 !F.hasFnAttribute("profile-sample-accurate"))) &&(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1009, __PRETTY_FUNCTION__))
1008 "ProfAccForSymsInList should be false when profile-sample-accurate "(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1009, __PRETTY_FUNCTION__))
1009 "is enabled")(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1009, __PRETTY_FUNCTION__))
;
1010
1011 DenseMap<CallBase *, const FunctionSamples *> LocalNotInlinedCallSites;
1012 bool Changed = false;
1013 bool LocalChanged = true;
1014 while (LocalChanged) {
1015 LocalChanged = false;
1016 SmallVector<CallBase *, 10> CIS;
1017 for (auto &BB : F) {
1018 bool Hot = false;
1019 SmallVector<CallBase *, 10> AllCandidates;
1020 SmallVector<CallBase *, 10> ColdCandidates;
1021 for (auto &I : BB.getInstList()) {
1022 const FunctionSamples *FS = nullptr;
1023 if (auto *CB = dyn_cast<CallBase>(&I)) {
1024 if (!isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(*CB))) {
1025 assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&(((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
"GUIDToFuncNameMap has to be populated") ? static_cast<void
> (0) : __assert_fail ("(!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && \"GUIDToFuncNameMap has to be populated\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1026, __PRETTY_FUNCTION__))
1026 "GUIDToFuncNameMap has to be populated")(((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
"GUIDToFuncNameMap has to be populated") ? static_cast<void
> (0) : __assert_fail ("(!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) && \"GUIDToFuncNameMap has to be populated\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1026, __PRETTY_FUNCTION__))
;
1027 AllCandidates.push_back(CB);
1028 if (FS->getEntrySamples() > 0 || ProfileIsCS)
1029 LocalNotInlinedCallSites.try_emplace(CB, FS);
1030 if (callsiteIsHot(FS, PSI, ProfAccForSymsInList))
1031 Hot = true;
1032 else if (shouldInlineColdCallee(*CB))
1033 ColdCandidates.push_back(CB);
1034 }
1035 }
1036 }
1037 if (Hot || ExternalInlineAdvisor) {
1038 CIS.insert(CIS.begin(), AllCandidates.begin(), AllCandidates.end());
1039 emitOptimizationRemarksForInlineCandidates(AllCandidates, F, true);
1040 } else {
1041 CIS.insert(CIS.begin(), ColdCandidates.begin(), ColdCandidates.end());
1042 emitOptimizationRemarksForInlineCandidates(ColdCandidates, F, false);
1043 }
1044 }
1045 for (CallBase *I : CIS) {
1046 Function *CalledFunction = I->getCalledFunction();
1047 InlineCandidate Candidate = {
1048 I,
1049 LocalNotInlinedCallSites.count(I) ? LocalNotInlinedCallSites[I]
1050 : nullptr,
1051 0 /* dummy count */, 1.0 /* dummy distribution factor */};
1052 // Do not inline recursive calls.
1053 if (CalledFunction == &F)
1054 continue;
1055 if (I->isIndirectCall()) {
1056 uint64_t Sum;
1057 for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
1058 uint64_t SumOrigin = Sum;
1059 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1060 findExternalInlineCandidate(FS, InlinedGUIDs, SymbolMap,
1061 PSI->getOrCompHotCountThreshold());
1062 continue;
1063 }
1064 if (!callsiteIsHot(FS, PSI, ProfAccForSymsInList))
1065 continue;
1066
1067 Candidate = {I, FS, FS->getEntrySamples(), 1.0};
1068 if (tryPromoteAndInlineCandidate(F, Candidate, SumOrigin, Sum)) {
1069 LocalNotInlinedCallSites.erase(I);
1070 LocalChanged = true;
1071 }
1072 }
1073 } else if (CalledFunction && CalledFunction->getSubprogram() &&
1074 !CalledFunction->isDeclaration()) {
1075 if (tryInlineCandidate(Candidate)) {
1076 LocalNotInlinedCallSites.erase(I);
1077 LocalChanged = true;
1078 }
1079 } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1080 findExternalInlineCandidate(findCalleeFunctionSamples(*I), InlinedGUIDs,
1081 SymbolMap,
1082 PSI->getOrCompHotCountThreshold());
1083 }
1084 }
1085 Changed |= LocalChanged;
1086 }
1087
1088 // For CS profile, profile for not inlined context will be merged when
1089 // base profile is being trieved
1090 if (ProfileIsCS)
1091 return Changed;
1092
1093 // Accumulate not inlined callsite information into notInlinedSamples
1094 for (const auto &Pair : LocalNotInlinedCallSites) {
1095 CallBase *I = Pair.getFirst();
1096 Function *Callee = I->getCalledFunction();
1097 if (!Callee || Callee->isDeclaration())
1098 continue;
1099
1100 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG"sample-profile" "-inline", "NotInline",
1101 I->getDebugLoc(), I->getParent())
1102 << "previous inlining not repeated: '"
1103 << ore::NV("Callee", Callee) << "' into '"
1104 << ore::NV("Caller", &F) << "'");
1105
1106 ++NumCSNotInlined;
1107 const FunctionSamples *FS = Pair.getSecond();
1108 if (FS->getTotalSamples() == 0 && FS->getEntrySamples() == 0) {
1109 continue;
1110 }
1111
1112 if (ProfileMergeInlinee) {
1113 // A function call can be replicated by optimizations like callsite
1114 // splitting or jump threading and the replicates end up sharing the
1115 // sample nested callee profile instead of slicing the original inlinee's
1116 // profile. We want to do merge exactly once by filtering out callee
1117 // profiles with a non-zero head sample count.
1118 if (FS->getHeadSamples() == 0) {
1119 // Use entry samples as head samples during the merge, as inlinees
1120 // don't have head samples.
1121 const_cast<FunctionSamples *>(FS)->addHeadSamples(
1122 FS->getEntrySamples());
1123
1124 // Note that we have to do the merge right after processing function.
1125 // This allows OutlineFS's profile to be used for annotation during
1126 // top-down processing of functions' annotation.
1127 FunctionSamples *OutlineFS = Reader->getOrCreateSamplesFor(*Callee);
1128 OutlineFS->merge(*FS);
1129 }
1130 } else {
1131 auto pair =
1132 notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
1133 pair.first->second.entryCount += FS->getEntrySamples();
1134 }
1135 }
1136 return Changed;
1137}
1138
1139bool SampleProfileLoader::tryInlineCandidate(
1140 InlineCandidate &Candidate, SmallVector<CallBase *, 8> *InlinedCallSites) {
1141
1142 CallBase &CB = *Candidate.CallInstr;
1143 Function *CalledFunction = CB.getCalledFunction();
1144 assert(CalledFunction && "Expect a callee with definition")((CalledFunction && "Expect a callee with definition"
) ? static_cast<void> (0) : __assert_fail ("CalledFunction && \"Expect a callee with definition\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1144, __PRETTY_FUNCTION__))
;
1145 DebugLoc DLoc = CB.getDebugLoc();
1146 BasicBlock *BB = CB.getParent();
1147
1148 InlineCost Cost = shouldInlineCandidate(Candidate);
1149 if (Cost.isNever()) {
1150 ORE->emit(OptimizationRemarkAnalysis(CSINLINE_DEBUG"sample-profile" "-inline", "InlineFail", DLoc, BB)
1151 << "incompatible inlining");
1152 return false;
1153 }
1154
1155 if (!Cost)
1156 return false;
1157
1158 InlineFunctionInfo IFI(nullptr, GetAC);
1159 IFI.UpdateProfile = false;
1160 if (InlineFunction(CB, IFI).isSuccess()) {
1161 // The call to InlineFunction erases I, so we can't pass it here.
1162 emitInlinedInto(*ORE, DLoc, BB, *CalledFunction, *BB->getParent(), Cost,
1163 true, CSINLINE_DEBUG"sample-profile" "-inline");
1164
1165 // Now populate the list of newly exposed call sites.
1166 if (InlinedCallSites) {
1167 InlinedCallSites->clear();
1168 for (auto &I : IFI.InlinedCallSites)
1169 InlinedCallSites->push_back(I);
1170 }
1171
1172 if (ProfileIsCS)
1173 ContextTracker->markContextSamplesInlined(Candidate.CalleeSamples);
1174 ++NumCSInlined;
1175
1176 // Prorate inlined probes for a duplicated inlining callsite which probably
1177 // has a distribution less than 100%. Samples for an inlinee should be
1178 // distributed among the copies of the original callsite based on each
1179 // callsite's distribution factor for counts accuracy. Note that an inlined
1180 // probe may come with its own distribution factor if it has been duplicated
1181 // in the inlinee body. The two factor are multiplied to reflect the
1182 // aggregation of duplication.
1183 if (Candidate.CallsiteDistribution < 1) {
1184 for (auto &I : IFI.InlinedCallSites) {
1185 if (Optional<PseudoProbe> Probe = extractProbe(*I))
1186 setProbeDistributionFactor(*I, Probe->Factor *
1187 Candidate.CallsiteDistribution);
1188 }
1189 NumDuplicatedInlinesite++;
1190 }
1191
1192 return true;
1193 }
1194 return false;
1195}
1196
1197bool SampleProfileLoader::getInlineCandidate(InlineCandidate *NewCandidate,
1198 CallBase *CB) {
1199 assert(CB && "Expect non-null call instruction")((CB && "Expect non-null call instruction") ? static_cast
<void> (0) : __assert_fail ("CB && \"Expect non-null call instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1199, __PRETTY_FUNCTION__))
;
1200
1201 if (isa<IntrinsicInst>(CB))
1202 return false;
1203
1204 // Find the callee's profile. For indirect call, find hottest target profile.
1205 const FunctionSamples *CalleeSamples = findCalleeFunctionSamples(*CB);
1206 if (!CalleeSamples)
1207 return false;
1208
1209 float Factor = 1.0;
1210 if (Optional<PseudoProbe> Probe = extractProbe(*CB))
1211 Factor = Probe->Factor;
1212
1213 uint64_t CallsiteCount = 0;
1214 ErrorOr<uint64_t> Weight = getBlockWeight(CB->getParent());
1215 if (Weight)
1216 CallsiteCount = Weight.get();
1217 if (CalleeSamples)
1218 CallsiteCount = std::max(
1219 CallsiteCount, uint64_t(CalleeSamples->getEntrySamples() * Factor));
1220
1221 *NewCandidate = {CB, CalleeSamples, CallsiteCount, Factor};
1222 return true;
1223}
1224
1225InlineCost
1226SampleProfileLoader::shouldInlineCandidate(InlineCandidate &Candidate) {
1227 std::unique_ptr<InlineAdvice> Advice = nullptr;
1228 if (ExternalInlineAdvisor) {
1229 Advice = ExternalInlineAdvisor->getAdvice(*Candidate.CallInstr);
1230 if (!Advice->isInliningRecommended()) {
1231 Advice->recordUnattemptedInlining();
1232 return InlineCost::getNever("not previously inlined");
1233 }
1234 Advice->recordInlining();
1235 return InlineCost::getAlways("previously inlined");
1236 }
1237
1238 // Adjust threshold based on call site hotness, only do this for callsite
1239 // prioritized inliner because otherwise cost-benefit check is done earlier.
1240 int SampleThreshold = SampleColdCallSiteThreshold;
1241 if (CallsitePrioritizedInline) {
1242 if (Candidate.CallsiteCount > PSI->getHotCountThreshold())
1243 SampleThreshold = SampleHotCallSiteThreshold;
1244 else if (!ProfileSizeInline)
1245 return InlineCost::getNever("cold callsite");
1246 }
1247
1248 Function *Callee = Candidate.CallInstr->getCalledFunction();
1249 assert(Callee && "Expect a definition for inline candidate of direct call")((Callee && "Expect a definition for inline candidate of direct call"
) ? static_cast<void> (0) : __assert_fail ("Callee && \"Expect a definition for inline candidate of direct call\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1249, __PRETTY_FUNCTION__))
;
1250
1251 InlineParams Params = getInlineParams();
1252 Params.ComputeFullInlineCost = true;
1253 // Checks if there is anything in the reachable portion of the callee at
1254 // this callsite that makes this inlining potentially illegal. Need to
1255 // set ComputeFullInlineCost, otherwise getInlineCost may return early
1256 // when cost exceeds threshold without checking all IRs in the callee.
1257 // The acutal cost does not matter because we only checks isNever() to
1258 // see if it is legal to inline the callsite.
1259 InlineCost Cost = getInlineCost(*Candidate.CallInstr, Callee, Params,
1260 GetTTI(*Callee), GetAC, GetTLI);
1261
1262 // Honor always inline and never inline from call analyzer
1263 if (Cost.isNever() || Cost.isAlways())
1264 return Cost;
1265
1266 // For old FDO inliner, we inline the call site as long as cost is not
1267 // "Never". The cost-benefit check is done earlier.
1268 if (!CallsitePrioritizedInline) {
1269 return InlineCost::get(Cost.getCost(), INT_MAX2147483647);
1270 }
1271
1272 // Otherwise only use the cost from call analyzer, but overwite threshold with
1273 // Sample PGO threshold.
1274 return InlineCost::get(Cost.getCost(), SampleThreshold);
1275}
1276
1277bool SampleProfileLoader::inlineHotFunctionsWithPriority(
1278 Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
1279 assert(ProfileIsCS && "Prioritiy based inliner only works with CSSPGO now")((ProfileIsCS && "Prioritiy based inliner only works with CSSPGO now"
) ? static_cast<void> (0) : __assert_fail ("ProfileIsCS && \"Prioritiy based inliner only works with CSSPGO now\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1279, __PRETTY_FUNCTION__))
;
1
Assuming field 'ProfileIsCS' is true
2
'?' condition is true
1280
1281 // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
1282 // Profile symbol list is ignored when profile-sample-accurate is on.
1283 assert((!ProfAccForSymsInList ||(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1287, __PRETTY_FUNCTION__))
3
Assuming field 'ProfAccForSymsInList' is true
4
Assuming the condition is true
5
Assuming the condition is true
6
'?' condition is true
1284 (!ProfileSampleAccurate &&(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1287, __PRETTY_FUNCTION__))
1285 !F.hasFnAttribute("profile-sample-accurate"))) &&(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1287, __PRETTY_FUNCTION__))
1286 "ProfAccForSymsInList should be false when profile-sample-accurate "(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1287, __PRETTY_FUNCTION__))
1287 "is enabled")(((!ProfAccForSymsInList || (!ProfileSampleAccurate &&
!F.hasFnAttribute("profile-sample-accurate"))) && "ProfAccForSymsInList should be false when profile-sample-accurate "
"is enabled") ? static_cast<void> (0) : __assert_fail (
"(!ProfAccForSymsInList || (!ProfileSampleAccurate && !F.hasFnAttribute(\"profile-sample-accurate\"))) && \"ProfAccForSymsInList should be false when profile-sample-accurate \" \"is enabled\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1287, __PRETTY_FUNCTION__))
;
1288
1289 // Populating worklist with initial call sites from root inliner, along
1290 // with call site weights.
1291 CandidateQueue CQueue;
1292 InlineCandidate NewCandidate;
1293 for (auto &BB : F) {
1294 for (auto &I : BB.getInstList()) {
1295 auto *CB = dyn_cast<CallBase>(&I);
1296 if (!CB)
1297 continue;
1298 if (getInlineCandidate(&NewCandidate, CB))
1299 CQueue.push(NewCandidate);
1300 }
1301 }
1302
1303 // Cap the size growth from profile guided inlining. This is needed even
1304 // though cost of each inline candidate already accounts for callee size,
1305 // because with top-down inlining, we can grow inliner size significantly
1306 // with large number of smaller inlinees each pass the cost check.
1307 assert(ProfileInlineLimitMax >= ProfileInlineLimitMin &&((ProfileInlineLimitMax >= ProfileInlineLimitMin &&
"Max inline size limit should not be smaller than min inline size "
"limit.") ? static_cast<void> (0) : __assert_fail ("ProfileInlineLimitMax >= ProfileInlineLimitMin && \"Max inline size limit should not be smaller than min inline size \" \"limit.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1309, __PRETTY_FUNCTION__))
7
Assuming the condition is true
8
'?' condition is true
1308 "Max inline size limit should not be smaller than min inline size "((ProfileInlineLimitMax >= ProfileInlineLimitMin &&
"Max inline size limit should not be smaller than min inline size "
"limit.") ? static_cast<void> (0) : __assert_fail ("ProfileInlineLimitMax >= ProfileInlineLimitMin && \"Max inline size limit should not be smaller than min inline size \" \"limit.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1309, __PRETTY_FUNCTION__))
1309 "limit.")((ProfileInlineLimitMax >= ProfileInlineLimitMin &&
"Max inline size limit should not be smaller than min inline size "
"limit.") ? static_cast<void> (0) : __assert_fail ("ProfileInlineLimitMax >= ProfileInlineLimitMin && \"Max inline size limit should not be smaller than min inline size \" \"limit.\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1309, __PRETTY_FUNCTION__))
;
1310 unsigned SizeLimit = F.getInstructionCount() * ProfileInlineGrowthLimit;
1311 SizeLimit = std::min(SizeLimit, (unsigned)ProfileInlineLimitMax);
1312 SizeLimit = std::max(SizeLimit, (unsigned)ProfileInlineLimitMin);
1313 if (ExternalInlineAdvisor)
9
Taking true branch
1314 SizeLimit = std::numeric_limits<unsigned>::max();
1315
1316 // Perform iterative BFS call site prioritized inlining
1317 bool Changed = false;
1318 while (!CQueue.empty() && F.getInstructionCount() < SizeLimit) {
10
Assuming the condition is true
11
Assuming the condition is true
12
Loop condition is true. Entering loop body
1319 InlineCandidate Candidate = CQueue.top();
1320 CQueue.pop();
1321 CallBase *I = Candidate.CallInstr;
1322 Function *CalledFunction = I->getCalledFunction();
1323
1324 if (CalledFunction == &F)
13
Taking false branch
1325 continue;
1326 if (I->isIndirectCall()) {
14
Assuming the condition is true
15
Taking true branch
1327 uint64_t Sum;
16
'Sum' declared without an initial value
1328 auto CalleeSamples = findIndirectCallFunctionSamples(*I, Sum);
17
Calling 'SampleProfileLoader::findIndirectCallFunctionSamples'
21
Returning from 'SampleProfileLoader::findIndirectCallFunctionSamples'
1329 uint64_t SumOrigin = Sum;
22
Assigned value is garbage or undefined
1330 Sum *= Candidate.CallsiteDistribution;
1331 for (const auto *FS : CalleeSamples) {
1332 // TODO: Consider disable pre-lTO ICP for MonoLTO as well
1333 if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1334 findExternalInlineCandidate(FS, InlinedGUIDs, SymbolMap,
1335 PSI->getOrCompHotCountThreshold());
1336 continue;
1337 }
1338 uint64_t EntryCountDistributed =
1339 FS->getEntrySamples() * Candidate.CallsiteDistribution;
1340 // In addition to regular inline cost check, we also need to make sure
1341 // ICP isn't introducing excessive speculative checks even if individual
1342 // target looks beneficial to promote and inline. That means we should
1343 // only do ICP when there's a small number dominant targets.
1344 if (EntryCountDistributed < SumOrigin / ProfileICPThreshold)
1345 break;
1346 // TODO: Fix CallAnalyzer to handle all indirect calls.
1347 // For indirect call, we don't run CallAnalyzer to get InlineCost
1348 // before actual inlining. This is because we could see two different
1349 // types from the same definition, which makes CallAnalyzer choke as
1350 // it's expecting matching parameter type on both caller and callee
1351 // side. See example from PR18962 for the triggering cases (the bug was
1352 // fixed, but we generate different types).
1353 if (!PSI->isHotCount(EntryCountDistributed))
1354 break;
1355 SmallVector<CallBase *, 8> InlinedCallSites;
1356 // Attach function profile for promoted indirect callee, and update
1357 // call site count for the promoted inline candidate too.
1358 Candidate = {I, FS, EntryCountDistributed,
1359 Candidate.CallsiteDistribution};
1360 if (tryPromoteAndInlineCandidate(F, Candidate, SumOrigin, Sum,
1361 &InlinedCallSites)) {
1362 for (auto *CB : InlinedCallSites) {
1363 if (getInlineCandidate(&NewCandidate, CB))
1364 CQueue.emplace(NewCandidate);
1365 }
1366 Changed = true;
1367 }
1368 }
1369 } else if (CalledFunction && CalledFunction->getSubprogram() &&
1370 !CalledFunction->isDeclaration()) {
1371 SmallVector<CallBase *, 8> InlinedCallSites;
1372 if (tryInlineCandidate(Candidate, &InlinedCallSites)) {
1373 for (auto *CB : InlinedCallSites) {
1374 if (getInlineCandidate(&NewCandidate, CB))
1375 CQueue.emplace(NewCandidate);
1376 }
1377 Changed = true;
1378 }
1379 } else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
1380 findExternalInlineCandidate(Candidate.CalleeSamples, InlinedGUIDs,
1381 SymbolMap, PSI->getOrCompHotCountThreshold());
1382 }
1383 }
1384
1385 if (!CQueue.empty()) {
1386 if (SizeLimit == (unsigned)ProfileInlineLimitMax)
1387 ++NumCSInlinedHitMaxLimit;
1388 else if (SizeLimit == (unsigned)ProfileInlineLimitMin)
1389 ++NumCSInlinedHitMinLimit;
1390 else
1391 ++NumCSInlinedHitGrowthLimit;
1392 }
1393
1394 return Changed;
1395}
1396
1397/// Returns the sorted CallTargetMap \p M by count in descending order.
1398static SmallVector<InstrProfValueData, 2>
1399GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) {
1400 SmallVector<InstrProfValueData, 2> R;
1401 for (const auto &I : SampleRecord::SortCallTargets(M)) {
1402 R.emplace_back(
1403 InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
1404 }
1405 return R;
1406}
1407
1408// Generate MD_prof metadata for every branch instruction using the
1409// edge weights computed during propagation.
1410void SampleProfileLoader::generateMDProfMetadata(Function &F) {
1411 // Generate MD_prof metadata for every branch instruction using the
1412 // edge weights computed during propagation.
1413 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch weights\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nPropagation complete. Setting branch weights\n"
; } } while (false)
;
1414 LLVMContext &Ctx = F.getContext();
1415 MDBuilder MDB(Ctx);
1416 for (auto &BI : F) {
1417 BasicBlock *BB = &BI;
1418
1419 if (BlockWeights[BB]) {
1420 for (auto &I : BB->getInstList()) {
1421 if (!isa<CallInst>(I) && !isa<InvokeInst>(I))
1422 continue;
1423 if (!cast<CallBase>(I).getCalledFunction()) {
1424 const DebugLoc &DLoc = I.getDebugLoc();
1425 if (!DLoc)
1426 continue;
1427 const DILocation *DIL = DLoc;
1428 const FunctionSamples *FS = findFunctionSamples(I);
1429 if (!FS)
1430 continue;
1431 auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL);
1432 auto T = FS->findCallTargetMapAt(CallSite);
1433 if (!T || T.get().empty())
1434 continue;
1435 // Prorate the callsite counts to reflect what is already done to the
1436 // callsite, such as ICP or calliste cloning.
1437 if (FunctionSamples::ProfileIsProbeBased) {
1438 if (Optional<PseudoProbe> Probe = extractProbe(I)) {
1439 if (Probe->Factor < 1)
1440 T = SampleRecord::adjustCallTargets(T.get(), Probe->Factor);
1441 }
1442 }
1443 SmallVector<InstrProfValueData, 2> SortedCallTargets =
1444 GetSortedValueDataFromCallTargets(T.get());
1445 uint64_t Sum = 0;
1446 for (const auto &C : T.get())
1447 Sum += C.second;
1448 // With CSSPGO all indirect call targets are counted torwards the
1449 // original indirect call site in the profile, including both
1450 // inlined and non-inlined targets.
1451 if (!FunctionSamples::ProfileIsCS) {
1452 if (const FunctionSamplesMap *M =
1453 FS->findFunctionSamplesMapAt(CallSite)) {
1454 for (const auto &NameFS : *M)
1455 Sum += NameFS.second.getEntrySamples();
1456 }
1457 }
1458 if (!Sum)
1459 continue;
1460 updateIDTMetaData(I, SortedCallTargets, Sum);
1461 } else if (!isa<IntrinsicInst>(&I)) {
1462 I.setMetadata(LLVMContext::MD_prof,
1463 MDB.createBranchWeights(
1464 {static_cast<uint32_t>(BlockWeights[BB])}));
1465 }
1466 }
1467 }
1468 Instruction *TI = BB->getTerminator();
1469 if (TI->getNumSuccessors() == 1)
1470 continue;
1471 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI) &&
1472 !isa<IndirectBrInst>(TI))
1473 continue;
1474
1475 DebugLoc BranchLoc = TI->getDebugLoc();
1476 LLVM_DEBUG(dbgs() << "\nGetting weights for branch at line "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
1477 << ((BranchLoc) ? Twine(BranchLoc.getLine())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
1478 : Twine("<UNKNOWN LOCATION>"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
1479 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\nGetting weights for branch at line "
<< ((BranchLoc) ? Twine(BranchLoc.getLine()) : Twine("<UNKNOWN LOCATION>"
)) << ".\n"; } } while (false)
;
1480 SmallVector<uint32_t, 4> Weights;
1481 uint32_t MaxWeight = 0;
1482 Instruction *MaxDestInst;
1483 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) {
1484 BasicBlock *Succ = TI->getSuccessor(I);
1485 Edge E = std::make_pair(BB, Succ);
1486 uint64_t Weight = EdgeWeights[E];
1487 LLVM_DEBUG(dbgs() << "\t"; printEdgeWeight(dbgs(), E))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\t"; printEdgeWeight(dbgs
(), E); } } while (false)
;
1488 // Use uint32_t saturated arithmetic to adjust the incoming weights,
1489 // if needed. Sample counts in profiles are 64-bit unsigned values,
1490 // but internally branch weights are expressed as 32-bit values.
1491 if (Weight > std::numeric_limits<uint32_t>::max()) {
1492 LLVM_DEBUG(dbgs() << " (saturated due to uint32_t overflow)")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << " (saturated due to uint32_t overflow)"
; } } while (false)
;
1493 Weight = std::numeric_limits<uint32_t>::max();
1494 }
1495 // Weight is added by one to avoid propagation errors introduced by
1496 // 0 weights.
1497 Weights.push_back(static_cast<uint32_t>(Weight + 1));
1498 if (Weight != 0) {
1499 if (Weight > MaxWeight) {
1500 MaxWeight = Weight;
1501 MaxDestInst = Succ->getFirstNonPHIOrDbgOrLifetime();
1502 }
1503 }
1504 }
1505
1506 uint64_t TempWeight;
1507 // Only set weights if there is at least one non-zero weight.
1508 // In any other case, let the analyzer set weights.
1509 // Do not set weights if the weights are present. In ThinLTO, the profile
1510 // annotation is done twice. If the first annotation already set the
1511 // weights, the second pass does not need to set it.
1512 if (MaxWeight > 0 && !TI->extractProfTotalWeight(TempWeight)) {
1513 LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "SUCCESS. Found non-zero weights.\n"
; } } while (false)
;
1514 TI->setMetadata(LLVMContext::MD_prof,
1515 MDB.createBranchWeights(Weights));
1516 ORE->emit([&]() {
1517 return OptimizationRemark(DEBUG_TYPE"sample-profile", "PopularDest", MaxDestInst)
1518 << "most popular destination for conditional branches at "
1519 << ore::NV("CondBranchesLoc", BranchLoc);
1520 });
1521 } else {
1522 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "SKIPPED. All branch weights are zero.\n"
; } } while (false)
;
1523 }
1524 }
1525}
1526
1527/// Once all the branch weights are computed, we emit the MD_prof
1528/// metadata on BB using the computed values for each of its branches.
1529///
1530/// \param F The function to query.
1531///
1532/// \returns true if \p F was modified. Returns false, otherwise.
1533bool SampleProfileLoader::emitAnnotations(Function &F) {
1534 bool Changed = false;
1535
1536 if (FunctionSamples::ProfileIsProbeBased) {
1537 if (!ProbeManager->profileIsValid(F, *Samples)) {
1538 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Profile is invalid due to CFG mismatch for Function "
<< F.getName(); } } while (false)
1539 dbgs() << "Profile is invalid due to CFG mismatch for Function "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Profile is invalid due to CFG mismatch for Function "
<< F.getName(); } } while (false)
1540 << F.getName())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Profile is invalid due to CFG mismatch for Function "
<< F.getName(); } } while (false)
;
1541 ++NumMismatchedProfile;
1542 return false;
1543 }
1544 ++NumMatchedProfile;
1545 } else {
1546 if (getFunctionLoc(F) == 0)
1547 return false;
1548
1549 LLVM_DEBUG(dbgs() << "Line number for the first instruction in "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Line number for the first instruction in "
<< F.getName() << ": " << getFunctionLoc(F
) << "\n"; } } while (false)
1550 << F.getName() << ": " << getFunctionLoc(F) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "Line number for the first instruction in "
<< F.getName() << ": " << getFunctionLoc(F
) << "\n"; } } while (false)
;
1551 }
1552
1553 DenseSet<GlobalValue::GUID> InlinedGUIDs;
1554 if (ProfileIsCS && CallsitePrioritizedInline)
1555 Changed |= inlineHotFunctionsWithPriority(F, InlinedGUIDs);
1556 else
1557 Changed |= inlineHotFunctions(F, InlinedGUIDs);
1558
1559 Changed |= computeAndPropagateWeights(F, InlinedGUIDs);
1560
1561 if (Changed)
1562 generateMDProfMetadata(F);
1563
1564 emitCoverageRemarks(F);
1565 return Changed;
1566}
1567
1568char SampleProfileLoaderLegacyPass::ID = 0;
1569
1570INITIALIZE_PASS_BEGIN(SampleProfileLoaderLegacyPass, "sample-profile",static void *initializeSampleProfileLoaderLegacyPassPassOnce(
PassRegistry &Registry) {
1571 "Sample Profile loader", false, false)static void *initializeSampleProfileLoaderLegacyPassPassOnce(
PassRegistry &Registry) {
1572INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
1573INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
1574INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)initializeTargetLibraryInfoWrapperPassPass(Registry);
1575INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)initializeProfileSummaryInfoWrapperPassPass(Registry);
1576INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile",PassInfo *PI = new PassInfo( "Sample Profile loader", "sample-profile"
, &SampleProfileLoaderLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<SampleProfileLoaderLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeSampleProfileLoaderLegacyPassPassFlag
; void llvm::initializeSampleProfileLoaderLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeSampleProfileLoaderLegacyPassPassFlag
, initializeSampleProfileLoaderLegacyPassPassOnce, std::ref(Registry
)); }
1577 "Sample Profile loader", false, false)PassInfo *PI = new PassInfo( "Sample Profile loader", "sample-profile"
, &SampleProfileLoaderLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<SampleProfileLoaderLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeSampleProfileLoaderLegacyPassPassFlag
; void llvm::initializeSampleProfileLoaderLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeSampleProfileLoaderLegacyPassPassFlag
, initializeSampleProfileLoaderLegacyPassPassOnce, std::ref(Registry
)); }
1578
1579std::unique_ptr<ProfiledCallGraph>
1580SampleProfileLoader::buildProfiledCallGraph(CallGraph &CG) {
1581 std::unique_ptr<ProfiledCallGraph> ProfiledCG;
1582 if (ProfileIsCS)
1583 ProfiledCG = std::make_unique<ProfiledCallGraph>(*ContextTracker);
1584 else
1585 ProfiledCG = std::make_unique<ProfiledCallGraph>(Reader->getProfiles());
1586
1587 // Add all functions into the profiled call graph even if they are not in
1588 // the profile. This makes sure functions missing from the profile still
1589 // gets a chance to be processed.
1590 for (auto &Node : CG) {
1591 const auto *F = Node.first;
1592 if (!F || F->isDeclaration() || !F->hasFnAttribute("use-sample-profile"))
1593 continue;
1594 ProfiledCG->addProfiledFunction(FunctionSamples::getCanonicalFnName(*F));
1595 }
1596
1597 return ProfiledCG;
1598}
1599
1600std::vector<Function *>
1601SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) {
1602 std::vector<Function *> FunctionOrderList;
1603 FunctionOrderList.reserve(M.size());
1604
1605 if (!ProfileTopDownLoad && UseProfiledCallGraph)
1606 errs() << "WARNING: -use-profiled-call-graph ignored, should be used "
1607 "together with -sample-profile-top-down-load.\n";
1608
1609 if (!ProfileTopDownLoad || CG == nullptr) {
1610 if (ProfileMergeInlinee) {
1611 // Disable ProfileMergeInlinee if profile is not loaded in top down order,
1612 // because the profile for a function may be used for the profile
1613 // annotation of its outline copy before the profile merging of its
1614 // non-inlined inline instances, and that is not the way how
1615 // ProfileMergeInlinee is supposed to work.
1616 ProfileMergeInlinee = false;
1617 }
1618
1619 for (Function &F : M)
1620 if (!F.isDeclaration() && F.hasFnAttribute("use-sample-profile"))
1621 FunctionOrderList.push_back(&F);
1622 return FunctionOrderList;
1623 }
1624
1625 assert(&CG->getModule() == &M)((&CG->getModule() == &M) ? static_cast<void>
(0) : __assert_fail ("&CG->getModule() == &M", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1625, __PRETTY_FUNCTION__))
;
1626
1627 if (UseProfiledCallGraph ||
1628 (ProfileIsCS && !UseProfiledCallGraph.getNumOccurrences())) {
1629 // Use profiled call edges to augment the top-down order. There are cases
1630 // that the top-down order computed based on the static call graph doesn't
1631 // reflect real execution order. For example
1632 //
1633 // 1. Incomplete static call graph due to unknown indirect call targets.
1634 // Adjusting the order by considering indirect call edges from the
1635 // profile can enable the inlining of indirect call targets by allowing
1636 // the caller processed before them.
1637 // 2. Mutual call edges in an SCC. The static processing order computed for
1638 // an SCC may not reflect the call contexts in the context-sensitive
1639 // profile, thus may cause potential inlining to be overlooked. The
1640 // function order in one SCC is being adjusted to a top-down order based
1641 // on the profile to favor more inlining. This is only a problem with CS
1642 // profile.
1643 // 3. Transitive indirect call edges due to inlining. When a callee function
1644 // (say B) is inlined into into a caller function (say A) in LTO prelink,
1645 // every call edge originated from the callee B will be transferred to
1646 // the caller A. If any transferred edge (say A->C) is indirect, the
1647 // original profiled indirect edge B->C, even if considered, would not
1648 // enforce a top-down order from the caller A to the potential indirect
1649 // call target C in LTO postlink since the inlined callee B is gone from
1650 // the static call graph.
1651 // 4. #3 can happen even for direct call targets, due to functions defined
1652 // in header files. A header function (say A), when included into source
1653 // files, is defined multiple times but only one definition survives due
1654 // to ODR. Therefore, the LTO prelink inlining done on those dropped
1655 // definitions can be useless based on a local file scope. More
1656 // importantly, the inlinee (say B), once fully inlined to a
1657 // to-be-dropped A, will have no profile to consume when its outlined
1658 // version is compiled. This can lead to a profile-less prelink
1659 // compilation for the outlined version of B which may be called from
1660 // external modules. while this isn't easy to fix, we rely on the
1661 // postlink AutoFDO pipeline to optimize B. Since the survived copy of
1662 // the A can be inlined in its local scope in prelink, it may not exist
1663 // in the merged IR in postlink, and we'll need the profiled call edges
1664 // to enforce a top-down order for the rest of the functions.
1665 //
1666 // Considering those cases, a profiled call graph completely independent of
1667 // the static call graph is constructed based on profile data, where
1668 // function objects are not even needed to handle case #3 and case 4.
1669 //
1670 // Note that static callgraph edges are completely ignored since they
1671 // can be conflicting with profiled edges for cyclic SCCs and may result in
1672 // an SCC order incompatible with profile-defined one. Using strictly
1673 // profile order ensures a maximum inlining experience. On the other hand,
1674 // static call edges are not so important when they don't correspond to a
1675 // context in the profile.
1676
1677 std::unique_ptr<ProfiledCallGraph> ProfiledCG = buildProfiledCallGraph(*CG);
1678 scc_iterator<ProfiledCallGraph *> CGI = scc_begin(ProfiledCG.get());
1679 while (!CGI.isAtEnd()) {
1680 for (ProfiledCallGraphNode *Node : *CGI) {
1681 Function *F = SymbolMap.lookup(Node->Name);
1682 if (F && !F->isDeclaration() && F->hasFnAttribute("use-sample-profile"))
1683 FunctionOrderList.push_back(F);
1684 }
1685 ++CGI;
1686 }
1687 } else {
1688 scc_iterator<CallGraph *> CGI = scc_begin(CG);
1689 while (!CGI.isAtEnd()) {
1690 for (CallGraphNode *Node : *CGI) {
1691 auto *F = Node->getFunction();
1692 if (F && !F->isDeclaration() && F->hasFnAttribute("use-sample-profile"))
1693 FunctionOrderList.push_back(F);
1694 }
1695 ++CGI;
1696 }
1697 }
1698
1699 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { { dbgs() << "Function processing order:\n"
; for (auto F : reverse(FunctionOrderList)) { dbgs() <<
F->getName() << "\n"; } }; } } while (false)
1700 dbgs() << "Function processing order:\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { { dbgs() << "Function processing order:\n"
; for (auto F : reverse(FunctionOrderList)) { dbgs() <<
F->getName() << "\n"; } }; } } while (false)
1701 for (auto F : reverse(FunctionOrderList)) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { { dbgs() << "Function processing order:\n"
; for (auto F : reverse(FunctionOrderList)) { dbgs() <<
F->getName() << "\n"; } }; } } while (false)
1702 dbgs() << F->getName() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { { dbgs() << "Function processing order:\n"
; for (auto F : reverse(FunctionOrderList)) { dbgs() <<
F->getName() << "\n"; } }; } } while (false)
1703 }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { { dbgs() << "Function processing order:\n"
; for (auto F : reverse(FunctionOrderList)) { dbgs() <<
F->getName() << "\n"; } }; } } while (false)
1704 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { { dbgs() << "Function processing order:\n"
; for (auto F : reverse(FunctionOrderList)) { dbgs() <<
F->getName() << "\n"; } }; } } while (false)
;
1705
1706 std::reverse(FunctionOrderList.begin(), FunctionOrderList.end());
1707 return FunctionOrderList;
1708}
1709
1710bool SampleProfileLoader::doInitialization(Module &M,
1711 FunctionAnalysisManager *FAM) {
1712 auto &Ctx = M.getContext();
1713
1714 auto ReaderOrErr =
1715 SampleProfileReader::create(Filename, Ctx, RemappingFilename);
1716 if (std::error_code EC = ReaderOrErr.getError()) {
1717 std::string Msg = "Could not open profile: " + EC.message();
1718 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1719 return false;
1720 }
1721 Reader = std::move(ReaderOrErr.get());
1722 Reader->setSkipFlatProf(LTOPhase == ThinOrFullLTOPhase::ThinLTOPostLink);
1723 // set module before reading the profile so reader may be able to only
1724 // read the function profiles which are used by the current module.
1725 Reader->setModule(&M);
1726 if (std::error_code EC = Reader->read()) {
1727 std::string Msg = "profile reading failed: " + EC.message();
1728 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1729 return false;
1730 }
1731
1732 PSL = Reader->getProfileSymbolList();
1733
1734 // While profile-sample-accurate is on, ignore symbol list.
1735 ProfAccForSymsInList =
1736 ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate;
1737 if (ProfAccForSymsInList) {
1738 NamesInProfile.clear();
1739 if (auto NameTable = Reader->getNameTable())
1740 NamesInProfile.insert(NameTable->begin(), NameTable->end());
1741 CoverageTracker.setProfAccForSymsInList(true);
1742 }
1743
1744 if (FAM && !ProfileInlineReplayFile.empty()) {
1745 ExternalInlineAdvisor = std::make_unique<ReplayInlineAdvisor>(
1746 M, *FAM, Ctx, /*OriginalAdvisor=*/nullptr, ProfileInlineReplayFile,
1747 /*EmitRemarks=*/false);
1748 if (!ExternalInlineAdvisor->areReplayRemarksLoaded())
1749 ExternalInlineAdvisor.reset();
1750 }
1751
1752 // Apply tweaks if context-sensitive profile is available.
1753 if (Reader->profileIsCS()) {
1754 ProfileIsCS = true;
1755 FunctionSamples::ProfileIsCS = true;
1756
1757 // Enable priority-base inliner and size inline by default for CSSPGO.
1758 if (!ProfileSizeInline.getNumOccurrences())
1759 ProfileSizeInline = true;
1760 if (!CallsitePrioritizedInline.getNumOccurrences())
1761 CallsitePrioritizedInline = true;
1762
1763 // Tracker for profiles under different context
1764 ContextTracker =
1765 std::make_unique<SampleContextTracker>(Reader->getProfiles());
1766 }
1767
1768 // Load pseudo probe descriptors for probe-based function samples.
1769 if (Reader->profileIsProbeBased()) {
1770 ProbeManager = std::make_unique<PseudoProbeManager>(M);
1771 if (!ProbeManager->moduleIsProbed(M)) {
1772 const char *Msg =
1773 "Pseudo-probe-based profile requires SampleProfileProbePass";
1774 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
1775 return false;
1776 }
1777 }
1778
1779 return true;
1780}
1781
1782ModulePass *llvm::createSampleProfileLoaderPass() {
1783 return new SampleProfileLoaderLegacyPass();
1784}
1785
1786ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) {
1787 return new SampleProfileLoaderLegacyPass(Name);
1788}
1789
1790bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
1791 ProfileSummaryInfo *_PSI, CallGraph *CG) {
1792 GUIDToFuncNameMapper Mapper(M, *Reader, GUIDToFuncNameMap);
1793
1794 PSI = _PSI;
1795 if (M.getProfileSummary(/* IsCS */ false) == nullptr) {
1796 M.setProfileSummary(Reader->getSummary().getMD(M.getContext()),
1797 ProfileSummary::PSK_Sample);
1798 PSI->refresh();
1799 }
1800 // Compute the total number of samples collected in this profile.
1801 for (const auto &I : Reader->getProfiles())
1802 TotalCollectedSamples += I.second.getTotalSamples();
1803
1804 auto Remapper = Reader->getRemapper();
1805 // Populate the symbol map.
1806 for (const auto &N_F : M.getValueSymbolTable()) {
1807 StringRef OrigName = N_F.getKey();
1808 Function *F = dyn_cast<Function>(N_F.getValue());
1809 if (F == nullptr || OrigName.empty())
1810 continue;
1811 SymbolMap[OrigName] = F;
1812 StringRef NewName = FunctionSamples::getCanonicalFnName(*F);
1813 if (OrigName != NewName && !NewName.empty()) {
1814 auto r = SymbolMap.insert(std::make_pair(NewName, F));
1815 // Failiing to insert means there is already an entry in SymbolMap,
1816 // thus there are multiple functions that are mapped to the same
1817 // stripped name. In this case of name conflicting, set the value
1818 // to nullptr to avoid confusion.
1819 if (!r.second)
1820 r.first->second = nullptr;
1821 OrigName = NewName;
1822 }
1823 // Insert the remapped names into SymbolMap.
1824 if (Remapper) {
1825 if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
1826 if (*MapName != OrigName && !MapName->empty())
1827 SymbolMap.insert(std::make_pair(*MapName, F));
1828 }
1829 }
1830 }
1831 assert(SymbolMap.count(StringRef()) == 0 &&((SymbolMap.count(StringRef()) == 0 && "No empty StringRef should be added in SymbolMap"
) ? static_cast<void> (0) : __assert_fail ("SymbolMap.count(StringRef()) == 0 && \"No empty StringRef should be added in SymbolMap\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1832, __PRETTY_FUNCTION__))
1832 "No empty StringRef should be added in SymbolMap")((SymbolMap.count(StringRef()) == 0 && "No empty StringRef should be added in SymbolMap"
) ? static_cast<void> (0) : __assert_fail ("SymbolMap.count(StringRef()) == 0 && \"No empty StringRef should be added in SymbolMap\""
, "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1832, __PRETTY_FUNCTION__))
;
1833
1834 bool retval = false;
1835 for (auto F : buildFunctionOrder(M, CG)) {
1836 assert(!F->isDeclaration())((!F->isDeclaration()) ? static_cast<void> (0) : __assert_fail
("!F->isDeclaration()", "/build/llvm-toolchain-snapshot-13~++20210405022414+5f57793c4fe4/llvm/lib/Transforms/IPO/SampleProfile.cpp"
, 1836, __PRETTY_FUNCTION__))
;
1837 clearFunctionData();
1838 retval |= runOnFunction(*F, AM);
1839 }
1840
1841 // Account for cold calls not inlined....
1842 if (!ProfileIsCS)
1843 for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
1844 notInlinedCallInfo)
1845 updateProfileCallee(pair.first, pair.second.entryCount);
1846
1847 return retval;
1848}
1849
1850bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) {
1851 ACT = &getAnalysis<AssumptionCacheTracker>();
1852 TTIWP = &getAnalysis<TargetTransformInfoWrapperPass>();
1853 TLIWP = &getAnalysis<TargetLibraryInfoWrapperPass>();
1854 ProfileSummaryInfo *PSI =
1855 &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1856 return SampleLoader.runOnModule(M, nullptr, PSI, nullptr);
1857}
1858
1859bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) {
1860 LLVM_DEBUG(dbgs() << "\n\nProcessing Function " << F.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("sample-profile")) { dbgs() << "\n\nProcessing Function "
<< F.getName() << "\n"; } } while (false)
;
1861 DILocation2SampleMap.clear();
1862 // By default the entry count is initialized to -1, which will be treated
1863 // conservatively by getEntryCount as the same as unknown (None). This is
1864 // to avoid newly added code to be treated as cold. If we have samples
1865 // this will be overwritten in emitAnnotations.
1866 uint64_t initialEntryCount = -1;
1867
1868 ProfAccForSymsInList = ProfileAccurateForSymsInList && PSL;
1869 if (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) {
1870 // initialize all the function entry counts to 0. It means all the
1871 // functions without profile will be regarded as cold.
1872 initialEntryCount = 0;
1873 // profile-sample-accurate is a user assertion which has a higher precedence
1874 // than symbol list. When profile-sample-accurate is on, ignore symbol list.
1875 ProfAccForSymsInList = false;
1876 }
1877 CoverageTracker.setProfAccForSymsInList(ProfAccForSymsInList);
1878
1879 // PSL -- profile symbol list include all the symbols in sampled binary.
1880 // If ProfileAccurateForSymsInList is enabled, PSL is used to treat
1881 // old functions without samples being cold, without having to worry
1882 // about new and hot functions being mistakenly treated as cold.
1883 if (ProfAccForSymsInList) {
1884 // Initialize the entry count to 0 for functions in the list.
1885 if (PSL->contains(F.getName()))
1886 initialEntryCount = 0;
1887
1888 // Function in the symbol list but without sample will be regarded as
1889 // cold. To minimize the potential negative performance impact it could
1890 // have, we want to be a little conservative here saying if a function
1891 // shows up in the profile, no matter as outline function, inline instance
1892 // or call targets, treat the function as not being cold. This will handle
1893 // the cases such as most callsites of a function are inlined in sampled
1894 // binary but not inlined in current build (because of source code drift,
1895 // imprecise debug information, or the callsites are all cold individually
1896 // but not cold accumulatively...), so the outline function showing up as
1897 // cold in sampled binary will actually not be cold after current build.
1898 StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
1899 if (NamesInProfile.count(CanonName))
1900 initialEntryCount = -1;
1901 }
1902
1903 // Initialize entry count when the function has no existing entry
1904 // count value.
1905 if (!F.getEntryCount().hasValue())
1906 F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real));
1907 std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
1908 if (AM) {
1909 auto &FAM =
1910 AM->getResult<FunctionAnalysisManagerModuleProxy>(*F.getParent())
1911 .getManager();
1912 ORE = &FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
1913 } else {
1914 OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&F);
1915 ORE = OwnedORE.get();
1916 }
1917
1918 if (ProfileIsCS)
1919 Samples = ContextTracker->getBaseSamplesFor(F);
1920 else
1921 Samples = Reader->getSamplesFor(F);
1922
1923 if (Samples && !Samples->empty())
1924 return emitAnnotations(F);
1925 return false;
1926}
1927
1928PreservedAnalyses SampleProfileLoaderPass::run(Module &M,
1929 ModuleAnalysisManager &AM) {
1930 FunctionAnalysisManager &FAM =
1931 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1932
1933 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
1934 return FAM.getResult<AssumptionAnalysis>(F);
1935 };
1936 auto GetTTI = [&](Function &F) -> TargetTransformInfo & {
1937 return FAM.getResult<TargetIRAnalysis>(F);
1938 };
1939 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
1940 return FAM.getResult<TargetLibraryAnalysis>(F);
1941 };
1942
1943 SampleProfileLoader SampleLoader(
1944 ProfileFileName.empty() ? SampleProfileFile : ProfileFileName,
1945 ProfileRemappingFileName.empty() ? SampleProfileRemappingFile
1946 : ProfileRemappingFileName,
1947 LTOPhase, GetAssumptionCache, GetTTI, GetTLI);
1948
1949 if (!SampleLoader.doInitialization(M, &FAM))
1950 return PreservedAnalyses::all();
1951
1952 ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
1953 CallGraph &CG = AM.getResult<CallGraphAnalysis>(M);
1954 if (!SampleLoader.runOnModule(M, &AM, PSI, &CG))
1955 return PreservedAnalyses::all();
1956
1957 return PreservedAnalyses::none();
1958}