68#define DEBUG_TYPE "inline"
70STATISTIC(NumInlined,
"Number of functions inlined");
71STATISTIC(NumDeleted,
"Number of functions deleted because all callers found");
76 "Cost multiplier to multiply onto inlined call sites where the "
77 "new call was previously an intra-SCC call (not relevant when the "
78 "original call was already intra-SCC). This can accumulate over "
79 "multiple inlinings (e.g. if a call site already had a cost "
80 "multiplier and one of its inlined calls was also subject to "
81 "this, the inlined call would have the original multiplier "
82 "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
83 "inlining through a child SCC which can cause terrible compile times"));
99 "Optimization remarks file containing inline remarks to be replayed "
100 "by cgscc inlining."),
104 "cgscc-inline-replay-scope",
105 cl::init(ReplayInlinerSettings::Scope::Function),
107 "Replay on functions that have remarks associated "
108 "with them (default)"),
109 clEnumValN(ReplayInlinerSettings::Scope::Module,
"Module",
110 "Replay on the entire module")),
111 cl::desc(
"Whether inline replay should be applied to the entire "
112 "Module or just the Functions (default) that are present as "
113 "callers in remarks during cgscc inlining."),
117 "cgscc-inline-replay-fallback",
118 cl::init(ReplayInlinerSettings::Fallback::Original),
121 ReplayInlinerSettings::Fallback::Original,
"Original",
122 "All decisions not in replay send to original advisor (default)"),
123 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
124 "AlwaysInline",
"All decisions not in replay are inlined"),
125 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline,
"NeverInline",
126 "All decisions not in replay are not inlined")),
128 "How cgscc inline replay treats sites that don't come from the replay. "
129 "Original: defers to original advisor, AlwaysInline: inline all sites "
130 "not in replay, NeverInline: inline no sites not in replay"),
134 "cgscc-inline-replay-format",
135 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
137 clEnumValN(CallSiteFormat::Format::Line,
"Line",
"<Line Number>"),
138 clEnumValN(CallSiteFormat::Format::LineColumn,
"LineColumn",
139 "<Line Number>:<Column Number>"),
140 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
141 "LineDiscriminator",
"<Line Number>.<Discriminator>"),
142 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
143 "LineColumnDiscriminator",
144 "<Line Number>:<Column Number>.<Discriminator> (default)")),
152 while (InlineHistoryID != -1) {
153 assert(
unsigned(InlineHistoryID) < InlineHistory.size() &&
154 "Invalid inline history ID");
155 if (InlineHistory[InlineHistoryID].first ==
F)
157 InlineHistoryID = InlineHistory[InlineHistoryID].second;
166 return *OwnedAdvisor;
178 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
184 M,
FAM,
M.getContext(), std::move(OwnedAdvisor),
186 CGSCCInlineReplayScope,
187 CGSCCInlineReplayFallback,
188 {CGSCCInlineReplayFormat}},
192 return *OwnedAdvisor;
194 assert(IAA->getAdvisor() &&
195 "Expected a present InlineAdvisorAnalysis also have an "
196 "InlineAdvisor initialized");
197 return *IAA->getAdvisor();
201 F.dropAllReferences();
203 BB.eraseFromParent();
211 const auto &MAMProxy =
213 bool Changed =
false;
215 assert(InitialC.
size() > 0 &&
"Cannot handle an empty SCC!");
216 Module &M = *InitialC.
begin()->getFunction().getParent();
254 for (
auto &
N : InitialC) {
263 if (
auto *CB = dyn_cast<CallBase>(&
I))
264 if (
Function *Callee = CB->getCalledFunction()) {
265 if (!Callee->isDeclaration())
267 else if (!isa<IntrinsicInst>(
I)) {
272 << NV(
"Callee", Callee) <<
" will not be inlined into "
273 << NV(
"Caller", CB->getCaller())
274 <<
" because its definition is unavailable"
310 for (
int I = 0;
I < (int)Calls.size(); ++
I) {
317 if (CG.lookupSCC(
N) !=
C)
321 <<
" Function size: " <<
F.getInstructionCount()
331 bool DidInline =
false;
332 for (;
I < (int)Calls.size() && Calls[
I].first->getCaller() == &
F; ++
I) {
335 const int InlineHistoryID =
P.second;
338 if (InlineHistoryID != -1 &&
340 LLVM_DEBUG(
dbgs() <<
"Skipping inlining due to history: " <<
F.getName()
341 <<
" -> " <<
Callee.getName() <<
"\n");
355 if (CalleeSCC ==
C && UR.InlinedInternalEdges.count({&N, C})) {
356 LLVM_DEBUG(
dbgs() <<
"Skipping inlining internal SCC edge from a node "
357 "previously split out of this SCC by inlining: "
358 <<
F.getName() <<
" -> " <<
Callee.getName() <<
"\n");
363 std::unique_ptr<InlineAdvice> Advice =
364 Advisor.getAdvice(*CB, OnlyMandatory);
370 if (!Advice->isInliningRecommended()) {
371 Advice->recordUnattemptedInlining();
383 GetAssumptionCache, PSI,
390 if (!
IR.isSuccess()) {
391 Advice->recordUnsuccessfulInlining(
IR);
396 InlinedCallees.
insert(&Callee);
400 <<
F.getInstructionCount() <<
"\n");
403 if (!IFI.InlinedCallSites.empty()) {
404 int NewHistoryID = InlineHistory.
size();
408 Function *NewCallee = ICB->getCalledFunction();
410 "Intrinsic calls should not be tracked.");
417 NewCallee = ICB->getCalledFunction();
421 Calls.push_back({ICB, NewHistoryID});
431 if (CalleeSCC !=
C &&
432 CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
437 ICB->addFnAttr(NewCBCostMult);
449 bool CalleeWasDeleted =
false;
450 if (
Callee.isDiscardableIfUnused() &&
Callee.hasZeroLiveUses() &&
451 !CG.isLibFunction(Callee)) {
454 std::remove_if(Calls.begin() +
I + 1, Calls.end(),
455 [&](
const std::pair<CallBase *, int> &Call) {
456 return Call.first->getCaller() == &Callee;
464 "Cannot put cause a function to become dead twice!");
466 CalleeWasDeleted =
true;
468 DeadFunctionsInComdats.
push_back(&Callee);
471 if (CalleeWasDeleted)
472 Advice->recordInliningWithCalleeDeleted();
474 Advice->recordInlining();
520 if ((
C != OldC || UR.CWorklist.count(OldC)) &&
522 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
524 LLVM_DEBUG(
dbgs() <<
"Inlined an internal call edge and split an SCC, "
525 "retaining this to avoid infinite inlining.\n");
526 UR.InlinedInternalEdges.insert({&
N, OldC});
528 InlinedCallees.
clear();
537 if (!DeadFunctionsInComdats.
empty()) {
539 for (
auto *Callee : DeadFunctionsInComdats)
541 DeadFunctions.
append(DeadFunctionsInComdats);
551 for (
Function *DeadF : DeadFunctions) {
552 CG.markDeadFunction(*DeadF);
555 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
556 FAM.
clear(*DeadF, DeadF->getName());
557 AM.clear(DeadC, DeadC.getName());
561 UR.InvalidatedSCCs.insert(&DeadC);
563 UR.DeadFunctions.push_back(DeadF);
585 : Params(Params), IC(IC), Mode(Mode),
592 if (MandatoryFirst) {
605 if (!IAA.tryCreate(Params, Mode,
606 {CGSCCInlineReplayFile,
607 CGSCCInlineReplayScope,
608 CGSCCInlineReplayFallback,
609 {CGSCCInlineReplayFormat}},
611 M.getContext().emitError(
612 "Could not setup Inlining Advisor for the requested "
613 "mode and/or options");
644 OS, MapClassName2PassName);
646 OS <<
"<only-mandatory>";
659 if (MaxDevirtIterations != 0)
660 OS <<
"devirt<" << MaxDevirtIterations <<
">(";
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This header provides classes for managing passes over SCCs of the call graph.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
void makeFunctionBodyUnreachable(Function &F)
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file provides a priority worklist.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A manager for alias analyses.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
A container for analyses that lazily runs them and caches their results.
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Printer pass for the InlineAdvisorAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
InlineResult is basically true or false.
The inliner pass for the new pass manager.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
A Module instance is used to store all the information related to an LLVM module.
Result proxy object for OuterAnalysisManagerProxy.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same_v< PassT, PassManager > > addPass(PassT &&Pass)
PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)
Run all of the passes in this manager over the given unit of IR.
bool isEmpty() const
Returns if the pass manager contains any passes.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
void clear()
Completely clear the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
This function has undefined behavior.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
@ C
The default llvm calling convention, compatible with C.
const char FunctionInlineCostMultiplierAttributeName[]
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Thresholds to tune inline cost analysis.
A CRTP mix-in to automatically provide informational APIs needed for passes.