68#define DEBUG_TYPE "inline"
70STATISTIC(NumInlined,
"Number of functions inlined");
71STATISTIC(NumDeleted,
"Number of functions deleted because all callers found");
79InlinerPass::getAdvisor(
const ModuleAnalysisManagerCGSCCProxy::Result &
MAM,
94 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
99 if (!Opts.cgscc_inline_replay.empty())
101 M,
FAM, M.getContext(), std::move(OwnedAdvisor),
103 Opts.cgscc_inline_replay_scope,
104 Opts.cgscc_inline_replay_fallback,
105 {Opts.cgscc_inline_replay_format}},
109 return *OwnedAdvisor;
111 assert(IAA->getAdvisor() &&
112 "Expected a present InlineAdvisorAnalysis also have an "
113 "InlineAdvisor initialized");
114 return *IAA->getAdvisor();
120 const auto &MAMProxy =
124 assert(InitialC.
size() > 0 &&
"Cannot handle an empty SCC!");
125 Module &M = *InitialC.
begin()->getFunction().getParent();
164 for (
auto &
N : InitialC) {
174 if (
Function *Callee = CB->getCalledFunction()) {
175 if (!Callee->isDeclaration())
182 << NV(
"Callee", Callee) <<
" will not be inlined into "
183 << NV(
"Caller", CB->getCaller())
184 <<
" because its definition is unavailable"
214 for (
int I = 0;
I < (int)Calls.
size(); ++
I) {
225 <<
" Function size: " <<
F.getInstructionCount()
235 bool DidInline =
false;
236 for (;
I < (int)Calls.
size() && Calls[
I]->getCaller() == &
F; ++
I) {
248 LLVM_DEBUG(
dbgs() <<
"Skipping inlining internal SCC edge from a node "
249 "previously split out of this SCC by inlining: "
250 <<
F.getName() <<
" -> " << Callee.getName() <<
"\n");
260 if (!LI || !LI->isSimple())
264 LI, LI->getParent(), BBI, Opts.inliner_forwarding_scan_limit);
272 if (
C->getType() != LI->getType()) {
273 if (
C->isNullValue())
278 LI->replaceAllUsesWith(
C);
279 LI->eraseFromParent();
283 std::unique_ptr<InlineAdvice> Advice =
290 if (!Advice->isInliningRecommended()) {
291 Advice->recordUnattemptedInlining();
303 GetAssumptionCache, PSI,
314 bool TrackInlineHistory = CalleeSCC->
size() != 1 ||
316 CalleeN->
lookup(CalleeN) !=
nullptr;
321 TrackInlineHistory,
nullptr,
323 if (!
IR.isSuccess()) {
324 Advice->recordUnsuccessfulInlining(
IR);
332 InlinedCallees.
insert(&Callee);
336 <<
F.getInstructionCount() <<
"\n");
341 Function *NewCallee = ICB->getCalledFunction();
343 "Intrinsic calls should not be tracked.");
350 NewCallee = ICB->getCalledFunction();
364 if (CalleeSCC !=
C &&
369 itostr(CBCostMult * Opts.intra_scc_cost_multiplier));
370 ICB->addFnAttr(NewCBCostMult);
382 bool CalleeWasDeleted =
false;
383 if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
385 if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
388 return CB->getCaller() == &Callee;
394 Advice->recordInliningWithCalleeDeleted();
399 "Cannot put cause a function to become dead twice!");
401 CalleeWasDeleted =
true;
403 DeadFunctionsInComdats.
push_back(&Callee);
406 if (!CalleeWasDeleted)
407 Advice->recordInlining();
453 if ((
C != OldC || UR.
CWorklist.count(OldC)) &&
455 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
457 LLVM_DEBUG(
dbgs() <<
"Inlined an internal call edge and split an SCC, "
458 "retaining this to avoid infinite inlining.\n");
461 InlinedCallees.
clear();
470 if (!DeadFunctionsInComdats.
empty()) {
472 for (
auto *Callee : DeadFunctionsInComdats)
474 DeadFunctions.
append(DeadFunctionsInComdats);
484 for (
Function *DeadF : DeadFunctions) {
489 FAM.clear(*DeadF, DeadF->getName());
490 AM.
clear(DeadC, DeadC.getName());
517 unsigned MaxDevirtIterations)
518 : Params(Params), IC(IC), Mode(Mode),
519 MaxDevirtIterations(MaxDevirtIterations) {
525 if (MandatoryFirst) {
539 if (!IAA.tryCreate(Params, Mode,
540 {Opts.cgscc_inline_replay,
541 Opts.cgscc_inline_replay_scope,
542 Opts.cgscc_inline_replay_fallback,
543 {Opts.cgscc_inline_replay_format}},
545 M.getContext().emitError(
546 "Could not setup Inlining Advisor for the requested "
547 "mode and/or options");
558 if (MaxDevirtIterations == 0)
564 MPM.addPass(std::move(AfterCGMPM));
570 if (!Opts.keep_inline_advisor_for_printing)
577 static_cast<PassInfoMixin<InlinerPass> *
>(
this)->
printPipeline(
578 OS, MapClassName2PassName);
580 OS <<
"<only-mandatory>";
588 if (!MPM.isEmpty()) {
589 MPM.printPipeline(OS, MapClassName2PassName);
593 if (MaxDevirtIterations != 0)
594 OS <<
"devirt<" << MaxDevirtIterations <<
">(";
595 PM.printPipeline(OS, MapClassName2PassName);
596 if (MaxDevirtIterations != 0)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
This header provides classes for managing passes over SCCs of the call graph.
@ Available
We know the block is fully available. This is a fixpoint.
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This file provides a priority worklist.
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A manager for alias analyses.
This templated class represents "all analyses that operate over <aparticular IR unit>" (e....
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
InstListType::iterator iterator
Instruction iterators...
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Printer pass for the InlineAdvisorAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
virtual void onPassExit(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
SmallVector< CallBase *, 8 > InlinedCallSites
All of the new call sites inlined into the caller.
InlineResult is basically true or false.
The inliner pass for the new pass manager.
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
A node in the call graph.
An SCC of the call graph.
RefSCC & getOuterRefSCC() const
A lazily constructed view of the call graph of a module.
bool isLibFunction(Function &F) const
Test whether a function is a known and defined library function tracked by the call graph.
LLVM_ABI void markDeadFunction(Function &F)
Mark a function as dead to be removed later by removeDeadFunctions().
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
Node * lookup(const Function &F) const
Lookup a function in the graph which has already been scanned and added.
LLVM_ABI PreservedAnalyses run(Module &, ModuleAnalysisManager &)
LLVM_ABI ModuleInlinerWrapperPass(InlineParams Params, bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
void clear()
Completely clear the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A SetVector that performs no allocations if smaller than a certain size.
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
LLVM Value Representation.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
const char FunctionInlineCostMultiplierAttributeName[]
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI void makeFunctionBodyUnreachable(Function &F)
Replace the body of the dead function F with a single unreachable block.
LLVM_ABI InlineParams getInlineParams(const LLVMContext &C)
Generate the parameters to tune the inline cost analysis based only on the commandline options.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
LLVM_ABI std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LLVM_ABI LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
LLVM_ABI void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
LLVM_ABI Value * FindAvailableLoadedValue(LoadInst *Load, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan=DefMaxInstsToScan, BatchAAResults *AA=nullptr, bool *IsLoadCSE=nullptr, unsigned *NumScanedInst=nullptr)
Scan backwards to see if we have the value of the given load available locally within a small number ...
LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, bool TrackInlineHistory=false, Function *ForwardVarArgsTo=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
This function inlines the called function into the basic block of the caller.
AnalysisManager< LazyCallGraph::SCC, LazyCallGraph & > CGSCCAnalysisManager
The CGSCC analysis manager.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
const IPOOptions & getIPOOptions(const LLVMContext &C)
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
OuterAnalysisManagerProxy< ModuleAnalysisManager, LazyCallGraph::SCC, LazyCallGraph & > ModuleAnalysisManagerCGSCCProxy
A proxy from a ModuleAnalysisManager to an SCC.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
LLVM_ABI void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
LLVM_ABI bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
std::string itostr(int64_t X)
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
SmallDenseSet< std::pair< LazyCallGraph::Node *, LazyCallGraph::SCC * >, 4 > & InlinedInternalEdges
A hacky area where the inliner can retain history about inlining decisions that mutated the call grap...
SmallVector< Function *, 4 > & DeadFunctions
Functions that a pass has considered to be dead to be removed at the end of the call graph walk in ba...
SmallPtrSetImpl< LazyCallGraph::SCC * > & InvalidatedSCCs
The set of invalidated SCCs which should be skipped if they are found in CWorklist.
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Thresholds to tune inline cost analysis.