41 #define DEBUG_TYPE "inline"
43 STATISTIC(NumInlined,
"Number of functions inlined");
44 STATISTIC(NumCallsDeleted,
"Number of call sites deleted, not inlined");
45 STATISTIC(NumDeleted,
"Number of functions deleted because all callers found");
46 STATISTIC(NumMergedAllocas,
"Number of allocas merged together");
51 STATISTIC(NumCallerCallersAnalyzed,
"Number of caller-callers analyzed");
72 "inliner-function-import-stats",
73 cl::init(InlinerFunctionImportStatsOpts::No),
76 clEnumValN(InlinerFunctionImportStatsOpts::Verbose,
"verbose",
77 "printing of statistics for each inlined function")),
137 if (InlineHistory != -1)
142 for (
unsigned AllocaNo = 0, e = IFI.
StaticAllocas.size(); AllocaNo != e;
154 std::vector<AllocaInst *> &AllocasForType = InlinedArrayAllocas[ATy];
161 bool MergedAwayAlloca =
false;
162 for (
AllocaInst *AvailableAlloca : AllocasForType) {
165 Align2 = AvailableAlloca->getAlignment();
169 if (AvailableAlloca->getParent() != AI->
getParent())
174 if (!UsedAllocas.
insert(AvailableAlloca).second)
179 DEBUG(
dbgs() <<
" ***MERGED ALLOCA: " << *AI
180 <<
"\n\t\tINTO: " << *AvailableAlloca <<
'\n');
188 DDI->moveBefore(AvailableAlloca->getNextNode());
192 if (Align1 != Align2) {
193 if (!Align1 || !Align2) {
197 Align1 = Align1 ? Align1 : TypeAlign;
198 Align2 = Align2 ? Align2 : TypeAlign;
206 MergedAwayAlloca =
true;
213 if (MergedAwayAlloca)
221 AllocasForType.push_back(AI);
249 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
267 int &TotalSecondaryCost,
289 TotalSecondaryCost = 0;
295 bool inliningPreventsSomeOuterInline =
false;
303 callerWillBeRemoved =
false;
308 ++NumCallerCallersAnalyzed;
310 callerWillBeRemoved =
false;
320 inliningPreventsSomeOuterInline =
true;
321 TotalSecondaryCost += IC2.
getCost();
328 if (callerWillBeRemoved && !Caller->
use_empty())
331 if (inliningPreventsSomeOuterInline && TotalSecondaryCost < IC.
getCost())
350 <<
NV(
"Callee", Callee)
351 <<
" should always be inlined (cost=always)");
356 DEBUG(
dbgs() <<
" NOT Inlining: cost=never"
359 <<
NV(
"Callee", Callee)
360 <<
" should never be inlined (cost=never)");
370 <<
NV(
"Callee", Callee) <<
" too costly to inline (cost="
371 <<
NV(
"Cost", IC.
getCost()) <<
", threshold="
376 int TotalSecondaryCost = 0;
380 <<
", outer Cost = " << TotalSecondaryCost <<
'\n');
382 "IncreaseCostInOtherContexts",
Call)
383 <<
"Not inlining. Cost of inlining " <<
NV(
"Callee", Callee)
384 <<
" increases the cost of inlining " <<
NV(
"Caller", Caller)
385 <<
" in other contexts");
393 <<
NV(
"Callee", Callee) <<
" can be inlined into "
394 <<
NV(
"Caller", Caller) <<
" with cost=" <<
NV(
"Cost", IC.
getCost())
405 while (InlineHistoryID != -1) {
406 assert(
unsigned(InlineHistoryID) < InlineHistory.size() &&
407 "Invalid inline history ID");
408 if (InlineHistory[InlineHistoryID].first == F)
410 InlineHistoryID = InlineHistory[InlineHistoryID].second;
416 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
436 DEBUG(
dbgs() <<
"Inliner visiting SCC:");
466 if (!CS || isa<IntrinsicInst>(
I))
473 if (Callee->isDeclaration()) {
476 <<
NV(
"Callee", Callee) <<
" will not be inlined into "
478 <<
" because its definition is unavailable"
483 CallSites.
push_back(std::make_pair(CS, -1));
487 DEBUG(
dbgs() <<
": " << CallSites.
size() <<
" call sites.\n");
490 if (CallSites.
empty())
495 unsigned FirstCallInSCC = CallSites.
size();
496 for (
unsigned i = 0;
i < FirstCallInSCC; ++
i)
497 if (
Function *
F = CallSites[
i].first.getCalledFunction())
498 if (SCCFunctions.
count(
F))
499 std::swap(CallSites[
i--], CallSites[--FirstCallInSCC]);
506 bool Changed =
false;
513 for (
unsigned CSi = 0; CSi != CallSites.
size(); ++CSi) {
527 CG[Caller]->removeCallEdgeFor(CS);
540 int InlineHistoryID = CallSites[CSi].second;
541 if (InlineHistoryID != -1 &&
559 <<
NV(
"Callee", Callee) <<
" will not be inlined into "
560 <<
NV(
"Caller", Caller));
566 InlineHistoryID, InsertLifetime, AARGetter,
567 ImportedFunctionsStats)) {
570 <<
NV(
"Callee", Callee) <<
" will not be inlined into "
571 <<
NV(
"Caller", Caller));
578 <<
NV(
"Callee", Callee) <<
" inlined into "
579 <<
NV(
"Caller", Caller));
586 int NewHistoryID = InlineHistory.
size();
587 InlineHistory.
push_back(std::make_pair(Callee, InlineHistoryID));
598 !SCCFunctions.
count(Callee) &&
603 CG[Callee]->getNumReferences() == 0) {
609 CalleeNode->removeAllCalledFunctions();
620 if (SCC.isSingular()) {
621 CallSites[CSi] = CallSites.
back();
631 }
while (LocalChange);
637 CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
638 ACT = &getAnalysis<AssumptionCacheTracker>();
639 PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
640 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
654 return inlineCallsImpl(SCC, CG, GetAssumptionCache, PSI, TLI, InsertLifetime,
662 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No)
664 InlinerFunctionImportStatsOpts::Verbose);
670 bool AlwaysInlineOnly) {
676 CGN->removeAllCalledFunctions();
689 for (
const auto &
I : CG) {
698 if (AlwaysInlineOnly && !F->
hasFnAttribute(Attribute::AlwaysInline))
721 if (!DeadFunctionsInComdats.
empty()) {
725 for (
Function *
F : DeadFunctionsInComdats)
729 if (FunctionsToRemove.
empty())
740 FunctionsToRemove.
erase(
741 std::unique(FunctionsToRemove.
begin(), FunctionsToRemove.
end()),
742 FunctionsToRemove.
end());
744 delete CG.removeFunctionFromModule(CGN);
758 bool Changed =
false;
760 assert(InitialC.size() > 0 &&
"Cannot handle an empty SCC!");
761 Module &M = *InitialC.begin()->getFunction().getParent();
764 std::function<AssumptionCache &(Function &)> GetAssumptionCache =
773 auto GetInlineCost = [&](
CallSite CS) {
776 return getInlineCost(CS, Params, CalleeTTI, GetAssumptionCache, PSI);
783 for (
auto &
N : InitialC)
786 auto *RC = &
C->getOuterRefSCC();
828 if (!Callee->isDeclaration())
831 bool DidInline =
false;
832 while (!Calls.
empty()) {
838 if (InlineHistoryID != -1 &&
849 InlinedCallees.
insert(&Callee);
853 int NewHistoryID = InlineHistory.
size();
854 InlineHistory.
push_back({&Callee, InlineHistoryID});
857 if (!NewCallee->isDeclaration())
868 if (Callee.hasLocalLinkage()) {
871 Callee.removeDeadConstantUsers();
872 if (Callee.use_empty()) {
877 Callee.dropAllReferences();
878 assert(
find(DeadFunctions, &Callee) == DeadFunctions.
end() &&
879 "Cannot put cause a function to become dead twice!");
897 for (
Function *InlinedCallee : InlinedCallees) {
900 RC->insertTrivialRefEdge(
N, *
E.getNode());
902 InlinedCallees.clear();
911 RC = &
C->getOuterRefSCC();
912 }
while (!Nodes.empty());
921 for (
Function *DeadF : DeadFunctions) {
924 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
925 auto &DeadRC = DeadC.getOuterRefSCC();
926 CG.removeDeadFunction(*DeadF);
930 UR.InvalidatedSCCs.insert(&DeadC);
931 UR.InvalidatedRefSCCs.insert(&DeadRC);
934 M.getFunctionList().erase(DeadF);
InlinerFunctionImportStatsOpts
bool isDefTriviallyDead() const
isDefTriviallyDead - Return true if it is trivially safe to remove this function definition from the ...
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
FunTy * getCaller() const
getCaller - Return the caller function for this call site
bool doFinalization(CallGraph &CG) override
Remove now-dead linkonce functions at the end of processing to avoid breaking the SCC traversal...
void setModuleInfo(const Module &M)
Set information like AllFunctions, ImportedFunctions, ModuleName.
DiagnosticInfoOptimizationBase::Argument NV
STATISTIC(NumFunctions,"Total number of functions")
static bool InlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int >> &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
BasicAAResult createLegacyPMBasicAAResult(Pass &P, Function &F)
A helper for the legacy pass manager to create a BasicAAResult object populated to the best of our ab...
A Module instance is used to store all the information related to an LLVM module. ...
void recordInline(const Function &Caller, const Function &Callee)
Record inline of.
virtual InlineCost getInlineCost(CallSite CS)=0
This method must be implemented by the subclass to determine the cost of inlining the specified call ...
Analysis providing profile information.
BBTy * getParent() const
Get the basic block containing the call site.
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
An immutable pass that tracks lazily created AssumptionCache objects.
An efficient, type-erasing, non-owning reference to a callable.
A cache of .assume calls within a function.
Analysis pass providing the TargetTransformInfo.
Calculate and dump ThinLTO specific inliner stats.
InlineFunctionInfo - This class captures the data input to the InlineFunction call, and records the auxiliary results produced by it.
static void mergeInlinedArrayAllocas(Function *Caller, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory)
Look at all of the allocas that we inlined through this call site.
A proxy from a FunctionAnalysisManager to an SCC.
A node in the call graph for a module.
void getAnalysisUsage(AnalysisUsage &Info) const override
getAnalysisUsage - For this class, we declare that we require and preserve the call graph...
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Represents the cost of inlining a function.
Module & getModule() const
Returns the module the call graph corresponds to.
StringRef getName() const
Return a constant reference to the value's name.
Function * getFunction() const
Returns the function that this call graph node represents.
bool InlineFunction(CallInst *C, InlineFunctionInfo &IFI, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true)
InlineFunction - This function inlines the called function into the basic block of the caller...
bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1...
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
AnalysisUsage & addRequired()
bool runOnSCC(CallGraphSCC &SCC) override
Main run interface method, this implements the interface required by the Pass class.
int getCost() const
Get the inline cost estimate.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
InlineCost getInlineCost(CallSite CS, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function< AssumptionCache &(Function &)> &GetAssumptionCache, ProfileSummaryInfo *PSI)
Get an InlineCost object representing the cost of inlining this callsite.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Function * removeFunctionFromModule(CallGraphNode *CGN)
Unlink the function from this module, returning it.
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
void filterDeadComdatFunctions(Module &M, SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive...
AAResults createLegacyPMAAResults(Pass &P, Function &F, BasicAAResult &BAR)
A helper for the legacy pass manager to create a AAResults object populated to the best of our abilit...
LLVM_NODISCARD bool empty() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
bool insert(const value_type &X)
Insert a new element into the SetVector.
Class to represent array types.
A lazily constructed view of the call graph of a module.
const int LastCallToStaticBonus
DiagnosticInfoOptimizationBase::setIsVerbose setIsVerbose
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
int getCostDelta() const
Get the cost delta from the threshold for inlining.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
ValuesClass values(OptsTy...Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
SmallVector< CallSite, 8 > InlinedCallSites
All of the new call sites inlined into the caller.
initializer< Ty > init(const Ty &Val)
void array_pod_sort(IteratorTy Start, IteratorTy End)
array_pod_sort - This sorts an array with the specified start and end extent.
void getAnalysisUsage(AnalysisUsage &Info) const override
For this class, we declare that we require and preserve the call graph.
void removeAnyCallEdgeTo(CallGraphNode *Callee)
Removes all call edges from this node to the specified callee function.
A set of analyses that are preserved following a run of a transformation pass.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
bool inlineCalls(CallGraphSCC &SCC)
This function performs the main work of the pass.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
unsigned getAlignment() const
Return the alignment of the memory that is being allocated by the instruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Represent the analysis usage information of a pass.
SmallVector< WeakVH, 8 > InlinedCalls
InlinedCalls - InlineFunction fills this in with callsites that were inlined from the callee...
A node in the call graph.
A class used to represent edges in the call graph.
iterator erase(const_iterator CI)
bool doInitialization(CallGraph &CG) override
doInitialization - This method is called before the SCC's of the program has been processed...
LLVMContext & getContext() const
All values hold a context through their type.
void getAAResultsAnalysisUsage(AnalysisUsage &AU)
A helper for the legacy pass manager to populate AU to add uses to make sure the analyses required by...
A function analysis which provides an AssumptionCache.
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
static bool inlineCallsImpl(CallGraphSCC &SCC, CallGraph &CG, std::function< AssumptionCache &(Function &)> GetAssumptionCache, ProfileSummaryInfo *PSI, TargetLibraryInfo &TLI, bool InsertLifetime, function_ref< InlineCost(CallSite CS)> GetInlineCost, function_ref< AAResults &(Function &)> AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats)
A SetVector that performs no allocations if smaller than a certain size.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
ImportedFunctionsInliningStatistics ImportedFunctionsStats
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
InstrTy * getInstruction() const
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
static cl::opt< bool > DisableInlinedAllocaMerging("disable-inlined-alloca-merging", cl::init(false), cl::Hidden)
Flag to disable manual alloca merging.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Provides information about what library functions are available for the current target.
LLVM_NODISCARD T pop_back_val()
AssumptionCacheTracker * ACT
bool removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly=false)
Remove dead functions.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void mergeAttributesForInlining(Function &Caller, const Function &Callee)
Merge caller's and callee's attributes.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
void dump(bool Verbose)
Dump stats computed with InlinerStatistics class.
iterator_range< user_iterator > users()
static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime, function_ref< AAResults &(Function &)> &AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats)
If it is possible to inline the specified call site, do so and update the CallGraph for this operatio...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
SmallVector< AllocaInst *, 4 > StaticAllocas
StaticAllocas - InlineFunction fills this in with all static allocas that get copied into the caller...
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
The basic data container for the call graph of a Module of IR.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
LegacyInlinerBase(char &ID)
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
bool hasLinkOnceODRLinkage() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static bool shouldInline(CallSite CS, function_ref< InlineCost(CallSite CS)> GetInlineCost, OptimizationRemarkEmitter &ORE)
Return true if the inliner should attempt to inline at the given CallSite.
bool hasLocalLinkage() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void removeDeadConstantUsers() const
If there are any dead constant users dangling off of this constant, remove them.
FunTy * getCalledFunction() const
getCalledFunction - Return the function being called if this is a direct call, otherwise return null ...
bool skipSCC(CallGraphSCC &SCC) const
Optional passes call this function to check whether the pass should be skipped.
Module * getParent()
Get the module that this global value is contained inside of...
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction has no side ef...
LLVM Value Representation.
An SCC of the call graph.
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
AssumptionCache & getAssumptionCache(Function &F)
Get the cached assumptions for a function.
print Print MemDeps of function
This is the interface for LLVM's primary stateless and local alias analysis.
inst_range instructions(Function *F)
A container for analyses that lazily runs them and caches their results.
static bool shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallSite CS)> GetInlineCost)
Return true if inlining of CS can block the caller from being inlined which is proved to be more bene...
void emplace(ArgTypes &&...Args)
Create a new object by constructing it in place with the given arguments.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, bool DebugLogging=false)
Helper to update the call graph after running a function pass.
CallGraphNode * getExternalCallingNode() const
Returns the CallGraphNode which is used to represent undetermined calls into the callgraph.
This represents the llvm.dbg.declare instruction.
const BasicBlock * getParent() const
DenseMap< ArrayType *, std::vector< AllocaInst * > > InlinedArrayAllocasTy
an instruction to allocate memory on the stack