LLVM 19.0.0git
AlwaysInliner.cpp
Go to the documentation of this file.
1//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a custom inliner that handles only functions that
10// are marked as "always inline".
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/SetVector.h"
21#include "llvm/IR/Module.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "inline"
30
31namespace {
32
33bool AlwaysInlineImpl(
34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
35 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
36 function_ref<AAResults &(Function &)> GetAAR,
39 bool Changed = false;
40 SmallVector<Function *, 16> InlinedFunctions;
41 for (Function &F : M) {
42 // When callee coroutine function is inlined into caller coroutine function
43 // before coro-split pass,
44 // coro-early pass can not handle this quiet well.
45 // So we won't inline the coroutine function if it have not been unsplited
46 if (F.isPresplitCoroutine())
47 continue;
48
49 if (!F.isDeclaration() && isInlineViable(F).isSuccess()) {
50 Calls.clear();
51
52 for (User *U : F.users())
53 if (auto *CB = dyn_cast<CallBase>(U))
54 if (CB->getCalledFunction() == &F &&
55 CB->hasFnAttr(Attribute::AlwaysInline) &&
56 !CB->getAttributes().hasFnAttr(Attribute::NoInline))
57 Calls.insert(CB);
58
59 for (CallBase *CB : Calls) {
60 Function *Caller = CB->getCaller();
61 OptimizationRemarkEmitter ORE(Caller);
62 DebugLoc DLoc = CB->getDebugLoc();
63 BasicBlock *Block = CB->getParent();
64
65 InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
66 GetBFI ? &GetBFI(*Caller) : nullptr,
67 GetBFI ? &GetBFI(F) : nullptr);
68
69 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
70 &GetAAR(F), InsertLifetime);
71 if (!Res.isSuccess()) {
72 ORE.emit([&]() {
73 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc,
74 Block)
75 << "'" << ore::NV("Callee", &F) << "' is not inlined into '"
76 << ore::NV("Caller", Caller)
77 << "': " << ore::NV("Reason", Res.getFailureReason());
78 });
79 continue;
80 }
81
83 ORE, DLoc, Block, F, *Caller,
84 InlineCost::getAlways("always inline attribute"),
85 /*ForProfileContext=*/false, DEBUG_TYPE);
86
87 Changed = true;
88 }
89
90 if (F.hasFnAttribute(Attribute::AlwaysInline)) {
91 // Remember to try and delete this function afterward. This both avoids
92 // re-walking the rest of the module and avoids dealing with any
93 // iterator invalidation issues while deleting functions.
94 InlinedFunctions.push_back(&F);
95 }
96 }
97 }
98
99 // Remove any live functions.
100 erase_if(InlinedFunctions, [&](Function *F) {
101 F->removeDeadConstantUsers();
102 return !F->isDefTriviallyDead();
103 });
104
105 // Delete the non-comdat ones from the module and also from our vector.
106 auto NonComdatBegin = partition(
107 InlinedFunctions, [&](Function *F) { return F->hasComdat(); });
108 for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end())) {
109 M.getFunctionList().erase(F);
110 Changed = true;
111 }
112 InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end());
113
114 if (!InlinedFunctions.empty()) {
115 // Now we just have the comdat functions. Filter out the ones whose comdats
116 // are not actually dead.
117 filterDeadComdatFunctions(InlinedFunctions);
118 // The remaining functions are actually dead.
119 for (Function *F : InlinedFunctions) {
120 M.getFunctionList().erase(F);
121 Changed = true;
122 }
123 }
124
125 return Changed;
126}
127
128struct AlwaysInlinerLegacyPass : public ModulePass {
129 bool InsertLifetime;
130
131 AlwaysInlinerLegacyPass()
132 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
133
134 AlwaysInlinerLegacyPass(bool InsertLifetime)
135 : ModulePass(ID), InsertLifetime(InsertLifetime) {
137 }
138
139 /// Main run interface method. We override here to avoid calling skipSCC().
140 bool runOnModule(Module &M) override {
141
142 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
143 auto GetAAR = [&](Function &F) -> AAResults & {
144 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
145 };
146 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
147 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
148 };
149
150 return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR,
151 /*GetBFI*/ nullptr);
152 }
153
154 static char ID; // Pass identification, replacement for typeid
155
156 void getAnalysisUsage(AnalysisUsage &AU) const override {
160 }
161};
162
163} // namespace
164
165char AlwaysInlinerLegacyPass::ID = 0;
166INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
167 "Inliner for always_inline functions", false, false)
171INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
172 "Inliner for always_inline functions", false, false)
173
175 return new AlwaysInlinerLegacyPass(InsertLifetime);
176}
177
182 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
184 };
185 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
187 };
188 auto GetAAR = [&](Function &F) -> AAResults & {
189 return FAM.getResult<AAManager>(F);
190 };
191 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
192
193 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache,
194 GetAAR, GetBFI);
195
196 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
197}
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
always Inliner for always_inline functions
always inline
#define DEBUG_TYPE
Provides passes to inlining "always_inline" functions.
#define F(x, y, z)
Definition: MD5.cpp:55
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file implements a set that has insertion order iteration characteristics.
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:500
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1259
A debug info location.
Definition: DebugLoc.h:33
static InlineCost getAlways(const char *Reason, std::optional< CostBenefitPair > CostBenefit=std::nullopt)
Definition: InlineCost.h:125
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:202
InlineResult is basically true or false.
Definition: InlineCost.h:179
bool isSuccess() const
Definition: InlineCost.h:188
const char * getFailureReason() const
Definition: InlineCost.h:189
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:658
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
virtual bool runOnModule(Module &M)=0
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
virtual void getAnalysisUsage(AnalysisUsage &) const
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Definition: Pass.cpp:98
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
void clear()
Completely clear the SetVector.
Definition: SetVector.h:273
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
bool empty() const
Definition: SmallVector.h:94
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
An efficient, type-erasing, non-owning reference to a callable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
InlineResult isInlineViable(Function &Callee)
Minimal filter to detect invalid constructs for inlining.
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
Pass * createAlwaysInlinerLegacyPass(bool InsertLifetime=true)
Create a legacy pass manager instance of a pass to inline and remove functions marked as "always_inli...
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
auto partition(R &&Range, UnaryPredicate P)
Provide wrappers to std::partition which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1944
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2031
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
void initializeAlwaysInlinerLegacyPassPass(PassRegistry &)