31#define DEBUG_TYPE "inline"
32#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
33#define LLVM_HAVE_TF_AOT
39STATISTIC(NumCallerCallersAnalyzed,
"Number of caller-callers analyzed");
45 cl::desc(
"Enable adding inline-remark attribute to"
46 " callsites processed by inliner but decided"
47 " to be not inlined"));
51 cl::desc(
"Enable deferred inlining"));
57 cl::desc(
"Scale to limit the cost of inline deferral"),
62 cl::desc(
"If true, annotate inline advisor remarks "
63 "with LTO and pass information."));
75 bool IsInliningMandatory)
82 if (IsInliningRecommended)
85 Remark <<
": always inline attribute";
90 if (IsInliningRecommended)
93 "NotInlined", DLoc,
Block)
94 <<
"'" <<
NV(
"Callee", Callee) <<
"' is not AlwaysInline into '"
95 <<
NV(
"Caller", Caller)
96 <<
"': " <<
NV(
"Reason", Result.getFailureReason());
101 assert(!IsInliningRecommended &&
"Expected to attempt inlining");
106void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
114 <<
"'" <<
NV(
"Callee",
Callee) <<
"' is not inlined into '"
116 <<
"': " <<
NV(
"Reason",
Result.getFailureReason());
120void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
127void DefaultInlineAdvice::recordInliningImpl() {
139 .getCachedResult<ProfileSummaryAnalysis>(
153 auto GetInlineCost = [&](
CallBase &CB) {
156 bool RemarksEnabled =
157 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
159 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
160 GetBFI, PSI, RemarksEnabled ? &ORE :
nullptr);
163 CB, GetInlineCost, ORE,
167std::unique_ptr<InlineAdvice>
168DefaultInlineAdvisor::getAdviceImpl(
CallBase &CB) {
170 return std::make_unique<DefaultInlineAdvice>(
177 bool IsInliningRecommended)
180 IsInliningRecommended(IsInliningRecommended) {}
182void InlineAdvice::recordInlineStatsIfNeeded() {
189 recordInlineStatsIfNeeded();
195 recordInlineStatsIfNeeded();
209 Advisor.reset(DA.Factory(M,
FAM, Params, IC));
212 auto GetDefaultAdvice = [&
FAM, Params](
CallBase &CB) {
214 return OIC.has_value();
224 std::move(Advisor), ReplaySettings,
229#ifdef LLVM_HAVE_TFLITE
252 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
275 TotalSecondaryCost = 0;
277 int CandidateCost = IC.
getCost() - 1;
280 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
282 bool InliningPreventsSomeOuterInline =
false;
283 unsigned NumCallerUsers = 0;
284 for (
User *U : Caller->users()) {
285 CallBase *CS2 = dyn_cast<CallBase>(U);
291 ApplyLastCallBonus =
false;
296 ++NumCallerCallersAnalyzed;
298 ApplyLastCallBonus =
false;
308 InliningPreventsSomeOuterInline =
true;
309 TotalSecondaryCost += IC2.
getCost();
314 if (!InliningPreventsSomeOuterInline)
321 if (ApplyLastCallBonus)
327 return TotalSecondaryCost < IC.
getCost();
329 int TotalCost = TotalSecondaryCost + IC.
getCost() * NumCallerUsers;
331 return TotalCost < Allowance;
339template <
class RemarkT>
343 R <<
"(cost=always)";
351 R <<
": " <<
ore::NV(
"Reason", Reason);
375std::optional<InlineCost>
388 <<
", Call: " << CB <<
"\n");
394 <<
", Call: " << CB <<
"\n");
398 <<
"'" <<
NV(
"Callee", Callee) <<
"' not inlined into '"
399 <<
NV(
"Caller", Caller)
400 <<
"' because it should never be inlined " << IC;
405 <<
"'" <<
NV(
"Callee", Callee) <<
"' not inlined into '"
406 <<
NV(
"Caller", Caller) <<
"' because too costly to inline "
414 int TotalSecondaryCost = 0;
415 if (EnableDeferral &&
419 <<
", outer Cost = " << TotalSecondaryCost <<
'\n');
423 <<
"Not inlining. Cost of inlining '" <<
NV(
"Callee", Callee)
424 <<
"' increases the cost of inlining '" <<
NV(
"Caller", Caller)
425 <<
"' in other contexts";
441 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
443 CallSiteLoc <<
" @ ";
448 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
449 uint32_t Discriminator = DIL->getBaseDiscriminator();
450 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
452 Name = DIL->getScope()->getSubprogram()->getName();
453 CallSiteLoc <<
Name.str() <<
":" << llvm::utostr(
Offset);
454 if (
Format.outputColumn())
455 CallSiteLoc <<
":" << llvm::utostr(DIL->getColumn());
456 if (
Format.outputDiscriminator() && Discriminator)
457 CallSiteLoc <<
"." << llvm::utostr(Discriminator);
461 return CallSiteLoc.
str();
470 Remark <<
" at callsite ";
471 for (
DILocation *DIL = DLoc.
get(); DIL; DIL = DIL->getInlinedAt()) {
474 unsigned int Offset = DIL->getLine();
475 Offset -= DIL->getScope()->getSubprogram()->getLine();
476 unsigned int Discriminator = DIL->getBaseDiscriminator();
477 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
479 Name = DIL->getScope()->getSubprogram()->getName();
481 <<
ore::NV(
"Column", DIL->getColumn());
496 StringRef RemarkName = AlwaysInline ?
"AlwaysInline" :
"Inlined";
499 Remark <<
"'" <<
ore::NV(
"Callee", &Callee) <<
"' inlined into '"
500 <<
ore::NV(
"Caller", &Caller) <<
"'";
511 bool ForProfileContext,
const char *
PassName) {
515 if (ForProfileContext)
516 Remark <<
" to match profiling context";
517 Remark <<
" with " << IC;
523 std::optional<InlineContext> IC)
530 std::make_unique<ImportedFunctionsInliningStatistics>();
545 return std::make_unique<MandatoryInlineAdvice>(
this, CB,
getCallerORE(CB),
566 return "always-inline";
568 return "cgscc-inline";
570 return "early-inline";
574 return "module-inline";
576 return "replay-cgscc-inline";
578 return "replay-sample-profile-inline";
580 return "sample-profile-inline";
602 auto TrivialDecision =
605 if (TrivialDecision) {
606 if (TrivialDecision->isSuccess())
615 bool MandatoryOnly) {
632 OS <<
"No Inline Advisor\n";
634 IA->getAdvisor()->print(OS);
641 const auto &MAMProxy =
644 if (InitialC.
size() == 0) {
645 OS <<
"SCC is empty!\n";
648 Module &M = *InitialC.
begin()->getFunction().getParent();
651 OS <<
"No Inline Advisor\n";
653 IA->getAdvisor()->print(OS);
static const Function * getParent(const Value *V)
static cl::opt< int > InlineDeferralScale("inline-deferral-scale", cl::desc("Scale to limit the cost of inline deferral"), cl::init(2), cl::Hidden)
static cl::opt< bool > InlineRemarkAttribute("inline-remark-attribute", cl::init(false), cl::Hidden, cl::desc("Enable adding inline-remark attribute to" " callsites processed by inliner but decided" " to be not inlined"))
Flag to add inline messages as callsite attributes 'inline-remark'.
static cl::opt< bool > EnableInlineDeferral("inline-deferral", cl::init(false), cl::Hidden, cl::desc("Enable deferred inlining"))
static cl::opt< bool > AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), cl::desc("If true, annotate inline advisor remarks " "with LTO and pass information."))
static const char * getLTOPhase(ThinOrFullLTOPhase LTOPhase)
static std::optional< llvm::InlineCost > getDefaultInlineAdvice(CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params)
static const char * getInlineAdvisorContext(InlinePass IP)
static bool shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, function_ref< InlineCost(CallBase &CB)> GetInlineCost)
Return true if inlining of CB can block the caller from being inlined which is proved to be more bene...
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static const Function * getCalledFunction(const Value *V, bool &IsNoBuiltin)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static const char PassName[]
A container for analyses that lazily runs them and caches their results.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
DILocation * get() const
Get the underlying DILocation.
The default (manual heuristics) implementation of the InlineAdvisor.
Module * getParent()
Get the module that this global value is contained inside of...
Capture state between an inlining decision having had been made, and its impact being observable.
Function *const Caller
Caller and Callee are pre-inlining.
virtual void recordInliningWithCalleeDeletedImpl()
virtual void recordInliningImpl()
const BasicBlock *const Block
void recordInlining()
Exactly one of the record* APIs must be called.
OptimizationRemarkEmitter & ORE
virtual void recordUnsuccessfulInliningImpl(const InlineResult &Result)
InlineAdvisor *const Advisor
InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)
virtual void recordUnattemptedInliningImpl()
void recordInliningWithCalleeDeleted()
Call after inlining succeeded, and results in the callee being delete-able, meaning,...
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
virtual std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice)
std::unique_ptr< ImportedFunctionsInliningStatistics > ImportedFunctionsStats
const char * getAnnotatedInlinePassName() const
NOTE pass name is annotated only when inline advisor constructor provides InlineContext.
InlineAdvisor(InlineAdvisor &&)=delete
virtual std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB)=0
std::unique_ptr< InlineAdvice > getAdvice(CallBase &CB, bool MandatoryOnly=false)
Get an InlineAdvice containing a recommendation on whether to inline or not.
Represents the cost of inlining a function.
int getThreshold() const
Get the threshold against which the cost was computed.
const char * getReason() const
Get the reason of Always or Never.
int getCost() const
Get the inline cost estimate.
int getCostDelta() const
Get the cost delta from the threshold for inlining.
InlineResult is basically true or false.
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const BasicBlock * getParent() const
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
A Module instance is used to store all the information related to an LLVM module.
LLVMContext & getContext() const
Get the global data context.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Used for dynamically registering InlineAdvisors as plugins.
static bool HasBeenRegistered
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Analysis providing profile information.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
LLVMContext & getContext() const
All values hold a context through their type.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
std::string & str()
Returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const int LastCallToStaticBonus
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
@ ReplaySampleProfileInliner
void emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, bool IsMandatory, function_ref< void(OptimizationRemark &)> ExtraContext={}, const char *PassName=nullptr)
Emit ORE message.
std::string inlineCostStr(const InlineCost &IC)
Utility for extracting the inline cost message to a string.
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
std::string AnnotateInlinePassName(InlineContext IC)
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
@ FullLTOPreLink
Full LTO prelink phase.
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
@ None
No LTO/ThinLTO behavior needed.
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void emitInlinedIntoBasedOnCost(OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, const Function &Callee, const Function &Caller, const InlineCost &IC, bool ForProfileContext=false, const char *PassName=nullptr)
Emit ORE message based in cost (default heuristic).
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
std::optional< InlineResult > getAttributeBasedInliningDecision(CallBase &Call, Function *Callee, TargetTransformInfo &CalleeTTI, function_ref< const TargetLibraryInfo &(Function &)> GetTLI)
Returns InlineResult::success() if the call site should be always inlined because of user directives,...
cl::opt< InlinerFunctionImportStatsOpts > InlinerFunctionImportStats("inliner-function-import-stats", cl::init(InlinerFunctionImportStatsOpts::No), cl::values(clEnumValN(InlinerFunctionImportStatsOpts::Basic, "basic", "basic statistics"), clEnumValN(InlinerFunctionImportStatsOpts::Verbose, "verbose", "printing of statistics for each inlined function")), cl::Hidden, cl::desc("Enable inliner stats for imported functions"))
std::optional< InlineCost > shouldInline(CallBase &CB, function_ref< InlineCost(CallBase &CB)> GetInlineCost, OptimizationRemarkEmitter &ORE, bool EnableDeferral=true)
Return the cost only if the inliner should attempt to inline at the given CallSite.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
std::unique_ptr< InlineAdvisor > getDevelopmentModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
void addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc)
Add location info to ORE message.
std::string formatCallSiteLocation(DebugLoc DLoc, const CallSiteFormat &Format)
Get call site location as a string with the given format.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Used in the streaming interface as the general argument type.
bool tryCreate(InlineParams Params, InliningAdvisorMode Mode, const ReplayInlinerSettings &ReplaySettings, InlineContext IC)
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
ThinOrFullLTOPhase LTOPhase
Thresholds to tune inline cost analysis.
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.