LLVM 19.0.0git
MLInlineAdvisor.cpp
Go to the documentation of this file.
1//===- MLInlineAdvisor.cpp - machine learned InlineAdvisor ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the interface between the inliner and a learned model.
10// It delegates model evaluation to either the AOT compiled model (the
11// 'release' mode) or a runtime-loaded model (the 'development' case).
12//
13//===----------------------------------------------------------------------===//
30#include "llvm/IR/Dominators.h"
32#include "llvm/IR/Module.h"
33#include "llvm/IR/PassManager.h"
35
36using namespace llvm;
37
39 "inliner-interactive-channel-base", cl::Hidden,
41 "Base file path for the interactive mode. The incoming filename should "
42 "have the name <inliner-interactive-channel-base>.in, while the "
43 "outgoing name should be <inliner-interactive-channel-base>.out"));
44static const std::string InclDefaultMsg =
45 (Twine("In interactive mode, also send the default policy decision: ") +
47 .str();
48static cl::opt<bool>
49 InteractiveIncludeDefault("inliner-interactive-include-default", cl::Hidden,
51
53
55 "ml-inliner-skip-policy", cl::Hidden, cl::init(SkipMLPolicyCriteria::Never),
58 "if-caller-not-cold", "if the caller is not cold")));
59
60static cl::opt<std::string> ModelSelector("ml-inliner-model-selector",
61 cl::Hidden, cl::init(""));
62
63#if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL)
64// codegen-ed file
65#include "InlinerSizeModel.h" // NOLINT
66using CompiledModelType = llvm::InlinerSizeModel;
67#else
69#endif
70
71std::unique_ptr<InlineAdvisor>
73 std::function<bool(CallBase &)> GetDefaultAdvice) {
74 if (!llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() &&
76 return nullptr;
77 std::unique_ptr<MLModelRunner> AOTRunner;
79 AOTRunner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
80 M.getContext(), FeatureMap, DecisionName,
82 else {
83 auto Features = FeatureMap;
85 Features.push_back(DefaultDecisionSpec);
86 AOTRunner = std::make_unique<InteractiveModelRunner>(
87 M.getContext(), Features, InlineDecisionSpec,
90 }
91 return std::make_unique<MLInlineAdvisor>(M, MAM, std::move(AOTRunner),
92 GetDefaultAdvice);
93}
94
95#define DEBUG_TYPE "inline-ml"
96
98 "ml-advisor-size-increase-threshold", cl::Hidden,
99 cl::desc("Maximum factor by which expected native size may increase before "
100 "blocking any further inlining."),
101 cl::init(2.0));
102
104 "ml-advisor-keep-fpi-cache", cl::Hidden,
105 cl::desc(
106 "For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"),
107 cl::init(false));
108
109// clang-format off
110const std::vector<TensorSpec> llvm::FeatureMap{
111#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec<DTYPE>(#NAME, SHAPE),
112// InlineCost features - these must come first
114
115// Non-cost features
117#undef POPULATE_NAMES
118};
119// clang-format on
120
121const char *const llvm::DecisionName = "inlining_decision";
123 TensorSpec::createSpec<int64_t>(DecisionName, {1});
124const char *const llvm::DefaultDecisionName = "inlining_default";
126 TensorSpec::createSpec<int64_t>(DefaultDecisionName, {1});
127const char *const llvm::RewardName = "delta_size";
128
130 if (auto *CS = dyn_cast<CallBase>(&I))
131 if (Function *Callee = CS->getCalledFunction()) {
132 if (!Callee->isDeclaration()) {
133 return CS;
134 }
135 }
136 return nullptr;
137}
138
141 std::unique_ptr<MLModelRunner> Runner,
142 std::function<bool(CallBase &)> GetDefaultAdvice)
144 M, MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager()),
145 ModelRunner(std::move(Runner)), GetDefaultAdvice(GetDefaultAdvice),
146 CG(MAM.getResult<LazyCallGraphAnalysis>(M)),
147 InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize),
148 PSI(MAM.getResult<ProfileSummaryAnalysis>(M)) {
150 ModelRunner->switchContext("");
151 // Extract the 'call site height' feature - the position of a call site
152 // relative to the farthest statically reachable SCC node. We don't mutate
153 // this value while inlining happens. Empirically, this feature proved
154 // critical in behavioral cloning - i.e. training a model to mimic the manual
155 // heuristic's decisions - and, thus, equally important for training for
156 // improvement.
157 CallGraph CGraph(M);
158 for (auto I = scc_begin(&CGraph); !I.isAtEnd(); ++I) {
159 const std::vector<CallGraphNode *> &CGNodes = *I;
160 unsigned Level = 0;
161 for (auto *CGNode : CGNodes) {
162 Function *F = CGNode->getFunction();
163 if (!F || F->isDeclaration())
164 continue;
165 for (auto &I : instructions(F)) {
166 if (auto *CS = getInlinableCS(I)) {
167 auto *Called = CS->getCalledFunction();
168 auto Pos = FunctionLevels.find(&CG.get(*Called));
169 // In bottom up traversal, an inlinable callee is either in the
170 // same SCC, or to a function in a visited SCC. So not finding its
171 // level means we haven't visited it yet, meaning it's in this SCC.
172 if (Pos == FunctionLevels.end())
173 continue;
174 Level = std::max(Level, Pos->second + 1);
175 }
176 }
177 }
178 for (auto *CGNode : CGNodes) {
179 Function *F = CGNode->getFunction();
180 if (F && !F->isDeclaration())
181 FunctionLevels[&CG.get(*F)] = Level;
182 }
183 }
184 for (auto KVP : FunctionLevels) {
185 AllNodes.insert(KVP.first);
186 EdgeCount += getLocalCalls(KVP.first->getFunction());
187 }
188 NodeCount = AllNodes.size();
189}
190
192 return CG.lookup(F) ? FunctionLevels.at(CG.lookup(F)) : 0;
193}
194
196 if (!CurSCC || ForceStop)
197 return;
198 FPICache.clear();
199 // Function passes executed between InlinerPass runs may have changed the
200 // module-wide features.
201 // The cgscc pass manager rules are such that:
202 // - if a pass leads to merging SCCs, then the pipeline is restarted on the
203 // merged SCC
204 // - if a pass leads to splitting the SCC, then we continue with one of the
205 // splits
206 // This means that the NodesInLastSCC is a superset (not strict) of the nodes
207 // that subsequent passes would have processed
208 // - in addition, if new Nodes were created by a pass (e.g. CoroSplit),
209 // they'd be adjacent to Nodes in the last SCC. So we just need to check the
210 // boundary of Nodes in NodesInLastSCC for Nodes we haven't seen. We don't
211 // care about the nature of the Edge (call or ref). `FunctionLevels`-wise, we
212 // record them at the same level as the original node (this is a choice, may
213 // need revisiting).
214 // - nodes are only deleted at the end of a call graph walk where they are
215 // batch deleted, so we shouldn't see any dead nodes here.
216 while (!NodesInLastSCC.empty()) {
217 const auto *N = *NodesInLastSCC.begin();
218 assert(!N->isDead());
219 NodesInLastSCC.erase(N);
220 EdgeCount += getLocalCalls(N->getFunction());
221 const auto NLevel = FunctionLevels.at(N);
222 for (const auto &E : *(*N)) {
223 const auto *AdjNode = &E.getNode();
224 assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration());
225 auto I = AllNodes.insert(AdjNode);
226 // We've discovered a new function.
227 if (I.second) {
228 ++NodeCount;
229 NodesInLastSCC.insert(AdjNode);
230 FunctionLevels[AdjNode] = NLevel;
231 }
232 }
233 }
234
235 EdgeCount -= EdgesOfLastSeenNodes;
236 EdgesOfLastSeenNodes = 0;
237
238 // (Re)use NodesInLastSCC to remember the nodes in the SCC right now,
239 // in case the SCC is split before onPassExit and some nodes are split out
240 assert(NodesInLastSCC.empty());
241 for (const auto &N : *CurSCC)
242 NodesInLastSCC.insert(&N);
243}
244
246 // No need to keep this around - function passes will invalidate it.
247 if (!KeepFPICache)
248 FPICache.clear();
249 if (!CurSCC || ForceStop)
250 return;
251 // Keep track of the nodes and edges we last saw. Then, in onPassEntry,
252 // we update the node count and edge count from the subset of these nodes that
253 // survived.
254 EdgesOfLastSeenNodes = 0;
255
256 // Check on nodes that were in SCC onPassEntry
257 for (const LazyCallGraph::Node *N : NodesInLastSCC) {
258 assert(!N->isDead());
259 EdgesOfLastSeenNodes += getLocalCalls(N->getFunction());
260 }
261
262 // Check on nodes that may have got added to SCC
263 for (const auto &N : *CurSCC) {
264 assert(!N.isDead());
265 auto I = NodesInLastSCC.insert(&N);
266 if (I.second)
267 EdgesOfLastSeenNodes += getLocalCalls(N.getFunction());
268 }
269 assert(NodeCount >= NodesInLastSCC.size());
270 assert(EdgeCount >= EdgesOfLastSeenNodes);
271}
272
275}
276
277// Update the internal state of the advisor, and force invalidate feature
278// analysis. Currently, we maintain minimal (and very simple) global state - the
279// number of functions and the number of static calls. We also keep track of the
280// total IR size in this module, to stop misbehaving policies at a certain bloat
281// factor (SizeIncreaseThreshold)
283 bool CalleeWasDeleted) {
284 assert(!ForceStop);
285 Function *Caller = Advice.getCaller();
286 Function *Callee = Advice.getCallee();
287 // The caller features aren't valid anymore.
288 {
292 PA.abandon<LoopAnalysis>();
293 FAM.invalidate(*Caller, PA);
294 }
296 int64_t IRSizeAfter =
297 getIRSize(*Caller) + (CalleeWasDeleted ? 0 : Advice.CalleeIRSize);
298 CurrentIRSize += IRSizeAfter - (Advice.CallerIRSize + Advice.CalleeIRSize);
299 if (CurrentIRSize > SizeIncreaseThreshold * InitialIRSize)
300 ForceStop = true;
301
302 // We can delta-update module-wide features. We know the inlining only changed
303 // the caller, and maybe the callee (by deleting the latter).
304 // Nodes are simple to update.
305 // For edges, we 'forget' the edges that the caller and callee used to have
306 // before inlining, and add back what they currently have together.
307 int64_t NewCallerAndCalleeEdges =
309
310 // A dead function's node is not actually removed from the call graph until
311 // the end of the call graph walk, but the node no longer belongs to any valid
312 // SCC.
313 if (CalleeWasDeleted) {
314 --NodeCount;
315 NodesInLastSCC.erase(CG.lookup(*Callee));
316 DeadFunctions.insert(Callee);
317 } else {
318 NewCallerAndCalleeEdges +=
320 }
321 EdgeCount += (NewCallerAndCalleeEdges - Advice.CallerAndCalleeEdges);
322 assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);
323}
324
325int64_t MLInlineAdvisor::getModuleIRSize() const {
326 int64_t Ret = 0;
327 for (auto &F : M)
328 if (!F.isDeclaration())
329 Ret += getIRSize(F);
330 return Ret;
331}
332
334 auto InsertPair =
335 FPICache.insert(std::make_pair(&F, FunctionPropertiesInfo()));
336 if (!InsertPair.second)
337 return InsertPair.first->second;
338 InsertPair.first->second = FAM.getResult<FunctionPropertiesAnalysis>(F);
339 return InsertPair.first->second;
340}
341
342std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdviceImpl(CallBase &CB) {
343 if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
344 return Skip;
345
346 auto &Caller = *CB.getCaller();
347 auto &Callee = *CB.getCalledFunction();
348
349 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
351 };
352 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
354
355 if (SkipPolicy == SkipMLPolicyCriteria::IfCallerIsNotCold) {
356 if (!PSI.isFunctionEntryCold(&Caller))
357 return std::make_unique<InlineAdvice>(this, CB, ORE,
358 GetDefaultAdvice(CB));
359 }
360 auto MandatoryKind = InlineAdvisor::getMandatoryKind(CB, FAM, ORE);
361 // If this is a "never inline" case, there won't be any changes to internal
362 // state we need to track, so we can just return the base InlineAdvice, which
363 // will do nothing interesting.
364 // Same thing if this is a recursive case.
365 if (MandatoryKind == InlineAdvisor::MandatoryInliningKind::Never ||
366 &Caller == &Callee)
367 return getMandatoryAdvice(CB, false);
368
369 bool Mandatory =
371
372 // If we need to stop, we won't want to track anymore any state changes, so
373 // we just return the base InlineAdvice, which acts as a noop.
374 if (ForceStop) {
375 ORE.emit([&] {
376 return OptimizationRemarkMissed(DEBUG_TYPE, "ForceStop", &CB)
377 << "Won't attempt inlining because module size grew too much.";
378 });
379 return std::make_unique<InlineAdvice>(this, CB, ORE, Mandatory);
380 }
381
382 int CostEstimate = 0;
383 if (!Mandatory) {
384 auto IsCallSiteInlinable =
385 llvm::getInliningCostEstimate(CB, TIR, GetAssumptionCache);
386 if (!IsCallSiteInlinable) {
387 // We can't inline this for correctness reasons, so return the base
388 // InlineAdvice, as we don't care about tracking any state changes (which
389 // won't happen).
390 return std::make_unique<InlineAdvice>(this, CB, ORE, false);
391 }
392 CostEstimate = *IsCallSiteInlinable;
393 }
394
395 const auto CostFeatures =
396 llvm::getInliningCostFeatures(CB, TIR, GetAssumptionCache);
397 if (!CostFeatures) {
398 return std::make_unique<InlineAdvice>(this, CB, ORE, false);
399 }
400
401 if (Mandatory)
402 return getMandatoryAdvice(CB, true);
403
404 auto NrCtantParams = 0;
405 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
406 NrCtantParams += (isa<Constant>(*I));
407 }
408
409 auto &CallerBefore = getCachedFPI(Caller);
410 auto &CalleeBefore = getCachedFPI(Callee);
411
412 *ModelRunner->getTensor<int64_t>(FeatureIndex::callee_basic_block_count) =
413 CalleeBefore.BasicBlockCount;
414 *ModelRunner->getTensor<int64_t>(FeatureIndex::callsite_height) =
416 *ModelRunner->getTensor<int64_t>(FeatureIndex::node_count) = NodeCount;
417 *ModelRunner->getTensor<int64_t>(FeatureIndex::nr_ctant_params) =
418 NrCtantParams;
419 *ModelRunner->getTensor<int64_t>(FeatureIndex::edge_count) = EdgeCount;
420 *ModelRunner->getTensor<int64_t>(FeatureIndex::caller_users) =
421 CallerBefore.Uses;
422 *ModelRunner->getTensor<int64_t>(
423 FeatureIndex::caller_conditionally_executed_blocks) =
424 CallerBefore.BlocksReachedFromConditionalInstruction;
425 *ModelRunner->getTensor<int64_t>(FeatureIndex::caller_basic_block_count) =
426 CallerBefore.BasicBlockCount;
427 *ModelRunner->getTensor<int64_t>(
428 FeatureIndex::callee_conditionally_executed_blocks) =
429 CalleeBefore.BlocksReachedFromConditionalInstruction;
430 *ModelRunner->getTensor<int64_t>(FeatureIndex::callee_users) =
431 CalleeBefore.Uses;
432 *ModelRunner->getTensor<int64_t>(FeatureIndex::cost_estimate) = CostEstimate;
433 *ModelRunner->getTensor<int64_t>(FeatureIndex::is_callee_avail_external) =
434 Callee.hasAvailableExternallyLinkage();
435 *ModelRunner->getTensor<int64_t>(FeatureIndex::is_caller_avail_external) =
436 Caller.hasAvailableExternallyLinkage();
437
438 // Add the cost features
439 for (size_t I = 0;
440 I < static_cast<size_t>(InlineCostFeatureIndex::NumberOfFeatures); ++I) {
441 *ModelRunner->getTensor<int64_t>(inlineCostFeatureToMlFeature(
442 static_cast<InlineCostFeatureIndex>(I))) = CostFeatures->at(I);
443 }
444 // This one would have been set up to be right at the end.
448 return getAdviceFromModel(CB, ORE);
449}
450
451std::unique_ptr<MLInlineAdvice>
454 return std::make_unique<MLInlineAdvice>(
455 this, CB, ORE, static_cast<bool>(ModelRunner->evaluate<int64_t>()));
456}
457
458std::unique_ptr<InlineAdvice>
459MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(CallBase &CB) {
461 .isReachableFromEntry(CB.getParent()))
462 return std::make_unique<InlineAdvice>(this, CB, getCallerORE(CB), false);
463 return nullptr;
464}
465
466std::unique_ptr<InlineAdvice> MLInlineAdvisor::getMandatoryAdvice(CallBase &CB,
467 bool Advice) {
468 // Make sure we track inlinings in all cases - mandatory or not.
469 if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB))
470 return Skip;
471 if (Advice && !ForceStop)
472 return getMandatoryAdviceImpl(CB);
473
474 // If this is a "never inline" case, there won't be any changes to internal
475 // state we need to track, so we can just return the base InlineAdvice, which
476 // will do nothing interesting.
477 // Same if we are forced to stop - we don't track anymore.
478 return std::make_unique<InlineAdvice>(this, CB, getCallerORE(CB), Advice);
479}
480
481std::unique_ptr<MLInlineAdvice>
483 return std::make_unique<MLInlineAdvice>(this, CB, getCallerORE(CB), true);
484}
485
486void MLInlineAdvisor::print(raw_ostream &OS) const {
487 OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount
488 << " EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes << "\n";
489 OS << "[MLInlineAdvisor] FPI:\n";
490 for (auto I : FPICache) {
491 OS << I.first->getName() << ":\n";
492 I.second.print(OS);
493 OS << "\n";
494 }
495 OS << "\n";
496 OS << "[MLInlineAdvisor] FuncLevels:\n";
497 for (auto I : FunctionLevels)
498 OS << (DeadFunctions.contains(&I.first->getFunction())
499 ? "<deleted>"
500 : I.first->getFunction().getName())
501 << " : " << I.second << "\n";
502
503 OS << "\n";
504}
505
508 bool Recommendation)
509 : InlineAdvice(Advisor, CB, ORE, Recommendation),
510 CallerIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Caller)),
511 CalleeIRSize(Advisor->isForcedToStop() ? 0 : Advisor->getIRSize(*Callee)),
512 CallerAndCalleeEdges(Advisor->isForcedToStop()
513 ? 0
514 : (Advisor->getLocalCalls(*Caller) +
515 Advisor->getLocalCalls(*Callee))),
516 PreInlineCallerFPI(Advisor->getCachedFPI(*Caller)) {
517 if (Recommendation)
518 FPU.emplace(Advisor->getCachedFPI(*getCaller()), CB);
519}
520
521void MLInlineAdvice::reportContextForRemark(
523 using namespace ore;
524 OR << NV("Callee", Callee->getName());
525 for (size_t I = 0; I < NumberOfFeatures; ++I)
526 OR << NV(FeatureMap[I].name(),
527 *getAdvisor()->getModelRunner().getTensor<int64_t>(I));
528 OR << NV("ShouldInline", isInliningRecommended());
529}
530
532 FPU->finish(FAM);
533}
534
536 ORE.emit([&]() {
537 OptimizationRemark R(DEBUG_TYPE, "InliningSuccess", DLoc, Block);
538 reportContextForRemark(R);
539 return R;
540 });
541 getAdvisor()->onSuccessfulInlining(*this, /*CalleeWasDeleted*/ false);
542}
543
545 ORE.emit([&]() {
546 OptimizationRemark R(DEBUG_TYPE, "InliningSuccessWithCalleeDeleted", DLoc,
547 Block);
548 reportContextForRemark(R);
549 return R;
550 });
551 getAdvisor()->onSuccessfulInlining(*this, /*CalleeWasDeleted*/ true);
552}
553
555 const InlineResult &Result) {
556 getAdvisor()->getCachedFPI(*Caller) = PreInlineCallerFPI;
557 ORE.emit([&]() {
558 OptimizationRemarkMissed R(DEBUG_TYPE, "InliningAttemptedAndUnsuccessful",
559 DLoc, Block);
560 reportContextForRemark(R);
561 return R;
562 });
563}
565 assert(!FPU);
566 ORE.emit([&]() {
567 OptimizationRemarkMissed R(DEBUG_TYPE, "IniningNotAttempted", DLoc, Block);
568 reportContextForRemark(R);
569 return R;
570 });
571}
Expand Atomic instructions
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
#define DEBUG_TYPE
#define INLINE_COST_FEATURE_ITERATOR(M)
#define INLINE_FEATURE_ITERATOR(M)
Implements a lazy call graph analysis and related passes for the new pass manager.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
static cl::opt< bool > KeepFPICache("ml-advisor-keep-fpi-cache", cl::Hidden, cl::desc("For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"), cl::init(false))
static cl::opt< std::string > ModelSelector("ml-inliner-model-selector", cl::Hidden, cl::init(""))
CallBase * getInlinableCS(Instruction &I)
SkipMLPolicyCriteria
static cl::opt< std::string > InteractiveChannelBaseName("inliner-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name <inliner-interactive-channel-base>.in, while the " "outgoing name should be <inliner-interactive-channel-base>.out"))
#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __)
static cl::opt< float > SizeIncreaseThreshold("ml-advisor-size-increase-threshold", cl::Hidden, cl::desc("Maximum factor by which expected native size may increase before " "blocking any further inlining."), cl::init(2.0))
static const std::string InclDefaultMsg
static cl::opt< SkipMLPolicyCriteria > SkipPolicy("ml-inliner-skip-policy", cl::Hidden, cl::init(SkipMLPolicyCriteria::Never), cl::values(clEnumValN(SkipMLPolicyCriteria::Never, "never", "never"), clEnumValN(SkipMLPolicyCriteria::IfCallerIsNotCold, "if-caller-not-cold", "if the caller is not cold")))
static cl::opt< bool > InteractiveIncludeDefault("inliner-interactive-include-default", cl::Hidden, cl::desc(InclDefaultMsg))
#define DecisionName
Module.h This file contains the declarations for the Module class.
if(VerifyEach)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:50
raw_pwrite_stream & OS
This pass exposes codegen information to IR-level passes.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1385
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1391
Function * getCaller()
Helper to get the caller (the parent function).
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
Common features for diagnostics dealing with optimization remarks that are used by both IR and MIR pa...
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:279
int64_t DirectCallsToDefinedFunctions
Number of direct calls made from this function to other functions defined in this module.
Capture state between an inlining decision having had been made, and its impact being observable.
Definition: InlineAdvisor.h:75
Function *const Callee
Function *const Caller
Caller and Callee are pre-inlining.
const BasicBlock *const Block
OptimizationRemarkEmitter & ORE
InlineAdvisor *const Advisor
const DebugLoc DLoc
bool isInliningRecommended() const
Get the inlining recommendation.
Interface for deciding whether to inline a call site or not.
OptimizationRemarkEmitter & getCallerORE(CallBase &CB)
FunctionAnalysisManager & FAM
static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)
InlineResult is basically true or false.
Definition: InlineCost.h:180
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:563
An analysis pass which computes the call graph for a module.
A node in the call graph.
An SCC of the call graph.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
Node * lookup(const Function &F) const
Lookup a function in the graph which has already been scanned and added.
Analysis pass that exposes the LoopInfo for a function.
Definition: LoopInfo.h:571
InlineAdvice that tracks changes post inlining.
void updateCachedCallerFPI(FunctionAnalysisManager &FAM) const
const int64_t CallerIRSize
MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool Recommendation)
const int64_t CalleeIRSize
void recordInliningImpl() override
Function * getCaller() const
const int64_t CallerAndCalleeEdges
void recordUnsuccessfulInliningImpl(const InlineResult &Result) override
Function * getCallee() const
void recordInliningWithCalleeDeletedImpl() override
void recordUnattemptedInliningImpl() override
std::unique_ptr< MLModelRunner > ModelRunner
FunctionPropertiesInfo & getCachedFPI(Function &) const
void onPassExit(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is exited, as function passes may be run subsequently.
MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::unique_ptr< MLModelRunner > ModelRunner, std::function< bool(CallBase &)> GetDefaultAdvice)
void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted)
virtual std::unique_ptr< MLInlineAdvice > getMandatoryAdviceImpl(CallBase &CB)
void onPassEntry(LazyCallGraph::SCC *SCC) override
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
int64_t getLocalCalls(Function &F)
virtual std::unique_ptr< MLInlineAdvice > getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE)
int64_t getIRSize(Function &F) const
std::function< bool(CallBase &)> GetDefaultAdvice
std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB) override
std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice) override
unsigned getInitialFunctionLevel(const Function &F) const
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A mock class satisfying the interface expected by ReleaseModeModelRunner for its TGen parameter.
The optimization diagnostic interface.
void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
void abandon()
Mark an analysis as abandoned.
Definition: Analysis.h:164
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
bool isFunctionEntryCold(const Function *F) const
Returns true if F has cold function entry.
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
const ParentTy * getParent() const
Definition: ilist_node.h:32
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
constexpr FeatureIndex inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature)
const char *const DefaultDecisionName
constexpr size_t NumberOfFeatures
std::optional< InlineCostFeatures > getInliningCostFeatures(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the expanded cost features.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
Definition: SCCIterator.h:233
std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)
const TensorSpec DefaultDecisionSpec
const char *const DecisionName
const std::vector< TensorSpec > FeatureMap
const TensorSpec InlineDecisionSpec
const char *const RewardName
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
std::optional< int > getInliningCostEstimate(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get the cost estimate ignoring thresholds.
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
#define N
ReleaseModeModelRunner - production mode implementation of the MLModelRunner.
EmbeddedModelRunnerOptions & setModelSelector(StringRef Value)