LLVM 20.0.0git
Inliner.cpp
Go to the documentation of this file.
1//===- Inliner.cpp - Code common to all inliners --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the mechanics required to implement inlining without
10// missing any calls and updating the call graph. The decisions of which calls
11// are profitable to inline are implemented elsewhere.
12//
13//===----------------------------------------------------------------------===//
14
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/ScopeExit.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/StringRef.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/BasicBlock.h"
39#include "llvm/IR/DebugLoc.h"
42#include "llvm/IR/Function.h"
44#include "llvm/IR/Instruction.h"
47#include "llvm/IR/Metadata.h"
48#include "llvm/IR/Module.h"
49#include "llvm/IR/PassManager.h"
50#include "llvm/IR/User.h"
51#include "llvm/IR/Value.h"
52#include "llvm/Pass.h"
55#include "llvm/Support/Debug.h"
61#include <algorithm>
62#include <cassert>
63#include <functional>
64#include <utility>
65
66using namespace llvm;
67
68#define DEBUG_TYPE "inline"
69
70STATISTIC(NumInlined, "Number of functions inlined");
71STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
72
74 "intra-scc-cost-multiplier", cl::init(2), cl::Hidden,
76 "Cost multiplier to multiply onto inlined call sites where the "
77 "new call was previously an intra-SCC call (not relevant when the "
78 "original call was already intra-SCC). This can accumulate over "
79 "multiple inlinings (e.g. if a call site already had a cost "
80 "multiplier and one of its inlined calls was also subject to "
81 "this, the inlined call would have the original multiplier "
82 "multiplied by intra-scc-cost-multiplier). This is to prevent tons of "
83 "inlining through a child SCC which can cause terrible compile times"));
84
85/// A flag for test, so we can print the content of the advisor when running it
86/// as part of the default (e.g. -O3) pipeline.
87static cl::opt<bool> KeepAdvisorForPrinting("keep-inline-advisor-for-printing",
88 cl::init(false), cl::Hidden);
89
90/// Allows printing the contents of the advisor after each SCC inliner pass.
91static cl::opt<bool>
92 EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing",
93 cl::init(false), cl::Hidden);
94
95
97 "cgscc-inline-replay", cl::init(""), cl::value_desc("filename"),
99 "Optimization remarks file containing inline remarks to be replayed "
100 "by cgscc inlining."),
101 cl::Hidden);
102
104 "cgscc-inline-replay-scope",
105 cl::init(ReplayInlinerSettings::Scope::Function),
106 cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function",
107 "Replay on functions that have remarks associated "
108 "with them (default)"),
109 clEnumValN(ReplayInlinerSettings::Scope::Module, "Module",
110 "Replay on the entire module")),
111 cl::desc("Whether inline replay should be applied to the entire "
112 "Module or just the Functions (default) that are present as "
113 "callers in remarks during cgscc inlining."),
114 cl::Hidden);
115
117 "cgscc-inline-replay-fallback",
118 cl::init(ReplayInlinerSettings::Fallback::Original),
121 ReplayInlinerSettings::Fallback::Original, "Original",
122 "All decisions not in replay send to original advisor (default)"),
123 clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline,
124 "AlwaysInline", "All decisions not in replay are inlined"),
125 clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline",
126 "All decisions not in replay are not inlined")),
127 cl::desc(
128 "How cgscc inline replay treats sites that don't come from the replay. "
129 "Original: defers to original advisor, AlwaysInline: inline all sites "
130 "not in replay, NeverInline: inline no sites not in replay"),
131 cl::Hidden);
132
134 "cgscc-inline-replay-format",
135 cl::init(CallSiteFormat::Format::LineColumnDiscriminator),
137 clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"),
138 clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn",
139 "<Line Number>:<Column Number>"),
140 clEnumValN(CallSiteFormat::Format::LineDiscriminator,
141 "LineDiscriminator", "<Line Number>.<Discriminator>"),
142 clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator,
143 "LineColumnDiscriminator",
144 "<Line Number>:<Column Number>.<Discriminator> (default)")),
145 cl::desc("How cgscc inline replay file is formatted"), cl::Hidden);
146
147/// Return true if the specified inline history ID
148/// indicates an inline history that includes the specified function.
150 Function *F, int InlineHistoryID,
151 const SmallVectorImpl<std::pair<Function *, int>> &InlineHistory) {
152 while (InlineHistoryID != -1) {
153 assert(unsigned(InlineHistoryID) < InlineHistory.size() &&
154 "Invalid inline history ID");
155 if (InlineHistory[InlineHistoryID].first == F)
156 return true;
157 InlineHistoryID = InlineHistory[InlineHistoryID].second;
158 }
159 return false;
160}
161
163InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM,
165 if (OwnedAdvisor)
166 return *OwnedAdvisor;
167
169 if (!IAA) {
170 // It should still be possible to run the inliner as a stand-alone SCC pass,
171 // for test scenarios. In that case, we default to the
172 // DefaultInlineAdvisor, which doesn't need to keep state between SCC pass
173 // runs. It also uses just the default InlineParams.
174 // In this case, we need to use the provided FAM, which is valid for the
175 // duration of the inliner pass, and thus the lifetime of the owned advisor.
176 // The one we would get from the MAM can be invalidated as a result of the
177 // inliner's activity.
178 OwnedAdvisor = std::make_unique<DefaultInlineAdvisor>(
179 M, FAM, getInlineParams(),
181
182 if (!CGSCCInlineReplayFile.empty())
183 OwnedAdvisor = getReplayInlineAdvisor(
184 M, FAM, M.getContext(), std::move(OwnedAdvisor),
185 ReplayInlinerSettings{CGSCCInlineReplayFile,
186 CGSCCInlineReplayScope,
187 CGSCCInlineReplayFallback,
188 {CGSCCInlineReplayFormat}},
189 /*EmitRemarks=*/true,
191
192 return *OwnedAdvisor;
193 }
194 assert(IAA->getAdvisor() &&
195 "Expected a present InlineAdvisorAnalysis also have an "
196 "InlineAdvisor initialized");
197 return *IAA->getAdvisor();
198}
199
201 F.dropAllReferences();
203 BB.eraseFromParent();
204 BasicBlock *BB = BasicBlock::Create(F.getContext(), "", &F);
205 new UnreachableInst(F.getContext(), BB);
206}
207
210 CGSCCUpdateResult &UR) {
211 const auto &MAMProxy =
213 bool Changed = false;
214
215 assert(InitialC.size() > 0 && "Cannot handle an empty SCC!");
216 Module &M = *InitialC.begin()->getFunction().getParent();
217 ProfileSummaryInfo *PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(M);
218
221 .getManager();
222
223 InlineAdvisor &Advisor = getAdvisor(MAMProxy, FAM, M);
224 Advisor.onPassEntry(&InitialC);
225
226 // We use a single common worklist for calls across the entire SCC. We
227 // process these in-order and append new calls introduced during inlining to
228 // the end. The PriorityInlineOrder is optional here, in which the smaller
229 // callee would have a higher priority to inline.
230 //
231 // Note that this particular order of processing is actually critical to
232 // avoid very bad behaviors. Consider *highly connected* call graphs where
233 // each function contains a small amount of code and a couple of calls to
234 // other functions. Because the LLVM inliner is fundamentally a bottom-up
235 // inliner, it can handle gracefully the fact that these all appear to be
236 // reasonable inlining candidates as it will flatten things until they become
237 // too big to inline, and then move on and flatten another batch.
238 //
239 // However, when processing call edges *within* an SCC we cannot rely on this
240 // bottom-up behavior. As a consequence, with heavily connected *SCCs* of
241 // functions we can end up incrementally inlining N calls into each of
242 // N functions because each incremental inlining decision looks good and we
243 // don't have a topological ordering to prevent explosions.
244 //
245 // To compensate for this, we don't process transitive edges made immediate
246 // by inlining until we've done one pass of inlining across the entire SCC.
247 // Large, highly connected SCCs still lead to some amount of code bloat in
248 // this model, but it is uniformly spread across all the functions in the SCC
249 // and eventually they all become too large to inline, rather than
250 // incrementally maknig a single function grow in a super linear fashion.
252
253 // Populate the initial list of calls in this SCC.
254 for (auto &N : InitialC) {
255 auto &ORE =
257 // We want to generally process call sites top-down in order for
258 // simplifications stemming from replacing the call with the returned value
259 // after inlining to be visible to subsequent inlining decisions.
260 // FIXME: Using instructions sequence is a really bad way to do this.
261 // Instead we should do an actual RPO walk of the function body.
262 for (Instruction &I : instructions(N.getFunction()))
263 if (auto *CB = dyn_cast<CallBase>(&I))
264 if (Function *Callee = CB->getCalledFunction()) {
265 if (!Callee->isDeclaration())
266 Calls.push_back({CB, -1});
267 else if (!isa<IntrinsicInst>(I)) {
268 using namespace ore;
269 setInlineRemark(*CB, "unavailable definition");
270 ORE.emit([&]() {
271 return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I)
272 << NV("Callee", Callee) << " will not be inlined into "
273 << NV("Caller", CB->getCaller())
274 << " because its definition is unavailable"
275 << setIsVerbose();
276 });
277 }
278 }
279 }
280
281 // Capture updatable variable for the current SCC.
282 auto *C = &InitialC;
283
284 auto AdvisorOnExit = make_scope_exit([&] { Advisor.onPassExit(C); });
285
286 if (Calls.empty())
287 return PreservedAnalyses::all();
288
289 // When inlining a callee produces new call sites, we want to keep track of
290 // the fact that they were inlined from the callee. This allows us to avoid
291 // infinite inlining in some obscure cases. To represent this, we use an
292 // index into the InlineHistory vector.
294
295 // Track a set vector of inlined callees so that we can augment the caller
296 // with all of their edges in the call graph before pruning out the ones that
297 // got simplified away.
298 SmallSetVector<Function *, 4> InlinedCallees;
299
300 // Track the dead functions to delete once finished with inlining calls. We
301 // defer deleting these to make it easier to handle the call graph updates.
302 SmallVector<Function *, 4> DeadFunctions;
303
304 // Track potentially dead non-local functions with comdats to see if they can
305 // be deleted as a batch after inlining.
306 SmallVector<Function *, 4> DeadFunctionsInComdats;
307
308 // Loop forward over all of the calls. Note that we cannot cache the size as
309 // inlining can introduce new calls that need to be processed.
310 for (int I = 0; I < (int)Calls.size(); ++I) {
311 // We expect the calls to typically be batched with sequences of calls that
312 // have the same caller, so we first set up some shared infrastructure for
313 // this caller. We also do any pruning we can at this layer on the caller
314 // alone.
315 Function &F = *Calls[I].first->getCaller();
316 LazyCallGraph::Node &N = *CG.lookup(F);
317 if (CG.lookupSCC(N) != C)
318 continue;
319
320 LLVM_DEBUG(dbgs() << "Inlining calls in: " << F.getName() << "\n"
321 << " Function size: " << F.getInstructionCount()
322 << "\n");
323
324 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
326 };
327
328 // Now process as many calls as we have within this caller in the sequence.
329 // We bail out as soon as the caller has to change so we can update the
330 // call graph and prepare the context of that new caller.
331 bool DidInline = false;
332 for (; I < (int)Calls.size() && Calls[I].first->getCaller() == &F; ++I) {
333 auto &P = Calls[I];
334 CallBase *CB = P.first;
335 const int InlineHistoryID = P.second;
337
338 if (InlineHistoryID != -1 &&
339 inlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) {
340 LLVM_DEBUG(dbgs() << "Skipping inlining due to history: " << F.getName()
341 << " -> " << Callee.getName() << "\n");
342 setInlineRemark(*CB, "recursive");
343 // Set noinline so that we don't forget this decision across CGSCC
344 // iterations.
345 CB->setIsNoInline();
346 continue;
347 }
348
349 // Check if this inlining may repeat breaking an SCC apart that has
350 // already been split once before. In that case, inlining here may
351 // trigger infinite inlining, much like is prevented within the inliner
352 // itself by the InlineHistory above, but spread across CGSCC iterations
353 // and thus hidden from the full inline history.
354 LazyCallGraph::SCC *CalleeSCC = CG.lookupSCC(*CG.lookup(Callee));
355 if (CalleeSCC == C && UR.InlinedInternalEdges.count({&N, C})) {
356 LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node "
357 "previously split out of this SCC by inlining: "
358 << F.getName() << " -> " << Callee.getName() << "\n");
359 setInlineRemark(*CB, "recursive SCC split");
360 continue;
361 }
362
363 std::unique_ptr<InlineAdvice> Advice =
364 Advisor.getAdvice(*CB, OnlyMandatory);
365
366 // Check whether we want to inline this callsite.
367 if (!Advice)
368 continue;
369
370 if (!Advice->isInliningRecommended()) {
371 Advice->recordUnattemptedInlining();
372 continue;
373 }
374
375 int CBCostMult =
378 .value_or(1);
379
380 // Setup the data structure used to plumb customization into the
381 // `InlineFunction` routine.
383 GetAssumptionCache, PSI,
386
388 InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
389 &FAM.getResult<AAManager>(*CB->getCaller()));
390 if (!IR.isSuccess()) {
391 Advice->recordUnsuccessfulInlining(IR);
392 continue;
393 }
394
395 DidInline = true;
396 InlinedCallees.insert(&Callee);
397 ++NumInlined;
398
399 LLVM_DEBUG(dbgs() << " Size after inlining: "
400 << F.getInstructionCount() << "\n");
401
402 // Add any new callsites to defined functions to the worklist.
403 if (!IFI.InlinedCallSites.empty()) {
404 int NewHistoryID = InlineHistory.size();
405 InlineHistory.push_back({&Callee, InlineHistoryID});
406
407 for (CallBase *ICB : reverse(IFI.InlinedCallSites)) {
408 Function *NewCallee = ICB->getCalledFunction();
409 assert(!(NewCallee && NewCallee->isIntrinsic()) &&
410 "Intrinsic calls should not be tracked.");
411 if (!NewCallee) {
412 // Try to promote an indirect (virtual) call without waiting for
413 // the post-inline cleanup and the next DevirtSCCRepeatedPass
414 // iteration because the next iteration may not happen and we may
415 // miss inlining it.
416 if (tryPromoteCall(*ICB))
417 NewCallee = ICB->getCalledFunction();
418 }
419 if (NewCallee) {
420 if (!NewCallee->isDeclaration()) {
421 Calls.push_back({ICB, NewHistoryID});
422 // Continually inlining through an SCC can result in huge compile
423 // times and bloated code since we arbitrarily stop at some point
424 // when the inliner decides it's not profitable to inline anymore.
425 // We attempt to mitigate this by making these calls exponentially
426 // more expensive.
427 // This doesn't apply to calls in the same SCC since if we do
428 // inline through the SCC the function will end up being
429 // self-recursive which the inliner bails out on, and inlining
430 // within an SCC is necessary for performance.
431 if (CalleeSCC != C &&
432 CalleeSCC == CG.lookupSCC(CG.get(*NewCallee))) {
433 Attribute NewCBCostMult = Attribute::get(
434 M.getContext(),
436 itostr(CBCostMult * IntraSCCCostMultiplier));
437 ICB->addFnAttr(NewCBCostMult);
438 }
439 }
440 }
441 }
442 }
443
444 // For local functions or discardable functions without comdats, check
445 // whether this makes the callee trivially dead. In that case, we can drop
446 // the body of the function eagerly which may reduce the number of callers
447 // of other functions to one, changing inline cost thresholds. Non-local
448 // discardable functions with comdats are checked later on.
449 bool CalleeWasDeleted = false;
450 if (Callee.isDiscardableIfUnused() && Callee.hasZeroLiveUses() &&
451 !CG.isLibFunction(Callee)) {
452 if (Callee.hasLocalLinkage() || !Callee.hasComdat()) {
453 Calls.erase(
454 std::remove_if(Calls.begin() + I + 1, Calls.end(),
455 [&](const std::pair<CallBase *, int> &Call) {
456 return Call.first->getCaller() == &Callee;
457 }),
458 Calls.end());
459
460 // Clear the body and queue the function itself for call graph
461 // updating when we finish inlining.
463 assert(!is_contained(DeadFunctions, &Callee) &&
464 "Cannot put cause a function to become dead twice!");
465 DeadFunctions.push_back(&Callee);
466 CalleeWasDeleted = true;
467 } else {
468 DeadFunctionsInComdats.push_back(&Callee);
469 }
470 }
471 if (CalleeWasDeleted)
472 Advice->recordInliningWithCalleeDeleted();
473 else
474 Advice->recordInlining();
475 }
476
477 // Back the call index up by one to put us in a good position to go around
478 // the outer loop.
479 --I;
480
481 if (!DidInline)
482 continue;
483 Changed = true;
484
485 // At this point, since we have made changes we have at least removed
486 // a call instruction. However, in the process we do some incremental
487 // simplification of the surrounding code. This simplification can
488 // essentially do all of the same things as a function pass and we can
489 // re-use the exact same logic for updating the call graph to reflect the
490 // change.
491
492 // Inside the update, we also update the FunctionAnalysisManager in the
493 // proxy for this particular SCC. We do this as the SCC may have changed and
494 // as we're going to mutate this particular function we want to make sure
495 // the proxy is in place to forward any invalidation events.
496 LazyCallGraph::SCC *OldC = C;
498 LLVM_DEBUG(dbgs() << "Updated inlining SCC: " << *C << "\n");
499
500 // If this causes an SCC to split apart into multiple smaller SCCs, there
501 // is a subtle risk we need to prepare for. Other transformations may
502 // expose an "infinite inlining" opportunity later, and because of the SCC
503 // mutation, we will revisit this function and potentially re-inline. If we
504 // do, and that re-inlining also has the potentially to mutate the SCC
505 // structure, the infinite inlining problem can manifest through infinite
506 // SCC splits and merges. To avoid this, we capture the originating caller
507 // node and the SCC containing the call edge. This is a slight over
508 // approximation of the possible inlining decisions that must be avoided,
509 // but is relatively efficient to store. We use C != OldC to know when
510 // a new SCC is generated and the original SCC may be generated via merge
511 // in later iterations.
512 //
513 // It is also possible that even if no new SCC is generated
514 // (i.e., C == OldC), the original SCC could be split and then merged
515 // into the same one as itself. and the original SCC will be added into
516 // UR.CWorklist again, we want to catch such cases too.
517 //
518 // FIXME: This seems like a very heavyweight way of retaining the inline
519 // history, we should look for a more efficient way of tracking it.
520 if ((C != OldC || UR.CWorklist.count(OldC)) &&
521 llvm::any_of(InlinedCallees, [&](Function *Callee) {
522 return CG.lookupSCC(*CG.lookup(*Callee)) == OldC;
523 })) {
524 LLVM_DEBUG(dbgs() << "Inlined an internal call edge and split an SCC, "
525 "retaining this to avoid infinite inlining.\n");
526 UR.InlinedInternalEdges.insert({&N, OldC});
527 }
528 InlinedCallees.clear();
529
530 // Invalidate analyses for this function now so that we don't have to
531 // invalidate analyses for all functions in this SCC later.
533 }
534
535 // We must ensure that we only delete functions with comdats if every function
536 // in the comdat is going to be deleted.
537 if (!DeadFunctionsInComdats.empty()) {
538 filterDeadComdatFunctions(DeadFunctionsInComdats);
539 for (auto *Callee : DeadFunctionsInComdats)
541 DeadFunctions.append(DeadFunctionsInComdats);
542 }
543
544 // Now that we've finished inlining all of the calls across this SCC, delete
545 // all of the trivially dead functions, updating the call graph and the CGSCC
546 // pass manager in the process.
547 //
548 // Note that this walks a pointer set which has non-deterministic order but
549 // that is OK as all we do is delete things and add pointers to unordered
550 // sets.
551 for (Function *DeadF : DeadFunctions) {
552 CG.markDeadFunction(*DeadF);
553 // Get the necessary information out of the call graph and nuke the
554 // function there. Also, clear out any cached analyses.
555 auto &DeadC = *CG.lookupSCC(*CG.lookup(*DeadF));
556 FAM.clear(*DeadF, DeadF->getName());
557 AM.clear(DeadC, DeadC.getName());
558
559 // Mark the relevant parts of the call graph as invalid so we don't visit
560 // them.
561 UR.InvalidatedSCCs.insert(&DeadC);
562
563 UR.DeadFunctions.push_back(DeadF);
564
565 ++NumDeleted;
566 }
567
568 if (!Changed)
569 return PreservedAnalyses::all();
570
572 // Even if we change the IR, we update the core CGSCC data structures and so
573 // can preserve the proxy to the function analysis manager.
575 // We have already invalidated all analyses on modified functions.
577 return PA;
578}
579
581 bool MandatoryFirst,
582 InlineContext IC,
584 unsigned MaxDevirtIterations)
585 : Params(Params), IC(IC), Mode(Mode),
587 // Run the inliner first. The theory is that we are walking bottom-up and so
588 // the callees have already been fully optimized, and we want to inline them
589 // into the callers so that our optimizations can reflect that.
590 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
591 // because it makes profile annotation in the backend inaccurate.
592 if (MandatoryFirst) {
593 PM.addPass(InlinerPass(/*OnlyMandatory*/ true));
596 }
597 PM.addPass(InlinerPass());
600}
601
604 auto &IAA = MAM.getResult<InlineAdvisorAnalysis>(M);
605 if (!IAA.tryCreate(Params, Mode,
606 {CGSCCInlineReplayFile,
607 CGSCCInlineReplayScope,
608 CGSCCInlineReplayFallback,
609 {CGSCCInlineReplayFormat}},
610 IC)) {
611 M.getContext().emitError(
612 "Could not setup Inlining Advisor for the requested "
613 "mode and/or options");
614 return PreservedAnalyses::all();
615 }
616
617 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
618 // to detect when we devirtualize indirect calls and iterate the SCC passes
619 // in that case to try and catch knock-on inlining or function attrs
620 // opportunities. Then we add it to the module pipeline by walking the SCCs
621 // in postorder (or bottom-up).
622 // If MaxDevirtIterations is 0, we just don't use the devirtualization
623 // wrapper.
624 if (MaxDevirtIterations == 0)
625 MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(PM)));
626 else
629
630 MPM.addPass(std::move(AfterCGMPM));
631 MPM.run(M, MAM);
632
633 // Discard the InlineAdvisor, a subsequent inlining session should construct
634 // its own.
635 auto PA = PreservedAnalyses::all();
637 PA.abandon<InlineAdvisorAnalysis>();
638 return PA;
639}
640
642 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
643 static_cast<PassInfoMixin<InlinerPass> *>(this)->printPipeline(
644 OS, MapClassName2PassName);
645 if (OnlyMandatory)
646 OS << "<only-mandatory>";
647}
648
650 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
651 // Print some info about passes added to the wrapper. This is however
652 // incomplete as InlineAdvisorAnalysis part isn't included (which also depends
653 // on Params and Mode).
654 if (!MPM.isEmpty()) {
655 MPM.printPipeline(OS, MapClassName2PassName);
656 OS << ',';
657 }
658 OS << "cgscc(";
659 if (MaxDevirtIterations != 0)
660 OS << "devirt<" << MaxDevirtIterations << ">(";
661 PM.printPipeline(OS, MapClassName2PassName);
662 if (MaxDevirtIterations != 0)
663 OS << ')';
664 OS << ')';
665}
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
This header provides classes for managing passes over SCCs of the call graph.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define DEBUG_TYPE
static bool inlineHistoryIncludes(Function *F, int InlineHistoryID, const SmallVectorImpl< std::pair< Function *, int > > &InlineHistory)
Return true if the specified inline history ID indicates an inline history that includes the specifie...
Definition: Inliner.cpp:149
static cl::opt< ReplayInlinerSettings::Scope > CGSCCInlineReplayScope("cgscc-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during cgscc inlining."), cl::Hidden)
static cl::opt< bool > KeepAdvisorForPrinting("keep-inline-advisor-for-printing", cl::init(false), cl::Hidden)
A flag for test, so we can print the content of the advisor when running it as part of the default (e...
static cl::opt< std::string > CGSCCInlineReplayFile("cgscc-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by cgscc inlining."), cl::Hidden)
static cl::opt< bool > EnablePostSCCAdvisorPrinting("enable-scc-inline-advisor-printing", cl::init(false), cl::Hidden)
Allows printing the contents of the advisor after each SCC inliner pass.
static cl::opt< int > IntraSCCCostMultiplier("intra-scc-cost-multiplier", cl::init(2), cl::Hidden, cl::desc("Cost multiplier to multiply onto inlined call sites where the " "new call was previously an intra-SCC call (not relevant when the " "original call was already intra-SCC). This can accumulate over " "multiple inlinings (e.g. if a call site already had a cost " "multiplier and one of its inlined calls was also subject to " "this, the inlined call would have the original multiplier " "multiplied by intra-scc-cost-multiplier). This is to prevent tons of " "inlining through a child SCC which can cause terrible compile times"))
static cl::opt< CallSiteFormat::Format > CGSCCInlineReplayFormat("cgscc-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden)
void makeFunctionBodyUnreachable(Function &F)
Definition: Inliner.cpp:200
static cl::opt< ReplayInlinerSettings::Fallback > CGSCCInlineReplayFallback("cgscc-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How cgscc inline replay treats sites that don't come from the replay. " "Original: defers to original advisor, AlwaysInline: inline all sites " "not in replay, NeverInline: inline no sites not in replay"), cl::Hidden)
Implements a lazy call graph analysis and related passes for the new pass manager.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file contains the declarations for metadata subclasses.
Module.h This file contains the declarations for the Module class.
#define P(N)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This header defines various interfaces for pass management in LLVM.
This file provides a priority worklist.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
This file contains some functions that are useful when dealing with strings.
A manager for alias analyses.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
Definition: Analysis.h:49
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
Definition: PassManager.h:424
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:94
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:212
Analysis pass which computes BlockFrequencyInfo.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
void setIsNoInline()
Definition: InstrTypes.h:1975
Function * getCaller()
Helper to get the caller (the parent function).
A proxy from a FunctionAnalysisManager to an SCC.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:254
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:290
Printer pass for the InlineAdvisorAnalysis results.
The InlineAdvisorAnalysis is a module pass because the InlineAdvisor needs to capture state right bef...
Interface for deciding whether to inline a call site or not.
virtual void onPassEntry(LazyCallGraph::SCC *SCC=nullptr)
This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
InlineResult is basically true or false.
Definition: InlineCost.h:180
The inliner pass for the new pass manager.
Definition: Inliner.h:35
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:641
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: Inliner.cpp:208
A node in the call graph.
An SCC of the call graph.
iterator begin() const
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &, ModuleAnalysisManager &)
Definition: Inliner.cpp:602
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: Inliner.cpp:649
ModuleInlinerWrapperPass(InlineParams Params=getInlineParams(), bool MandatoryFirst=true, InlineContext IC={}, InliningAdvisorMode Mode=InliningAdvisorMode::Default, unsigned MaxDevirtIterations=0)
Definition: Inliner.cpp:580
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Diagnostic information for missed-optimization remarks.
Result proxy object for OuterAnalysisManagerProxy.
Definition: PassManager.h:691
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Definition: PassManager.h:688
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition: PassManager.h:178
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same_v< PassT, PassManager > > addPass(PassT &&Pass)
Definition: PassManager.h:195
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:217
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:114
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
void preserveSet()
Mark an analysis set as preserved.
Definition: Analysis.h:146
void preserve()
Mark an analysis as preserved.
Definition: Analysis.h:131
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
void clear()
Completely clear the SetVector.
Definition: SetVector.h:273
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
This function has undefined behavior.
An efficient, type-erasing, non-owning reference to a callable.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
const char FunctionInlineCostMultiplierAttributeName[]
Definition: InlineCost.h:59
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
InliningAdvisorMode
There are 4 scenarios we can use the InlineAdvisor:
Definition: InlineAdvisor.h:42
std::optional< int > getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind)
Definition: InlineCost.cpp:186
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT &&Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
void setInlineRemark(CallBase &CB, StringRef Message)
Set the inline-remark attribute.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
std::unique_ptr< InlineAdvisor > getReplayInlineAdvisor(Module &M, FunctionAnalysisManager &FAM, LLVMContext &Context, std::unique_ptr< InlineAdvisor > OriginalAdvisor, const ReplayInlinerSettings &ReplaySettings, bool EmitRemarks, InlineContext IC)
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1886
void filterDeadComdatFunctions(SmallVectorImpl< Function * > &DeadComdatFunctions)
Filter out potentially dead comdat functions where other entries keep the entire comdat group alive.
bool tryPromoteCall(CallBase &CB)
Try to promote (devirtualize) a virtual call on an Alloca.
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
#define N
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Definition: InlineAdvisor.h:58
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:206
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:69