LCOV - code coverage report
Current view: top level - lib/Passes - PassBuilder.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 527 552 95.5 %
Date: 2017-09-14 15:23:50 Functions: 44 44 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- Parsing, selection, and construction of pass pipelines -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : /// \file
      10             : ///
      11             : /// This file provides the implementation of the PassBuilder based on our
      12             : /// static pass registry as well as related functionality. It also provides
      13             : /// helpers to aid in analyzing, debugging, and testing passes and pass
      14             : /// pipelines.
      15             : ///
      16             : //===----------------------------------------------------------------------===//
      17             : 
      18             : #include "llvm/Passes/PassBuilder.h"
      19             : #include "llvm/ADT/StringSwitch.h"
      20             : #include "llvm/Analysis/AliasAnalysis.h"
      21             : #include "llvm/Analysis/AliasAnalysisEvaluator.h"
      22             : #include "llvm/Analysis/AssumptionCache.h"
      23             : #include "llvm/Analysis/BasicAliasAnalysis.h"
      24             : #include "llvm/Analysis/BlockFrequencyInfo.h"
      25             : #include "llvm/Analysis/BlockFrequencyInfoImpl.h"
      26             : #include "llvm/Analysis/BranchProbabilityInfo.h"
      27             : #include "llvm/Analysis/CFGPrinter.h"
      28             : #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
      29             : #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
      30             : #include "llvm/Analysis/CGSCCPassManager.h"
      31             : #include "llvm/Analysis/CallGraph.h"
      32             : #include "llvm/Analysis/DemandedBits.h"
      33             : #include "llvm/Analysis/DependenceAnalysis.h"
      34             : #include "llvm/Analysis/DominanceFrontier.h"
      35             : #include "llvm/Analysis/GlobalsModRef.h"
      36             : #include "llvm/Analysis/IVUsers.h"
      37             : #include "llvm/Analysis/LazyCallGraph.h"
      38             : #include "llvm/Analysis/LazyValueInfo.h"
      39             : #include "llvm/Analysis/LoopAccessAnalysis.h"
      40             : #include "llvm/Analysis/LoopInfo.h"
      41             : #include "llvm/Analysis/MemoryDependenceAnalysis.h"
      42             : #include "llvm/Analysis/MemorySSA.h"
      43             : #include "llvm/Analysis/ModuleSummaryAnalysis.h"
      44             : #include "llvm/Analysis/OptimizationDiagnosticInfo.h"
      45             : #include "llvm/Analysis/PostDominators.h"
      46             : #include "llvm/Analysis/ProfileSummaryInfo.h"
      47             : #include "llvm/Analysis/RegionInfo.h"
      48             : #include "llvm/Analysis/ScalarEvolution.h"
      49             : #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
      50             : #include "llvm/Analysis/ScopedNoAliasAA.h"
      51             : #include "llvm/Analysis/TargetLibraryInfo.h"
      52             : #include "llvm/Analysis/TargetTransformInfo.h"
      53             : #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
      54             : #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
      55             : #include "llvm/CodeGen/UnreachableBlockElim.h"
      56             : #include "llvm/IR/Dominators.h"
      57             : #include "llvm/IR/IRPrintingPasses.h"
      58             : #include "llvm/IR/PassManager.h"
      59             : #include "llvm/IR/Verifier.h"
      60             : #include "llvm/Support/Debug.h"
      61             : #include "llvm/Support/Regex.h"
      62             : #include "llvm/Target/TargetMachine.h"
      63             : #include "llvm/Transforms/GCOVProfiler.h"
      64             : #include "llvm/Transforms/IPO/AlwaysInliner.h"
      65             : #include "llvm/Transforms/IPO/ArgumentPromotion.h"
      66             : #include "llvm/Transforms/IPO/ConstantMerge.h"
      67             : #include "llvm/Transforms/IPO/CrossDSOCFI.h"
      68             : #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
      69             : #include "llvm/Transforms/IPO/ElimAvailExtern.h"
      70             : #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
      71             : #include "llvm/Transforms/IPO/FunctionAttrs.h"
      72             : #include "llvm/Transforms/IPO/FunctionImport.h"
      73             : #include "llvm/Transforms/IPO/GlobalDCE.h"
      74             : #include "llvm/Transforms/IPO/GlobalOpt.h"
      75             : #include "llvm/Transforms/IPO/GlobalSplit.h"
      76             : #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
      77             : #include "llvm/Transforms/IPO/Inliner.h"
      78             : #include "llvm/Transforms/IPO/Internalize.h"
      79             : #include "llvm/Transforms/IPO/LowerTypeTests.h"
      80             : #include "llvm/Transforms/IPO/PartialInlining.h"
      81             : #include "llvm/Transforms/IPO/SCCP.h"
      82             : #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
      83             : #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
      84             : #include "llvm/Transforms/InstCombine/InstCombine.h"
      85             : #include "llvm/Transforms/InstrProfiling.h"
      86             : #include "llvm/Transforms/PGOInstrumentation.h"
      87             : #include "llvm/Transforms/SampleProfile.h"
      88             : #include "llvm/Transforms/Scalar/ADCE.h"
      89             : #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
      90             : #include "llvm/Transforms/Scalar/BDCE.h"
      91             : #include "llvm/Transforms/Scalar/ConstantHoisting.h"
      92             : #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
      93             : #include "llvm/Transforms/Scalar/DCE.h"
      94             : #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
      95             : #include "llvm/Transforms/Scalar/DivRemPairs.h"
      96             : #include "llvm/Transforms/Scalar/EarlyCSE.h"
      97             : #include "llvm/Transforms/Scalar/Float2Int.h"
      98             : #include "llvm/Transforms/Scalar/GVN.h"
      99             : #include "llvm/Transforms/Scalar/GuardWidening.h"
     100             : #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
     101             : #include "llvm/Transforms/Scalar/IndVarSimplify.h"
     102             : #include "llvm/Transforms/Scalar/JumpThreading.h"
     103             : #include "llvm/Transforms/Scalar/LICM.h"
     104             : #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
     105             : #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
     106             : #include "llvm/Transforms/Scalar/LoopDeletion.h"
     107             : #include "llvm/Transforms/Scalar/LoopDistribute.h"
     108             : #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
     109             : #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
     110             : #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
     111             : #include "llvm/Transforms/Scalar/LoopPassManager.h"
     112             : #include "llvm/Transforms/Scalar/LoopPredication.h"
     113             : #include "llvm/Transforms/Scalar/LoopRotation.h"
     114             : #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
     115             : #include "llvm/Transforms/Scalar/LoopSink.h"
     116             : #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
     117             : #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
     118             : #include "llvm/Transforms/Scalar/LowerAtomic.h"
     119             : #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
     120             : #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
     121             : #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
     122             : #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
     123             : #include "llvm/Transforms/Scalar/NaryReassociate.h"
     124             : #include "llvm/Transforms/Scalar/NewGVN.h"
     125             : #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
     126             : #include "llvm/Transforms/Scalar/Reassociate.h"
     127             : #include "llvm/Transforms/Scalar/SCCP.h"
     128             : #include "llvm/Transforms/Scalar/SROA.h"
     129             : #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
     130             : #include "llvm/Transforms/Scalar/SimplifyCFG.h"
     131             : #include "llvm/Transforms/Scalar/Sink.h"
     132             : #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
     133             : #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
     134             : #include "llvm/Transforms/Utils/AddDiscriminators.h"
     135             : #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
     136             : #include "llvm/Transforms/Utils/LCSSA.h"
     137             : #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
     138             : #include "llvm/Transforms/Utils/LoopSimplify.h"
     139             : #include "llvm/Transforms/Utils/LowerInvoke.h"
     140             : #include "llvm/Transforms/Utils/Mem2Reg.h"
     141             : #include "llvm/Transforms/Utils/NameAnonGlobals.h"
     142             : #include "llvm/Transforms/Utils/PredicateInfo.h"
     143             : #include "llvm/Transforms/Utils/SimplifyInstructions.h"
     144             : #include "llvm/Transforms/Utils/SymbolRewriter.h"
     145             : #include "llvm/Transforms/Vectorize/LoopVectorize.h"
     146             : #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
     147             : 
     148             : #include <type_traits>
     149             : 
     150             : using namespace llvm;
     151             : 
     152       72306 : static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
     153      144612 :                                              cl::ReallyHidden, cl::init(4));
     154             : static cl::opt<bool>
     155      289224 :     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
     156             :                        cl::Hidden, cl::ZeroOrMore,
     157      289224 :                        cl::desc("Run Partial inlinining pass"));
     158             : 
     159             : static cl::opt<bool>
     160      289224 :     RunNewGVN("enable-npm-newgvn", cl::init(false),
     161             :               cl::Hidden, cl::ZeroOrMore,
     162      289224 :               cl::desc("Run NewGVN instead of GVN"));
     163             : 
     164       72306 : static cl::opt<bool> EnableEarlyCSEMemSSA(
     165      216918 :     "enable-npm-earlycse-memssa", cl::init(true), cl::Hidden,
     166      289224 :     cl::desc("Enable the EarlyCSE w/ MemorySSA pass for the new PM (default = on)"));
     167             : 
     168       72306 : static cl::opt<bool> EnableGVNHoist(
     169      216918 :     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
     170      289224 :     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
     171             : 
     172       72306 : static cl::opt<bool> EnableGVNSink(
     173      216918 :     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
     174      289224 :     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
     175             : 
     176       72306 : static Regex DefaultAliasRegex(
     177       72306 :     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
     178             : 
     179             : static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
     180          55 :   switch (Level) {
     181             :   case PassBuilder::O0:
     182             :   case PassBuilder::O1:
     183             :   case PassBuilder::O2:
     184             :   case PassBuilder::O3:
     185             :     return false;
     186             : 
     187             :   case PassBuilder::Os:
     188             :   case PassBuilder::Oz:
     189             :     return true;
     190             :   }
     191           0 :   llvm_unreachable("Invalid optimization level!");
     192             : }
     193             : 
     194             : namespace {
     195             : 
     196             : /// \brief No-op module pass which does nothing.
     197             : struct NoOpModulePass {
     198             :   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
     199             :     return PreservedAnalyses::all();
     200             :   }
     201          14 :   static StringRef name() { return "NoOpModulePass"; }
     202             : };
     203             : 
     204             : /// \brief No-op module analysis.
     205             : class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
     206             :   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
     207             :   static AnalysisKey Key;
     208             : 
     209             : public:
     210             :   struct Result {};
     211             :   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
     212          12 :   static StringRef name() { return "NoOpModuleAnalysis"; }
     213             : };
     214             : 
     215             : /// \brief No-op CGSCC pass which does nothing.
     216             : struct NoOpCGSCCPass {
     217             :   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
     218             :                         LazyCallGraph &, CGSCCUpdateResult &UR) {
     219             :     return PreservedAnalyses::all();
     220             :   }
     221          10 :   static StringRef name() { return "NoOpCGSCCPass"; }
     222             : };
     223             : 
     224             : /// \brief No-op CGSCC analysis.
     225             : class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
     226             :   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
     227             :   static AnalysisKey Key;
     228             : 
     229             : public:
     230             :   struct Result {};
     231             :   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
     232             :     return Result();
     233             :   }
     234          10 :   static StringRef name() { return "NoOpCGSCCAnalysis"; }
     235             : };
     236             : 
     237             : /// \brief No-op function pass which does nothing.
     238             : struct NoOpFunctionPass {
     239             :   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
     240             :     return PreservedAnalyses::all();
     241             :   }
     242          24 :   static StringRef name() { return "NoOpFunctionPass"; }
     243             : };
     244             : 
     245             : /// \brief No-op function analysis.
     246             : class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
     247             :   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
     248             :   static AnalysisKey Key;
     249             : 
     250             : public:
     251             :   struct Result {};
     252             :   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
     253          13 :   static StringRef name() { return "NoOpFunctionAnalysis"; }
     254             : };
     255             : 
     256             : /// \brief No-op loop pass which does nothing.
     257             : struct NoOpLoopPass {
     258             :   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
     259             :                         LoopStandardAnalysisResults &, LPMUpdater &) {
     260             :     return PreservedAnalyses::all();
     261             :   }
     262          39 :   static StringRef name() { return "NoOpLoopPass"; }
     263             : };
     264             : 
     265             : /// \brief No-op loop analysis.
     266             : class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
     267             :   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
     268             :   static AnalysisKey Key;
     269             : 
     270             : public:
     271             :   struct Result {};
     272             :   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
     273             :     return Result();
     274             :   }
     275           0 :   static StringRef name() { return "NoOpLoopAnalysis"; }
     276             : };
     277             : 
     278             : AnalysisKey NoOpModuleAnalysis::Key;
     279             : AnalysisKey NoOpCGSCCAnalysis::Key;
     280             : AnalysisKey NoOpFunctionAnalysis::Key;
     281             : AnalysisKey NoOpLoopAnalysis::Key;
     282             : 
     283             : } // End anonymous namespace.
     284             : 
     285         235 : void PassBuilder::invokePeepholeEPCallbacks(
     286             :     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
     287         712 :   for (auto &C : PeepholeEPCallbacks)
     288           7 :     C(FPM, Level);
     289         235 : }
     290             : 
     291         776 : void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
     292             : #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
     293             :   MAM.registerPass([&] { return CREATE_PASS; });
     294             : #include "PassRegistry.def"
     295             : 
     296        2331 :   for (auto &C : ModuleAnalysisRegistrationCallbacks)
     297           3 :     C(MAM);
     298         776 : }
     299             : 
     300         772 : void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
     301             : #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
     302             :   CGAM.registerPass([&] { return CREATE_PASS; });
     303             : #include "PassRegistry.def"
     304             : 
     305        2318 :   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
     306           2 :     C(CGAM);
     307         772 : }
     308             : 
     309         776 : void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
     310             : #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
     311             :   FAM.registerPass([&] { return CREATE_PASS; });
     312             : #include "PassRegistry.def"
     313             : 
     314        3050 :   for (auto &C : FunctionAnalysisRegistrationCallbacks)
     315         722 :     C(FAM);
     316         776 : }
     317             : 
     318         772 : void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
     319             : #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
     320             :   LAM.registerPass([&] { return CREATE_PASS; });
     321             : #include "PassRegistry.def"
     322             : 
     323        2318 :   for (auto &C : LoopAnalysisRegistrationCallbacks)
     324           2 :     C(LAM);
     325         772 : }
     326             : 
     327             : FunctionPassManager
     328          53 : PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
     329             :                                                  ThinLTOPhase Phase,
     330             :                                                  bool DebugLogging) {
     331             :   assert(Level != O0 && "Must request optimizations!");
     332         106 :   FunctionPassManager FPM(DebugLogging);
     333             : 
     334             :   // Form SSA out of local memory accesses after breaking apart aggregates into
     335             :   // scalars.
     336          53 :   FPM.addPass(SROA());
     337             : 
     338             :   // Catch trivial redundancies
     339         106 :   FPM.addPass(EarlyCSEPass(EnableEarlyCSEMemSSA));
     340             : 
     341             :   // Hoisting of scalars and load expressions.
     342          53 :   if (EnableGVNHoist)
     343           0 :     FPM.addPass(GVNHoistPass());
     344             : 
     345             :   // Global value numbering based sinking.
     346          53 :   if (EnableGVNSink) {
     347           0 :     FPM.addPass(GVNSinkPass());
     348           0 :     FPM.addPass(SimplifyCFGPass());
     349             :   }
     350             : 
     351             :   // Speculative execution if the target has divergent branches; otherwise nop.
     352          53 :   FPM.addPass(SpeculativeExecutionPass());
     353             : 
     354             :   // Optimize based on known information about branches, and cleanup afterward.
     355          53 :   FPM.addPass(JumpThreadingPass());
     356          53 :   FPM.addPass(CorrelatedValuePropagationPass());
     357          53 :   FPM.addPass(SimplifyCFGPass());
     358         106 :   FPM.addPass(InstCombinePass());
     359             : 
     360          42 :   if (!isOptimizingForSize(Level))
     361          42 :     FPM.addPass(LibCallsShrinkWrapPass());
     362             : 
     363          53 :   invokePeepholeEPCallbacks(FPM, Level);
     364             : 
     365          53 :   FPM.addPass(TailCallElimPass());
     366          53 :   FPM.addPass(SimplifyCFGPass());
     367             : 
     368             :   // Form canonically associated expression trees, and simplify the trees using
     369             :   // basic mathematical properties. For example, this will form (nearly)
     370             :   // minimal multiplication trees.
     371          53 :   FPM.addPass(ReassociatePass());
     372             : 
     373             :   // Add the primary loop simplification pipeline.
     374             :   // FIXME: Currently this is split into two loop pass pipelines because we run
     375             :   // some function passes in between them. These can and should be replaced by
     376             :   // loop pass equivalenst but those aren't ready yet. Specifically,
     377             :   // `SimplifyCFGPass` and `InstCombinePass` are used. We have
     378             :   // `LoopSimplifyCFGPass` which isn't yet powerful enough, and the closest to
     379             :   // the other we have is `LoopInstSimplify`.
     380         265 :   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
     381             : 
     382             :   // Rotate Loop - disable header duplication at -Oz
     383          53 :   LPM1.addPass(LoopRotatePass(Level != Oz));
     384          53 :   LPM1.addPass(LICMPass());
     385          53 :   LPM1.addPass(SimpleLoopUnswitchPass());
     386          53 :   LPM2.addPass(IndVarSimplifyPass());
     387          53 :   LPM2.addPass(LoopIdiomRecognizePass());
     388             : 
     389         160 :   for (auto &C : LateLoopOptimizationsEPCallbacks)
     390           1 :     C(LPM2, Level);
     391             : 
     392          53 :   LPM2.addPass(LoopDeletionPass());
     393             :   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
     394             :   // because it changes IR to makes profile annotation in back compile
     395             :   // inaccurate.
     396          13 :   if (Phase != ThinLTOPhase::PreLink ||
     397          62 :       !PGOOpt || PGOOpt->SampleProfileFile.empty())
     398         102 :     LPM2.addPass(LoopFullUnrollPass(Level));
     399             : 
     400         160 :   for (auto &C : LoopOptimizerEndEPCallbacks)
     401           1 :     C(LPM2, Level);
     402             : 
     403             :   // We provide the opt remark emitter pass for LICM to use. We only need to do
     404             :   // this once as it is immutable.
     405          53 :   FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
     406         212 :   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1)));
     407          53 :   FPM.addPass(SimplifyCFGPass());
     408         106 :   FPM.addPass(InstCombinePass());
     409         212 :   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2)));
     410             : 
     411             :   // Eliminate redundancies.
     412          53 :   if (Level != O1) {
     413             :     // These passes add substantial compile time so skip them at O1.
     414          45 :     FPM.addPass(MergedLoadStoreMotionPass());
     415          45 :     if (RunNewGVN)
     416           0 :       FPM.addPass(NewGVNPass());
     417             :     else
     418          45 :       FPM.addPass(GVN());
     419             :   }
     420             : 
     421             :   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
     422         106 :   FPM.addPass(MemCpyOptPass());
     423             : 
     424             :   // Sparse conditional constant propagation.
     425             :   // FIXME: It isn't clear why we do this *after* loop passes rather than
     426             :   // before...
     427          53 :   FPM.addPass(SCCPPass());
     428             : 
     429             :   // Delete dead bit computations (instcombine runs after to fold away the dead
     430             :   // computations, and then ADCE will run later to exploit any new DCE
     431             :   // opportunities that creates).
     432          53 :   FPM.addPass(BDCEPass());
     433             : 
     434             :   // Run instcombine after redundancy and dead bit elimination to exploit
     435             :   // opportunities opened up by them.
     436         106 :   FPM.addPass(InstCombinePass());
     437          53 :   invokePeepholeEPCallbacks(FPM, Level);
     438             : 
     439             :   // Re-consider control flow based optimizations after redundancy elimination,
     440             :   // redo DCE, etc.
     441          53 :   FPM.addPass(JumpThreadingPass());
     442          53 :   FPM.addPass(CorrelatedValuePropagationPass());
     443          53 :   FPM.addPass(DSEPass());
     444         159 :   FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass()));
     445             : 
     446         160 :   for (auto &C : ScalarOptimizerLateEPCallbacks)
     447           1 :     C(FPM, Level);
     448             : 
     449             :   // Finally, do an expensive DCE pass to catch all the dead code exposed by
     450             :   // the simplifications and basic cleanup after all the simplifications.
     451          53 :   FPM.addPass(ADCEPass());
     452          53 :   FPM.addPass(SimplifyCFGPass());
     453         106 :   FPM.addPass(InstCombinePass());
     454          53 :   invokePeepholeEPCallbacks(FPM, Level);
     455             : 
     456          53 :   return FPM;
     457             : }
     458             : 
     459           2 : void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
     460             :                                     PassBuilder::OptimizationLevel Level,
     461             :                                     bool RunProfileGen,
     462             :                                     std::string ProfileGenFile,
     463             :                                     std::string ProfileUseFile) {
     464             :   // Generally running simplification passes and the inliner with an high
     465             :   // threshold results in smaller executables, but there may be cases where
     466             :   // the size grows, so let's be conservative here and skip this simplification
     467             :   // at -Os/Oz.
     468           2 :   if (!isOptimizingForSize(Level)) {
     469           4 :     InlineParams IP;
     470             : 
     471             :     // In the old pass manager, this is a cl::opt. Should still this be one?
     472           2 :     IP.DefaultThreshold = 75;
     473             : 
     474             :     // FIXME: The hint threshold has the same value used by the regular inliner.
     475             :     // This should probably be lowered after performance testing.
     476             :     // FIXME: this comment is cargo culted from the old pass manager, revisit).
     477           4 :     IP.HintThreshold = 325;
     478             : 
     479           6 :     CGSCCPassManager CGPipeline(DebugLogging);
     480             : 
     481           8 :     CGPipeline.addPass(InlinerPass(IP));
     482             : 
     483           4 :     FunctionPassManager FPM;
     484           2 :     FPM.addPass(SROA());
     485           2 :     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
     486           2 :     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
     487           4 :     FPM.addPass(InstCombinePass()); // Combine silly sequences.
     488           2 :     invokePeepholeEPCallbacks(FPM, Level);
     489             : 
     490           8 :     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
     491             : 
     492           8 :     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
     493             :   }
     494             : 
     495             :   // Delete anything that is now dead to make sure that we don't instrument
     496             :   // dead code. Instrumentation can end up keeping dead code around and
     497             :   // dramatically increase code size.
     498           4 :   MPM.addPass(GlobalDCEPass());
     499             : 
     500           2 :   if (RunProfileGen) {
     501           1 :     MPM.addPass(PGOInstrumentationGen());
     502             : 
     503           2 :     FunctionPassManager FPM;
     504           3 :     FPM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass()));
     505           4 :     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
     506             : 
     507             :     // Add the profile lowering pass.
     508           2 :     InstrProfOptions Options;
     509           1 :     if (!ProfileGenFile.empty())
     510             :       Options.InstrProfileOutput = ProfileGenFile;
     511           1 :     Options.DoCounterPromotion = true;
     512           1 :     MPM.addPass(InstrProfiling(Options));
     513             :   }
     514             : 
     515           2 :   if (!ProfileUseFile.empty())
     516           3 :     MPM.addPass(PGOInstrumentationUse(ProfileUseFile));
     517           2 : }
     518             : 
     519             : static InlineParams
     520             : getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
     521          60 :   auto O3 = PassBuilder::O3;
     522          60 :   unsigned OptLevel = Level > O3 ? 2 : Level;
     523          60 :   unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
     524          60 :   return getInlineParams(OptLevel, SizeLevel);
     525             : }
     526             : 
     527             : ModulePassManager
     528          53 : PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
     529             :                                                ThinLTOPhase Phase,
     530             :                                                bool DebugLogging) {
     531         106 :   ModulePassManager MPM(DebugLogging);
     532             : 
     533             :   // Do basic inference of function attributes from known properties of system
     534             :   // libraries and other oracles.
     535          53 :   MPM.addPass(InferFunctionAttrsPass());
     536             : 
     537             :   // Create an early function pass manager to cleanup the output of the
     538             :   // frontend.
     539         159 :   FunctionPassManager EarlyFPM(DebugLogging);
     540          53 :   EarlyFPM.addPass(SimplifyCFGPass());
     541          53 :   EarlyFPM.addPass(SROA());
     542          53 :   EarlyFPM.addPass(EarlyCSEPass());
     543          53 :   EarlyFPM.addPass(LowerExpectIntrinsicPass());
     544             :   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
     545             :   // to convert bitcast to direct calls so that they can be inlined during the
     546             :   // profile annotation prepration step.
     547             :   // More details about SamplePGO design can be found in:
     548             :   // https://research.google.com/pubs/pub45290.html
     549             :   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
     550          75 :   if (PGOOpt && !PGOOpt->SampleProfileFile.empty() &&
     551             :       Phase == ThinLTOPhase::PostLink)
     552           4 :     EarlyFPM.addPass(InstCombinePass());
     553         212 :   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
     554             : 
     555          75 :   if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) {
     556             :     // Annotate sample profile right after early FPM to ensure freshness of
     557             :     // the debug info.
     558          36 :     MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile));
     559             :     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
     560             :     // for the profile annotation to be accurate in the ThinLTO backend.
     561           6 :     if (Phase != ThinLTOPhase::PreLink)
     562             :       // We perform early indirect call promotion here, before globalopt.
     563             :       // This is important for the ThinLTO backend phase because otherwise
     564             :       // imported available_externally functions look unreferenced and are
     565             :       // removed.
     566           8 :       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
     567             :                                            true));
     568             :   }
     569             : 
     570             :   // Interprocedural constant propagation now that basic cleanup has occured
     571             :   // and prior to optimizing globals.
     572             :   // FIXME: This position in the pipeline hasn't been carefully considered in
     573             :   // years, it should be re-analyzed.
     574          53 :   MPM.addPass(IPSCCPPass());
     575             : 
     576             :   // Optimize globals to try and fold them into constants.
     577          53 :   MPM.addPass(GlobalOptPass());
     578             : 
     579             :   // Promote any localized globals to SSA registers.
     580             :   // FIXME: Should this instead by a run of SROA?
     581             :   // FIXME: We should probably run instcombine and simplify-cfg afterward to
     582             :   // delete control flows that are dead once globals have been folded to
     583             :   // constants.
     584         106 :   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
     585             : 
     586             :   // Remove any dead arguments exposed by cleanups and constand folding
     587             :   // globals.
     588          53 :   MPM.addPass(DeadArgumentEliminationPass());
     589             : 
     590             :   // Create a small function pass pipeline to cleanup after all the global
     591             :   // optimizations.
     592         159 :   FunctionPassManager GlobalCleanupPM(DebugLogging);
     593         106 :   GlobalCleanupPM.addPass(InstCombinePass());
     594          53 :   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
     595             : 
     596          53 :   GlobalCleanupPM.addPass(SimplifyCFGPass());
     597         212 :   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
     598             : 
     599             :   // Add all the requested passes for instrumentation PGO, if requested.
     600          61 :   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
     601          45 :       (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) {
     602          14 :     addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen,
     603           8 :                       PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile);
     604           2 :     MPM.addPass(PGOIndirectCallPromotion(false, false));
     605             :   }
     606             : 
     607             :   // Require the GlobalsAA analysis for the module so we can query it within
     608             :   // the CGSCC pipeline.
     609          53 :   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
     610             : 
     611             :   // Require the ProfileSummaryAnalysis for the module so we can query it within
     612             :   // the inliner pass.
     613          53 :   MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
     614             : 
     615             :   // Now begin the main postorder CGSCC pipeline.
     616             :   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
     617             :   // manager and trying to emulate its precise behavior. Much of this doesn't
     618             :   // make a lot of sense and we should revisit the core CGSCC structure.
     619         159 :   CGSCCPassManager MainCGPipeline(DebugLogging);
     620             : 
     621             :   // Note: historically, the PruneEH pass was run first to deduce nounwind and
     622             :   // generally clean up exception handling overhead. It isn't clear this is
     623             :   // valuable as the inliner doesn't currently care whether it is inlining an
     624             :   // invoke or a call.
     625             : 
     626             :   // Run the inliner first. The theory is that we are walking bottom-up and so
     627             :   // the callees have already been fully optimized, and we want to inline them
     628             :   // into the callers so that our optimizations can reflect that.
     629             :   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
     630             :   // because it makes profile annotation in the backend inaccurate.
     631         106 :   InlineParams IP = getInlineParamsFromOptLevel(Level);
     632          13 :   if (Phase == ThinLTOPhase::PreLink &&
     633          62 :       PGOOpt && !PGOOpt->SampleProfileFile.empty())
     634           2 :     IP.HotCallSiteThreshold = 0;
     635         212 :   MainCGPipeline.addPass(InlinerPass(IP));
     636             : 
     637             :   // Now deduce any function attributes based in the current code.
     638          53 :   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
     639             : 
     640             :   // When at O3 add argument promotion to the pass pipeline.
     641             :   // FIXME: It isn't at all clear why this should be limited to O3.
     642          53 :   if (Level == O3)
     643          14 :     MainCGPipeline.addPass(ArgumentPromotionPass());
     644             : 
     645             :   // Lastly, add the core function simplification pipeline nested inside the
     646             :   // CGSCC walk.
     647         106 :   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
     648         106 :       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
     649             : 
     650         160 :   for (auto &C : CGSCCOptimizerLateEPCallbacks)
     651           1 :     C(MainCGPipeline, Level);
     652             : 
     653             :   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
     654             :   // to detect when we devirtualize indirect calls and iterate the SCC passes
     655             :   // in that case to try and catch knock-on inlining or function attrs
     656             :   // opportunities. Then we add it to the module pipeline by walking the SCCs
     657             :   // in postorder (or bottom-up).
     658          53 :   MPM.addPass(
     659         318 :       createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
     660         106 :           std::move(MainCGPipeline), MaxDevirtIterations)));
     661             : 
     662          53 :   return MPM;
     663             : }
     664             : 
     665             : ModulePassManager
     666          40 : PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
     667             :                                              bool DebugLogging) {
     668          80 :   ModulePassManager MPM(DebugLogging);
     669             : 
     670             :   // Optimize globals now that the module is fully simplified.
     671          40 :   MPM.addPass(GlobalOptPass());
     672             : 
     673             :   // Run partial inlining pass to partially inline functions that have
     674             :   // large bodies.
     675          40 :   if (RunPartialInlining)
     676           0 :     MPM.addPass(PartialInlinerPass());
     677             : 
     678             :   // Remove avail extern fns and globals definitions since we aren't compiling
     679             :   // an object file for later LTO. For LTO we want to preserve these so they
     680             :   // are eligible for inlining at link-time. Note if they are unreferenced they
     681             :   // will be removed by GlobalDCE later, so this only impacts referenced
     682             :   // available externally globals. Eventually they will be suppressed during
     683             :   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
     684             :   // may make globals referenced by available external functions dead and saves
     685             :   // running remaining passes on the eliminated functions.
     686          40 :   MPM.addPass(EliminateAvailableExternallyPass());
     687             : 
     688             :   // Do RPO function attribute inference across the module to forward-propagate
     689             :   // attributes where applicable.
     690             :   // FIXME: Is this really an optimization rather than a canonicalization?
     691          40 :   MPM.addPass(ReversePostOrderFunctionAttrsPass());
     692             : 
     693             :   // Re-require GloblasAA here prior to function passes. This is particularly
     694             :   // useful as the above will have inlined, DCE'ed, and function-attr
     695             :   // propagated everything. We should at this point have a reasonably minimal
     696             :   // and richly annotated call graph. By computing aliasing and mod/ref
     697             :   // information for all local globals here, the late loop passes and notably
     698             :   // the vectorizer will be able to use them to help recognize vectorizable
     699             :   // memory operations.
     700          40 :   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
     701             : 
     702         120 :   FunctionPassManager OptimizePM(DebugLogging);
     703          40 :   OptimizePM.addPass(Float2IntPass());
     704             :   // FIXME: We need to run some loop optimizations to re-rotate loops after
     705             :   // simplify-cfg and others undo their rotation.
     706             : 
     707             :   // Optimize the loop execution. These passes operate on entire loop nests
     708             :   // rather than on each loop in an inside-out manner, and so they are actually
     709             :   // function passes.
     710             : 
     711         146 :   for (auto &C : VectorizerStartEPCallbacks)
     712          26 :     C(OptimizePM, Level);
     713             : 
     714             :   // First rotate loops that may have been un-rotated by prior passes.
     715         120 :   OptimizePM.addPass(createFunctionToLoopPassAdaptor(LoopRotatePass()));
     716             : 
     717             :   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
     718             :   // into separate loop that would otherwise inhibit vectorization.  This is
     719             :   // currently only performed for loops marked with the metadata
     720             :   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
     721          40 :   OptimizePM.addPass(LoopDistributePass());
     722             : 
     723             :   // Now run the core loop vectorizer.
     724          40 :   OptimizePM.addPass(LoopVectorizePass());
     725             : 
     726             :   // Eliminate loads by forwarding stores from the previous iteration to loads
     727             :   // of the current iteration.
     728          40 :   OptimizePM.addPass(LoopLoadEliminationPass());
     729             : 
     730             :   // Cleanup after the loop optimization passes.
     731          80 :   OptimizePM.addPass(InstCombinePass());
     732             : 
     733             : 
     734             :   // Now that we've formed fast to execute loop structures, we do further
     735             :   // optimizations. These are run afterward as they might block doing complex
     736             :   // analyses and transforms such as what are needed for loop vectorization.
     737             : 
     738             :   // Optimize parallel scalar instruction chains into SIMD instructions.
     739          40 :   OptimizePM.addPass(SLPVectorizerPass());
     740             : 
     741             :   // Cleanup after all of the vectorizers.
     742          40 :   OptimizePM.addPass(SimplifyCFGPass());
     743          80 :   OptimizePM.addPass(InstCombinePass());
     744             : 
     745             :   // Unroll small loops to hide loop backedge latency and saturate any parallel
     746             :   // execution resources of an out-of-order processor. We also then need to
     747             :   // clean up redundancies and loop invariant code.
     748             :   // FIXME: It would be really good to use a loop-integrated instruction
     749             :   // combiner for cleanup here so that the unrolling and LICM can be pipelined
     750             :   // across the loop nests.
     751          80 :   OptimizePM.addPass(LoopUnrollPass(Level));
     752          80 :   OptimizePM.addPass(InstCombinePass());
     753          40 :   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
     754         120 :   OptimizePM.addPass(createFunctionToLoopPassAdaptor(LICMPass()));
     755             : 
     756             :   // Now that we've vectorized and unrolled loops, we may have more refined
     757             :   // alignment information, try to re-derive it here.
     758          80 :   OptimizePM.addPass(AlignmentFromAssumptionsPass());
     759             : 
     760             :   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
     761             :   // canonicalization pass that enables other optimizations. As a result,
     762             :   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
     763             :   // result too early.
     764          40 :   OptimizePM.addPass(LoopSinkPass());
     765             : 
     766             :   // And finally clean up LCSSA form before generating code.
     767          40 :   OptimizePM.addPass(InstSimplifierPass());
     768             : 
     769             :   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
     770             :   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
     771             :   // flattening of blocks.
     772          40 :   OptimizePM.addPass(DivRemPairsPass());
     773             : 
     774             :   // LoopSink (and other loop passes since the last simplifyCFG) might have
     775             :   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
     776          40 :   OptimizePM.addPass(SimplifyCFGPass());
     777             : 
     778             :   // Add the core optimizing pipeline.
     779         160 :   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
     780             : 
     781             :   // Now we need to do some global optimization transforms.
     782             :   // FIXME: It would seem like these should come first in the optimization
     783             :   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
     784             :   // ordering here.
     785          80 :   MPM.addPass(GlobalDCEPass());
     786          40 :   MPM.addPass(ConstantMergePass());
     787             : 
     788          40 :   return MPM;
     789             : }
     790             : 
     791             : ModulePassManager
     792          31 : PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
     793             :                                            bool DebugLogging) {
     794             :   assert(Level != O0 && "Must request optimizations for the default pipeline!");
     795             : 
     796          62 :   ModulePassManager MPM(DebugLogging);
     797             : 
     798             :   // Force any function attributes we want the rest of the pipeline to observe.
     799          31 :   MPM.addPass(ForceFunctionAttrsPass());
     800             : 
     801          36 :   if (PGOOpt && PGOOpt->SamplePGOSupport)
     802           6 :     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
     803             : 
     804             :   // Add the core simplification pipeline.
     805          62 :   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
     806             :                                                 DebugLogging));
     807             : 
     808             :   // Now add the optimization pipeline.
     809          62 :   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
     810             : 
     811          31 :   return MPM;
     812             : }
     813             : 
     814             : ModulePassManager
     815          13 : PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
     816             :                                                 bool DebugLogging) {
     817             :   assert(Level != O0 && "Must request optimizations for the default pipeline!");
     818             : 
     819          26 :   ModulePassManager MPM(DebugLogging);
     820             : 
     821             :   // Force any function attributes we want the rest of the pipeline to observe.
     822          13 :   MPM.addPass(ForceFunctionAttrsPass());
     823             : 
     824          16 :   if (PGOOpt && PGOOpt->SamplePGOSupport)
     825           6 :     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
     826             : 
     827             :   // If we are planning to perform ThinLTO later, we don't bloat the code with
     828             :   // unrolling/vectorization/... now. Just simplify the module as much as we
     829             :   // can.
     830          26 :   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
     831             :                                                 DebugLogging));
     832             : 
     833             :   // Run partial inlining pass to partially inline functions that have
     834             :   // large bodies.
     835             :   // FIXME: It isn't clear whether this is really the right place to run this
     836             :   // in ThinLTO. Because there is another canonicalization and simplification
     837             :   // phase that will run after the thin link, running this here ends up with
     838             :   // less information than will be available later and it may grow functions in
     839             :   // ways that aren't beneficial.
     840          13 :   if (RunPartialInlining)
     841           0 :     MPM.addPass(PartialInlinerPass());
     842             : 
     843             :   // Reduce the size of the IR as much as possible.
     844          13 :   MPM.addPass(GlobalOptPass());
     845             : 
     846          13 :   return MPM;
     847             : }
     848             : 
     849             : ModulePassManager
     850           9 : PassBuilder::buildThinLTODefaultPipeline(OptimizationLevel Level,
     851             :                                          bool DebugLogging) {
     852             :   // FIXME: The summary index is not hooked in the new pass manager yet.
     853             :   // When it's going to be hooked, enable WholeProgramDevirt and LowerTypeTest
     854             :   // here.
     855             : 
     856          18 :   ModulePassManager MPM(DebugLogging);
     857             : 
     858             :   // Force any function attributes we want the rest of the pipeline to observe.
     859           9 :   MPM.addPass(ForceFunctionAttrsPass());
     860             : 
     861             :   // During the ThinLTO backend phase we perform early indirect call promotion
     862             :   // here, before globalopt. Otherwise imported available_externally functions
     863             :   // look unreferenced and are removed.
     864             :   // FIXME: move this into buildModuleSimplificationPipeline to merge the logic
     865             :   //        with SamplePGO.
     866          15 :   if (!PGOOpt || PGOOpt->SampleProfileFile.empty())
     867           7 :     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */,
     868             :                                          false /* SamplePGO */));
     869             : 
     870             :   // Add the core simplification pipeline.
     871          18 :   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
     872             :                                                 DebugLogging));
     873             : 
     874             :   // Now add the optimization pipeline.
     875          18 :   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
     876             : 
     877           9 :   return MPM;
     878             : }
     879             : 
     880             : ModulePassManager
     881           6 : PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
     882             :                                             bool DebugLogging) {
     883             :   assert(Level != O0 && "Must request optimizations for the default pipeline!");
     884             :   // FIXME: We should use a customized pre-link pipeline!
     885           6 :   return buildPerModuleDefaultPipeline(Level, DebugLogging);
     886             : }
     887             : 
     888           8 : ModulePassManager PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
     889             :                                                        bool DebugLogging) {
     890             :   assert(Level != O0 && "Must request optimizations for the default pipeline!");
     891          16 :   ModulePassManager MPM(DebugLogging);
     892             : 
     893             :   // Remove unused virtual tables to improve the quality of code generated by
     894             :   // whole-program devirtualization and bitset lowering.
     895          16 :   MPM.addPass(GlobalDCEPass());
     896             : 
     897             :   // Force any function attributes we want the rest of the pipeline to observe.
     898           8 :   MPM.addPass(ForceFunctionAttrsPass());
     899             : 
     900             :   // Do basic inference of function attributes from known properties of system
     901             :   // libraries and other oracles.
     902           8 :   MPM.addPass(InferFunctionAttrsPass());
     903             : 
     904           8 :   if (Level > 1) {
     905             :     // Indirect call promotion. This should promote all the targets that are
     906             :     // left by the earlier promotion pass that promotes intra-module targets.
     907             :     // This two-step promotion is to save the compile time. For LTO, it should
     908             :     // produce the same result as if we only do promotion here.
     909          14 :     MPM.addPass(PGOIndirectCallPromotion(
     910           7 :         true /* InLTO */, PGOOpt && !PGOOpt->SampleProfileFile.empty()));
     911             : 
     912             :     // Propagate constants at call sites into the functions they call.  This
     913             :     // opens opportunities for globalopt (and inlining) by substituting function
     914             :     // pointers passed as arguments to direct uses of functions.
     915           7 :    MPM.addPass(IPSCCPPass());
     916             :   }
     917             : 
     918             :   // Now deduce any function attributes based in the current code.
     919          16 :   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
     920             :               PostOrderFunctionAttrsPass()));
     921             : 
     922             :   // Do RPO function attribute inference across the module to forward-propagate
     923             :   // attributes where applicable.
     924             :   // FIXME: Is this really an optimization rather than a canonicalization?
     925           8 :   MPM.addPass(ReversePostOrderFunctionAttrsPass());
     926             : 
     927             :   // Use inragne annotations on GEP indices to split globals where beneficial.
     928           8 :   MPM.addPass(GlobalSplitPass());
     929             : 
     930             :   // Run whole program optimization of virtual call when the list of callees
     931             :   // is fixed.
     932           8 :   MPM.addPass(WholeProgramDevirtPass());
     933             : 
     934             :   // Stop here at -O1.
     935           8 :   if (Level == 1)
     936             :     return MPM;
     937             : 
     938             :   // Optimize globals to try and fold them into constants.
     939           7 :   MPM.addPass(GlobalOptPass());
     940             : 
     941             :   // Promote any localized globals to SSA registers.
     942          14 :   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
     943             : 
     944             :   // Linking modules together can lead to duplicate global constant, only
     945             :   // keep one copy of each constant.
     946           7 :   MPM.addPass(ConstantMergePass());
     947             : 
     948             :   // Remove unused arguments from functions.
     949           7 :   MPM.addPass(DeadArgumentEliminationPass());
     950             : 
     951             :   // Reduce the code after globalopt and ipsccp.  Both can open up significant
     952             :   // simplification opportunities, and both can propagate functions through
     953             :   // function pointers.  When this happens, we often have to resolve varargs
     954             :   // calls, etc, so let instcombine do this.
     955          14 :   FunctionPassManager PeepholeFPM(DebugLogging);
     956          14 :   PeepholeFPM.addPass(InstCombinePass());
     957           7 :   invokePeepholeEPCallbacks(PeepholeFPM, Level);
     958             : 
     959          28 :   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
     960             : 
     961             :   // Note: historically, the PruneEH pass was run first to deduce nounwind and
     962             :   // generally clean up exception handling overhead. It isn't clear this is
     963             :   // valuable as the inliner doesn't currently care whether it is inlining an
     964             :   // invoke or a call.
     965             :   // Run the inliner now.
     966          14 :   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
     967          28 :       InlinerPass(getInlineParamsFromOptLevel(Level))));
     968             : 
     969             :   // Optimize globals again after we ran the inliner.
     970           7 :   MPM.addPass(GlobalOptPass());
     971             : 
     972             :   // Garbage collect dead functions.
     973             :   // FIXME: Add ArgumentPromotion pass after once it's ported.
     974          14 :   MPM.addPass(GlobalDCEPass());
     975             : 
     976          21 :   FunctionPassManager FPM(DebugLogging);
     977             :   // The IPO Passes may leave cruft around. Clean up after them.
     978          14 :   FPM.addPass(InstCombinePass());
     979           7 :   invokePeepholeEPCallbacks(FPM, Level);
     980             : 
     981           7 :   FPM.addPass(JumpThreadingPass());
     982             : 
     983             :   // Break up allocas
     984           7 :   FPM.addPass(SROA());
     985             : 
     986             :   // Run a few AA driver optimizations here and now to cleanup the code.
     987          28 :   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
     988             : 
     989          14 :   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
     990             :               PostOrderFunctionAttrsPass()));
     991             :   // FIXME: here we run IP alias analysis in the legacy PM.
     992             : 
     993          14 :   FunctionPassManager MainFPM;
     994             : 
     995             :   // FIXME: once we fix LoopPass Manager, add LICM here.
     996             :   // FIXME: once we provide support for enabling MLSM, add it here.
     997             :   // FIXME: once we provide support for enabling NewGVN, add it here.
     998           7 :   if (RunNewGVN)
     999           0 :     MainFPM.addPass(NewGVNPass());
    1000             :   else
    1001           7 :     MainFPM.addPass(GVN());
    1002             : 
    1003             :   // Remove dead memcpy()'s.
    1004          14 :   MainFPM.addPass(MemCpyOptPass());
    1005             : 
    1006             :   // Nuke dead stores.
    1007           7 :   MainFPM.addPass(DSEPass());
    1008             : 
    1009             :   // FIXME: at this point, we run a bunch of loop passes:
    1010             :   // indVarSimplify, loopDeletion, loopInterchange, loopUnrool,
    1011             :   // loopVectorize. Enable them once the remaining issue with LPM
    1012             :   // are sorted out.
    1013             : 
    1014          14 :   MainFPM.addPass(InstCombinePass());
    1015           7 :   MainFPM.addPass(SimplifyCFGPass());
    1016           7 :   MainFPM.addPass(SCCPPass());
    1017          14 :   MainFPM.addPass(InstCombinePass());
    1018           7 :   MainFPM.addPass(BDCEPass());
    1019             : 
    1020             :   // FIXME: We may want to run SLPVectorizer here.
    1021             :   // After vectorization, assume intrinsics may tell us more
    1022             :   // about pointer alignments.
    1023             : #if 0
    1024             :   MainFPM.add(AlignmentFromAssumptionsPass());
    1025             : #endif
    1026             : 
    1027             :   // FIXME: Conditionally run LoadCombine here, after it's ported
    1028             :   // (in case we still have this pass, given its questionable usefulness).
    1029             : 
    1030          14 :   MainFPM.addPass(InstCombinePass());
    1031           7 :   invokePeepholeEPCallbacks(MainFPM, Level);
    1032           7 :   MainFPM.addPass(JumpThreadingPass());
    1033          28 :   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
    1034             : 
    1035             :   // Create a function that performs CFI checks for cross-DSO calls with
    1036             :   // targets in the current module.
    1037           7 :   MPM.addPass(CrossDSOCFIPass());
    1038             : 
    1039             :   // Lower type metadata and the type.test intrinsic. This pass supports
    1040             :   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
    1041             :   // to be run at link time if CFI is enabled. This pass does nothing if
    1042             :   // CFI is disabled.
    1043             :   // Enable once we add support for the summary in the new PM.
    1044             : #if 0
    1045             :   MPM.addPass(LowerTypeTestsPass(Summary ? PassSummaryAction::Export :
    1046             :                                            PassSummaryAction::None,
    1047             :                                 Summary));
    1048             : #endif
    1049             : 
    1050             :   // Add late LTO optimization passes.
    1051             :   // Delete basic blocks, which optimization passes may have killed.
    1052          14 :   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
    1053             : 
    1054             :   // Drop bodies of available eternally objects to improve GlobalDCE.
    1055           7 :   MPM.addPass(EliminateAvailableExternallyPass());
    1056             : 
    1057             :   // Now that we have optimized the program, discard unreachable functions.
    1058          14 :   MPM.addPass(GlobalDCEPass());
    1059             : 
    1060             :   // FIXME: Enable MergeFuncs, conditionally, after ported, maybe.
    1061             :   return MPM;
    1062             : }
    1063             : 
    1064          39 : AAManager PassBuilder::buildDefaultAAPipeline() {
    1065          39 :   AAManager AA;
    1066             : 
    1067             :   // The order in which these are registered determines their priority when
    1068             :   // being queried.
    1069             : 
    1070             :   // First we register the basic alias analysis that provides the majority of
    1071             :   // per-function local AA logic. This is a stateless, on-demand local set of
    1072             :   // AA techniques.
    1073          39 :   AA.registerFunctionAnalysis<BasicAA>();
    1074             : 
    1075             :   // Next we query fast, specialized alias analyses that wrap IR-embedded
    1076             :   // information about aliasing.
    1077          39 :   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
    1078          39 :   AA.registerFunctionAnalysis<TypeBasedAA>();
    1079             : 
    1080             :   // Add support for querying global aliasing information when available.
    1081             :   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
    1082             :   // analysis, all that the `AAManager` can do is query for any *cached*
    1083             :   // results from `GlobalsAA` through a readonly proxy.
    1084          39 :   AA.registerModuleAnalysis<GlobalsAA>();
    1085             : 
    1086          39 :   return AA;
    1087             : }
    1088             : 
    1089        1321 : static Optional<int> parseRepeatPassName(StringRef Name) {
    1090        1331 :   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
    1091             :     return None;
    1092             :   int Count;
    1093           5 :   if (Name.getAsInteger(0, Count) || Count <= 0)
    1094             :     return None;
    1095             :   return Count;
    1096             : }
    1097             : 
    1098         379 : static Optional<int> parseDevirtPassName(StringRef Name) {
    1099         385 :   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
    1100             :     return None;
    1101             :   int Count;
    1102           3 :   if (Name.getAsInteger(0, Count) || Count <= 0)
    1103             :     return None;
    1104             :   return Count;
    1105             : }
    1106             : 
    1107             : /// Tests whether a pass name starts with a valid prefix for a default pipeline
    1108             : /// alias.
    1109        1008 : static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
    1110        1980 :   return Name.startswith("default") || Name.startswith("thinlto") ||
    1111        1952 :          Name.startswith("lto");
    1112             : }
    1113             : 
    1114             : /// Tests whether registered callbacks will accept a given pass name.
    1115             : ///
    1116             : /// When parsing a pipeline text, the type of the outermost pipeline may be
    1117             : /// omitted, in which case the type is automatically determined from the first
    1118             : /// pass name in the text. This may be a name that is handled through one of the
    1119             : /// callbacks. We check this through the oridinary parsing callbacks by setting
    1120             : /// up a dummy PassManager in order to not force the client to also handle this
    1121             : /// type of query.
    1122             : template <typename PassManagerT, typename CallbacksT>
    1123         745 : static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
    1124         745 :   if (!Callbacks.empty()) {
    1125          36 :     PassManagerT DummyPM;
    1126          48 :     for (auto &CB : Callbacks)
    1127          68 :       if (CB(Name, DummyPM, {}))
    1128          16 :         return true;
    1129             :   }
    1130             :   return false;
    1131             : }
    1132             : 
    1133             : template <typename CallbacksT>
    1134         725 : static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
    1135             :   // Manually handle aliases for pre-configured pipeline fragments.
    1136         725 :   if (startsWithDefaultPipelineAliasPrefix(Name))
    1137          39 :     return DefaultAliasRegex.match(Name);
    1138             : 
    1139             :   // Explicitly handle pass manager names.
    1140         686 :   if (Name == "module")
    1141             :     return true;
    1142         681 :   if (Name == "cgscc")
    1143             :     return true;
    1144         594 :   if (Name == "function")
    1145             :     return true;
    1146             : 
    1147             :   // Explicitly handle custom-parsed pass names.
    1148        1142 :   if (parseRepeatPassName(Name))
    1149             :     return true;
    1150             : 
    1151             : #define MODULE_PASS(NAME, CREATE_PASS)                                         \
    1152             :   if (Name == NAME)                                                            \
    1153             :     return true;
    1154             : #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
    1155             :   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
    1156             :     return true;
    1157             : #include "PassRegistry.def"
    1158             : 
    1159         375 :   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
    1160             : }
    1161             : 
    1162             : template <typename CallbacksT>
    1163         375 : static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
    1164             :   // Explicitly handle pass manager names.
    1165         375 :   if (Name == "cgscc")
    1166             :     return true;
    1167         375 :   if (Name == "function")
    1168             :     return true;
    1169             : 
    1170             :   // Explicitly handle custom-parsed pass names.
    1171         750 :   if (parseRepeatPassName(Name))
    1172             :     return true;
    1173         750 :   if (parseDevirtPassName(Name))
    1174             :     return true;
    1175             : 
    1176             : #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
    1177             :   if (Name == NAME)                                                            \
    1178             :     return true;
    1179             : #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
    1180             :   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
    1181             :     return true;
    1182             : #include "PassRegistry.def"
    1183             : 
    1184         344 :   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
    1185             : }
    1186             : 
    1187             : template <typename CallbacksT>
    1188         355 : static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
    1189             :   // Explicitly handle pass manager names.
    1190         355 :   if (Name == "function")
    1191             :     return true;
    1192         355 :   if (Name == "loop")
    1193             :     return true;
    1194             : 
    1195             :   // Explicitly handle custom-parsed pass names.
    1196         690 :   if (parseRepeatPassName(Name))
    1197             :     return true;
    1198             : 
    1199             : #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
    1200             :   if (Name == NAME)                                                            \
    1201             :     return true;
    1202             : #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
    1203             :   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
    1204             :     return true;
    1205             : #include "PassRegistry.def"
    1206             : 
    1207          20 :   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
    1208             : }
    1209             : 
    1210             : template <typename CallbacksT>
    1211          18 : static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
    1212             :   // Explicitly handle pass manager names.
    1213          18 :   if (Name == "loop")
    1214             :     return true;
    1215             : 
    1216             :   // Explicitly handle custom-parsed pass names.
    1217          36 :   if (parseRepeatPassName(Name))
    1218             :     return true;
    1219             : 
    1220             : #define LOOP_PASS(NAME, CREATE_PASS)                                           \
    1221             :   if (Name == NAME)                                                            \
    1222             :     return true;
    1223             : #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
    1224             :   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
    1225             :     return true;
    1226             : #include "PassRegistry.def"
    1227             : 
    1228           6 :   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
    1229             : }
    1230             : 
    1231             : Optional<std::vector<PassBuilder::PipelineElement>>
    1232         755 : PassBuilder::parsePipelineText(StringRef Text) {
    1233        1510 :   std::vector<PipelineElement> ResultPipeline;
    1234             : 
    1235             :   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
    1236        2265 :       &ResultPipeline};
    1237             :   for (;;) {
    1238        2918 :     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
    1239        1459 :     size_t Pos = Text.find_first_of(",()");
    1240        5836 :     Pipeline.push_back({Text.substr(0, Pos), {}});
    1241             : 
    1242             :     // If we have a single terminating name, we're done.
    1243        1459 :     if (Pos == Text.npos)
    1244             :       break;
    1245             : 
    1246        1868 :     char Sep = Text[Pos];
    1247        1868 :     Text = Text.substr(Pos + 1);
    1248         934 :     if (Sep == ',')
    1249             :       // Just a name ending in a comma, continue.
    1250         382 :       continue;
    1251             : 
    1252         841 :     if (Sep == '(') {
    1253             :       // Push the inner pipeline onto the stack to continue processing.
    1254         289 :       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
    1255         289 :       continue;
    1256             :     }
    1257             : 
    1258             :     assert(Sep == ')' && "Bogus separator!");
    1259             :     // When handling the close parenthesis, we greedily consume them to avoid
    1260             :     // empty strings in the pipeline.
    1261             :     do {
    1262             :       // If we try to pop the outer pipeline we have unbalanced parentheses.
    1263         293 :       if (PipelineStack.size() == 1)
    1264             :         return None;
    1265             : 
    1266         286 :       PipelineStack.pop_back();
    1267         316 :     } while (Text.consume_front(")"));
    1268             : 
    1269             :     // Check if we've finished parsing.
    1270         256 :     if (Text.empty())
    1271             :       break;
    1272             : 
    1273             :     // Otherwise, the end of an inner pipeline always has to be followed by
    1274             :     // a comma, and then we can continue.
    1275          67 :     if (!Text.consume_front(","))
    1276             :       return None;
    1277             :   }
    1278             : 
    1279         747 :   if (PipelineStack.size() > 1)
    1280             :     // Unbalanced paretheses.
    1281             :     return None;
    1282             : 
    1283             :   assert(PipelineStack.back() == &ResultPipeline &&
    1284             :          "Wrong pipeline at the bottom of the stack!");
    1285         744 :   return {std::move(ResultPipeline)};
    1286             : }
    1287             : 
    1288         802 : bool PassBuilder::parseModulePass(ModulePassManager &MPM,
    1289             :                                   const PipelineElement &E, bool VerifyEachPass,
    1290             :                                   bool DebugLogging) {
    1291         802 :   auto &Name = E.Name;
    1292         802 :   auto &InnerPipeline = E.InnerPipeline;
    1293             : 
    1294             :   // First handle complex passes like the pass managers which carry pipelines.
    1295         802 :   if (!InnerPipeline.empty()) {
    1296         526 :     if (Name == "module") {
    1297          21 :       ModulePassManager NestedMPM(DebugLogging);
    1298          14 :       if (!parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass,
    1299             :                                    DebugLogging))
    1300             :         return false;
    1301          21 :       MPM.addPass(std::move(NestedMPM));
    1302           7 :       return true;
    1303             :     }
    1304         647 :     if (Name == "cgscc") {
    1305         405 :       CGSCCPassManager CGPM(DebugLogging);
    1306         270 :       if (!parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
    1307             :                                   DebugLogging))
    1308             :         return false;
    1309         536 :       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
    1310         134 :       return true;
    1311             :     }
    1312         752 :     if (Name == "function") {
    1313        1125 :       FunctionPassManager FPM(DebugLogging);
    1314         750 :       if (!parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass,
    1315             :                                      DebugLogging))
    1316             :         return false;
    1317        1484 :       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
    1318         371 :       return true;
    1319             :     }
    1320           3 :     if (auto Count = parseRepeatPassName(Name)) {
    1321           3 :       ModulePassManager NestedMPM(DebugLogging);
    1322           2 :       if (!parseModulePassPipeline(NestedMPM, InnerPipeline, VerifyEachPass,
    1323             :                                    DebugLogging))
    1324             :         return false;
    1325           6 :       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
    1326           1 :       return true;
    1327             :     }
    1328             : 
    1329           3 :     for (auto &C : ModulePipelineParsingCallbacks)
    1330           0 :       if (C(Name, MPM, InnerPipeline))
    1331             :         return true;
    1332             : 
    1333             :     // Normal passes can't have pipelines.
    1334             :     return false;
    1335             :   }
    1336             : 
    1337             :   // Manually handle aliases for pre-configured pipeline fragments.
    1338         283 :   if (startsWithDefaultPipelineAliasPrefix(Name)) {
    1339          78 :     SmallVector<StringRef, 3> Matches;
    1340          39 :     if (!DefaultAliasRegex.match(Name, &Matches))
    1341             :       return false;
    1342             :     assert(Matches.size() == 3 && "Must capture two matched strings!");
    1343             : 
    1344          39 :     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
    1345         117 :                               .Case("O0", O0)
    1346         117 :                               .Case("O1", O1)
    1347         117 :                               .Case("O2", O2)
    1348         117 :                               .Case("O3", O3)
    1349         117 :                               .Case("Os", Os)
    1350         117 :                               .Case("Oz", Oz);
    1351          39 :     if (L == O0)
    1352             :       // At O0 we do nothing at all!
    1353             :       return true;
    1354             : 
    1355          93 :     if (Matches[1] == "default") {
    1356          34 :       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
    1357          49 :     } else if (Matches[1] == "thinlto-pre-link") {
    1358          14 :       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
    1359          35 :     } else if (Matches[1] == "thinlto") {
    1360          14 :       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging));
    1361          15 :     } else if (Matches[1] == "lto-pre-link") {
    1362           2 :       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
    1363             :     } else {
    1364             :       assert(Matches[1] == "lto" && "Not one of the matched options!");
    1365          12 :       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging));
    1366             :     }
    1367             :     return true;
    1368             :   }
    1369             : 
    1370             :   // Finally expand the basic registered passes from the .inc file.
    1371             : #define MODULE_PASS(NAME, CREATE_PASS)                                         \
    1372             :   if (Name == NAME) {                                                          \
    1373             :     MPM.addPass(CREATE_PASS);                                                  \
    1374             :     return true;                                                               \
    1375             :   }
    1376             : #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
    1377             :   if (Name == "require<" NAME ">") {                                           \
    1378             :     MPM.addPass(                                                               \
    1379             :         RequireAnalysisPass<                                                   \
    1380             :             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
    1381             :     return true;                                                               \
    1382             :   }                                                                            \
    1383             :   if (Name == "invalidate<" NAME ">") {                                        \
    1384             :     MPM.addPass(InvalidateAnalysisPass<                                        \
    1385             :                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
    1386             :     return true;                                                               \
    1387             :   }
    1388             : #include "PassRegistry.def"
    1389             : 
    1390           9 :   for (auto &C : ModulePipelineParsingCallbacks)
    1391           6 :     if (C(Name, MPM, InnerPipeline))
    1392             :       return true;
    1393             :   return false;
    1394             : }
    1395             : 
    1396         173 : bool PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
    1397             :                                  const PipelineElement &E, bool VerifyEachPass,
    1398             :                                  bool DebugLogging) {
    1399         173 :   auto &Name = E.Name;
    1400         173 :   auto &InnerPipeline = E.InnerPipeline;
    1401             : 
    1402             :   // First handle complex passes like the pass managers which carry pipelines.
    1403         173 :   if (!InnerPipeline.empty()) {
    1404          26 :     if (Name == "cgscc") {
    1405           0 :       CGSCCPassManager NestedCGPM(DebugLogging);
    1406           0 :       if (!parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass,
    1407             :                                   DebugLogging))
    1408             :         return false;
    1409             :       // Add the nested pass manager with the appropriate adaptor.
    1410           0 :       CGPM.addPass(std::move(NestedCGPM));
    1411           0 :       return true;
    1412             :     }
    1413          47 :     if (Name == "function") {
    1414          63 :       FunctionPassManager FPM(DebugLogging);
    1415          42 :       if (!parseFunctionPassPipeline(FPM, InnerPipeline, VerifyEachPass,
    1416             :                                      DebugLogging))
    1417             :         return false;
    1418             :       // Add the nested pass manager with the appropriate adaptor.
    1419          84 :       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
    1420          21 :       return true;
    1421             :     }
    1422           9 :     if (auto Count = parseRepeatPassName(Name)) {
    1423           3 :       CGSCCPassManager NestedCGPM(DebugLogging);
    1424           2 :       if (!parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass,
    1425             :                                   DebugLogging))
    1426             :         return false;
    1427           6 :       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
    1428           1 :       return true;
    1429             :     }
    1430           5 :     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
    1431           9 :       CGSCCPassManager NestedCGPM(DebugLogging);
    1432           6 :       if (!parseCGSCCPassPipeline(NestedCGPM, InnerPipeline, VerifyEachPass,
    1433             :                                   DebugLogging))
    1434             :         return false;
    1435           3 :       CGPM.addPass(
    1436          15 :           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
    1437           3 :       return true;
    1438             :     }
    1439             : 
    1440           3 :     for (auto &C : CGSCCPipelineParsingCallbacks)
    1441           0 :       if (C(Name, CGPM, InnerPipeline))
    1442             :         return true;
    1443             : 
    1444             :     // Normal passes can't have pipelines.
    1445             :     return false;
    1446             :   }
    1447             : 
    1448             : // Now expand the basic registered passes from the .inc file.
    1449             : #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
    1450             :   if (Name == NAME) {                                                          \
    1451             :     CGPM.addPass(CREATE_PASS);                                                 \
    1452             :     return true;                                                               \
    1453             :   }
    1454             : #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
    1455             :   if (Name == "require<" NAME ">") {                                           \
    1456             :     CGPM.addPass(RequireAnalysisPass<                                          \
    1457             :                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
    1458             :                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
    1459             :                  CGSCCUpdateResult &>());                                      \
    1460             :     return true;                                                               \
    1461             :   }                                                                            \
    1462             :   if (Name == "invalidate<" NAME ">") {                                        \
    1463             :     CGPM.addPass(InvalidateAnalysisPass<                                       \
    1464             :                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
    1465             :     return true;                                                               \
    1466             :   }
    1467             : #include "PassRegistry.def"
    1468             : 
    1469           9 :   for (auto &C : CGSCCPipelineParsingCallbacks)
    1470           6 :     if (C(Name, CGPM, InnerPipeline))
    1471             :       return true;
    1472             :   return false;
    1473             : }
    1474             : 
    1475         704 : bool PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
    1476             :                                     const PipelineElement &E,
    1477             :                                     bool VerifyEachPass, bool DebugLogging) {
    1478         704 :   auto &Name = E.Name;
    1479         704 :   auto &InnerPipeline = E.InnerPipeline;
    1480             : 
    1481             :   // First handle complex passes like the pass managers which carry pipelines.
    1482         704 :   if (!InnerPipeline.empty()) {
    1483         114 :     if (Name == "function") {
    1484           0 :       FunctionPassManager NestedFPM(DebugLogging);
    1485           0 :       if (!parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass,
    1486             :                                      DebugLogging))
    1487             :         return false;
    1488             :       // Add the nested pass manager with the appropriate adaptor.
    1489           0 :       FPM.addPass(std::move(NestedFPM));
    1490           0 :       return true;
    1491             :     }
    1492         225 :     if (Name == "loop") {
    1493         333 :       LoopPassManager LPM(DebugLogging);
    1494         222 :       if (!parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
    1495             :                                  DebugLogging))
    1496             :         return false;
    1497             :       // Add the nested pass manager with the appropriate adaptor.
    1498         440 :       FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM)));
    1499         110 :       return true;
    1500             :     }
    1501           5 :     if (auto Count = parseRepeatPassName(Name)) {
    1502           3 :       FunctionPassManager NestedFPM(DebugLogging);
    1503           2 :       if (!parseFunctionPassPipeline(NestedFPM, InnerPipeline, VerifyEachPass,
    1504             :                                      DebugLogging))
    1505             :         return false;
    1506           6 :       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
    1507           1 :       return true;
    1508             :     }
    1509             : 
    1510          10 :     for (auto &C : FunctionPipelineParsingCallbacks)
    1511           8 :       if (C(Name, FPM, InnerPipeline))
    1512             :         return true;
    1513             : 
    1514             :     // Normal passes can't have pipelines.
    1515             :     return false;
    1516             :   }
    1517             : 
    1518             : // Now expand the basic registered passes from the .inc file.
    1519             : #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
    1520             :   if (Name == NAME) {                                                          \
    1521             :     FPM.addPass(CREATE_PASS);                                                  \
    1522             :     return true;                                                               \
    1523             :   }
    1524             : #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
    1525             :   if (Name == "require<" NAME ">") {                                           \
    1526             :     FPM.addPass(                                                               \
    1527             :         RequireAnalysisPass<                                                   \
    1528             :             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
    1529             :     return true;                                                               \
    1530             :   }                                                                            \
    1531             :   if (Name == "invalidate<" NAME ">") {                                        \
    1532             :     FPM.addPass(InvalidateAnalysisPass<                                        \
    1533             :                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
    1534             :     return true;                                                               \
    1535             :   }
    1536             : #include "PassRegistry.def"
    1537             : 
    1538          14 :   for (auto &C : FunctionPipelineParsingCallbacks)
    1539          10 :     if (C(Name, FPM, InnerPipeline))
    1540             :       return true;
    1541             :   return false;
    1542             : }
    1543             : 
    1544         125 : bool PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
    1545             :                                 bool VerifyEachPass, bool DebugLogging) {
    1546         125 :   StringRef Name = E.Name;
    1547         125 :   auto &InnerPipeline = E.InnerPipeline;
    1548             : 
    1549             :   // First handle complex passes like the pass managers which carry pipelines.
    1550         125 :   if (!InnerPipeline.empty()) {
    1551           2 :     if (Name == "loop") {
    1552           0 :       LoopPassManager NestedLPM(DebugLogging);
    1553           0 :       if (!parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass,
    1554             :                                  DebugLogging))
    1555             :         return false;
    1556             :       // Add the nested pass manager with the appropriate adaptor.
    1557           0 :       LPM.addPass(std::move(NestedLPM));
    1558           0 :       return true;
    1559             :     }
    1560           3 :     if (auto Count = parseRepeatPassName(Name)) {
    1561           3 :       LoopPassManager NestedLPM(DebugLogging);
    1562           2 :       if (!parseLoopPassPipeline(NestedLPM, InnerPipeline, VerifyEachPass,
    1563             :                                  DebugLogging))
    1564             :         return false;
    1565           6 :       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
    1566           1 :       return true;
    1567             :     }
    1568             : 
    1569           3 :     for (auto &C : LoopPipelineParsingCallbacks)
    1570           0 :       if (C(Name, LPM, InnerPipeline))
    1571             :         return true;
    1572             : 
    1573             :     // Normal passes can't have pipelines.
    1574             :     return false;
    1575             :   }
    1576             : 
    1577             : // Now expand the basic registered passes from the .inc file.
    1578             : #define LOOP_PASS(NAME, CREATE_PASS)                                           \
    1579             :   if (Name == NAME) {                                                          \
    1580             :     LPM.addPass(CREATE_PASS);                                                  \
    1581             :     return true;                                                               \
    1582             :   }
    1583             : #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
    1584             :   if (Name == "require<" NAME ">") {                                           \
    1585             :     LPM.addPass(RequireAnalysisPass<                                           \
    1586             :                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
    1587             :                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
    1588             :                 LPMUpdater &>());                                              \
    1589             :     return true;                                                               \
    1590             :   }                                                                            \
    1591             :   if (Name == "invalidate<" NAME ">") {                                        \
    1592             :     LPM.addPass(InvalidateAnalysisPass<                                        \
    1593             :                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
    1594             :     return true;                                                               \
    1595             :   }
    1596             : #include "PassRegistry.def"
    1597             : 
    1598           9 :   for (auto &C : LoopPipelineParsingCallbacks)
    1599           6 :     if (C(Name, LPM, InnerPipeline))
    1600             :       return true;
    1601             :   return false;
    1602             : }
    1603             : 
    1604         137 : bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
    1605             : #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
    1606             :   if (Name == NAME) {                                                          \
    1607             :     AA.registerModuleAnalysis<                                                 \
    1608             :         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
    1609             :     return true;                                                               \
    1610             :   }
    1611             : #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
    1612             :   if (Name == NAME) {                                                          \
    1613             :     AA.registerFunctionAnalysis<                                               \
    1614             :         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
    1615             :     return true;                                                               \
    1616             :   }
    1617             : #include "PassRegistry.def"
    1618             : 
    1619           6 :   for (auto &C : AAParsingCallbacks)
    1620           0 :     if (C(Name, AA))
    1621             :       return true;
    1622             :   return false;
    1623             : }
    1624             : 
    1625         116 : bool PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
    1626             :                                         ArrayRef<PipelineElement> Pipeline,
    1627             :                                         bool VerifyEachPass,
    1628             :                                         bool DebugLogging) {
    1629         356 :   for (const auto &Element : Pipeline) {
    1630         125 :     if (!parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
    1631             :       return false;
    1632             :     // FIXME: No verifier support for Loop passes!
    1633             :   }
    1634             :   return true;
    1635             : }
    1636             : 
    1637         410 : bool PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
    1638             :                                             ArrayRef<PipelineElement> Pipeline,
    1639             :                                             bool VerifyEachPass,
    1640             :                                             bool DebugLogging) {
    1641        1520 :   for (const auto &Element : Pipeline) {
    1642         704 :     if (!parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
    1643             :       return false;
    1644         700 :     if (VerifyEachPass)
    1645          13 :       FPM.addPass(VerifierPass());
    1646             :   }
    1647             :   return true;
    1648             : }
    1649             : 
    1650         141 : bool PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
    1651             :                                          ArrayRef<PipelineElement> Pipeline,
    1652             :                                          bool VerifyEachPass,
    1653             :                                          bool DebugLogging) {
    1654         454 :   for (const auto &Element : Pipeline) {
    1655         173 :     if (!parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
    1656             :       return false;
    1657             :     // FIXME: No verifier support for CGSCC passes!
    1658             :   }
    1659             :   return true;
    1660             : }
    1661             : 
    1662         772 : void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
    1663             :                                        FunctionAnalysisManager &FAM,
    1664             :                                        CGSCCAnalysisManager &CGAM,
    1665             :                                        ModuleAnalysisManager &MAM) {
    1666        1544 :   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
    1667        1544 :   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
    1668        1544 :   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
    1669        1544 :   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
    1670        1544 :   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
    1671        1544 :   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
    1672        1544 :   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
    1673         772 : }
    1674             : 
    1675         729 : bool PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
    1676             :                                           ArrayRef<PipelineElement> Pipeline,
    1677             :                                           bool VerifyEachPass,
    1678             :                                           bool DebugLogging) {
    1679        2253 :   for (const auto &Element : Pipeline) {
    1680         802 :     if (!parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
    1681             :       return false;
    1682         795 :     if (VerifyEachPass)
    1683          15 :       MPM.addPass(VerifierPass());
    1684             :   }
    1685             :   return true;
    1686             : }
    1687             : 
    1688             : // Primary pass pipeline description parsing routine for a \c ModulePassManager
    1689             : // FIXME: Should this routine accept a TargetMachine or require the caller to
    1690             : // pre-populate the analysis managers with target-specific stuff?
    1691         736 : bool PassBuilder::parsePassPipeline(ModulePassManager &MPM,
    1692             :                                     StringRef PipelineText, bool VerifyEachPass,
    1693             :                                     bool DebugLogging) {
    1694        1471 :   auto Pipeline = parsePipelineText(PipelineText);
    1695        2186 :   if (!Pipeline || Pipeline->empty())
    1696             :     return false;
    1697             : 
    1698             :   // If the first name isn't at the module layer, wrap the pipeline up
    1699             :   // automatically.
    1700        1450 :   StringRef FirstName = Pipeline->front().Name;
    1701             : 
    1702         725 :   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
    1703         373 :     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
    1704         217 :       Pipeline = {{"cgscc", std::move(*Pipeline)}};
    1705         342 :     } else if (isFunctionPassName(FirstName,
    1706             :                                   FunctionPipelineParsingCallbacks)) {
    1707        2268 :       Pipeline = {{"function", std::move(*Pipeline)}};
    1708          18 :     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
    1709         168 :       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
    1710             :     } else {
    1711          13 :       for (auto &C : TopLevelPipelineParsingCallbacks)
    1712           8 :         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
    1713             :           return true;
    1714             : 
    1715             :       // Unknown pass name!
    1716             :       return false;
    1717             :     }
    1718             :   }
    1719             : 
    1720        2163 :   return parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging);
    1721             : }
    1722             : 
    1723             : // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
    1724           2 : bool PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
    1725             :                                     StringRef PipelineText, bool VerifyEachPass,
    1726             :                                     bool DebugLogging) {
    1727           4 :   auto Pipeline = parsePipelineText(PipelineText);
    1728           6 :   if (!Pipeline || Pipeline->empty())
    1729             :     return false;
    1730             : 
    1731           4 :   StringRef FirstName = Pipeline->front().Name;
    1732           2 :   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
    1733             :     return false;
    1734             : 
    1735           6 :   return parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging);
    1736             : }
    1737             : 
    1738             : // Primary pass pipeline description parsing routine for a \c
    1739             : // FunctionPassManager
    1740          13 : bool PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
    1741             :                                     StringRef PipelineText, bool VerifyEachPass,
    1742             :                                     bool DebugLogging) {
    1743          26 :   auto Pipeline = parsePipelineText(PipelineText);
    1744          39 :   if (!Pipeline || Pipeline->empty())
    1745             :     return false;
    1746             : 
    1747          26 :   StringRef FirstName = Pipeline->front().Name;
    1748          13 :   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
    1749             :     return false;
    1750             : 
    1751          52 :   return parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
    1752          13 :                                    DebugLogging);
    1753             : }
    1754             : 
    1755             : // Primary pass pipeline description parsing routine for a \c LoopPassManager
    1756           4 : bool PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
    1757             :                                     StringRef PipelineText, bool VerifyEachPass,
    1758             :                                     bool DebugLogging) {
    1759           8 :   auto Pipeline = parsePipelineText(PipelineText);
    1760          12 :   if (!Pipeline || Pipeline->empty())
    1761             :     return false;
    1762             : 
    1763          12 :   return parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging);
    1764             : }
    1765             : 
    1766         728 : bool PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
    1767             :   // If the pipeline just consists of the word 'default' just replace the AA
    1768             :   // manager with our default one.
    1769         734 :   if (PipelineText == "default") {
    1770          18 :     AA = buildDefaultAAPipeline();
    1771           6 :     return true;
    1772             :   }
    1773             : 
    1774         992 :   while (!PipelineText.empty()) {
    1775             :     StringRef Name;
    1776         411 :     std::tie(Name, PipelineText) = PipelineText.split(',');
    1777         137 :     if (!parseAAPassName(AA, Name))
    1778           2 :       return false;
    1779             :   }
    1780             : 
    1781             :   return true;
    1782      216918 : }

Generated by: LCOV version 1.13