LLVM  13.0.0git
PassBuilder.cpp
Go to the documentation of this file.
1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16 
18 #include "llvm/ADT/StringSwitch.h"
30 #include "llvm/Analysis/DDG.h"
40 #include "llvm/Analysis/IVUsers.h"
46 #include "llvm/Analysis/Lint.h"
49 #include "llvm/Analysis/LoopInfo.h"
71 #include "llvm/IR/Dominators.h"
73 #include "llvm/IR/PassManager.h"
74 #include "llvm/IR/PrintPasses.h"
76 #include "llvm/IR/Verifier.h"
78 #include "llvm/Support/Debug.h"
81 #include "llvm/Support/Regex.h"
138 #include "llvm/Transforms/ObjCARC.h"
241 
242 using namespace llvm;
243 
245  "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
246  cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
248  "Heuristics-based inliner version."),
250  "Use development mode (runtime-loadable model)."),
252  "Use release mode (AOT-compiled model).")));
253 
255  "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
256  cl::desc("Run synthetic function entry count generation "
257  "pass"));
258 
259 static const Regex DefaultAliasRegex(
260  "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
261 
262 /// Flag to enable inline deferral during PGO.
263 static cl::opt<bool>
264  EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
265  cl::Hidden,
266  cl::desc("Enable inline deferral during PGO"));
267 
268 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
270  cl::desc("Enable memory profiler"));
271 
273  "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
274  cl::desc("Perform mandatory inlinings module-wide, before performing "
275  "inlining."));
276 
278  "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
279  cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"));
280 
282  LoopInterleaving = true;
283  LoopVectorization = true;
284  SLPVectorization = false;
285  LoopUnrolling = true;
287  Coroutines = false;
290  CallGraphProfile = true;
291  MergeFunctions = false;
292 }
293 
294 namespace llvm {
302 extern cl::opt<bool> EnableCHR;
306 extern cl::opt<bool> RunNewGVN;
309 
311 
314 
316 
319 } // namespace llvm
320 
322  /*SpeedLevel*/ 0,
323  /*SizeLevel*/ 0};
325  /*SpeedLevel*/ 1,
326  /*SizeLevel*/ 0};
328  /*SpeedLevel*/ 2,
329  /*SizeLevel*/ 0};
331  /*SpeedLevel*/ 3,
332  /*SizeLevel*/ 0};
334  /*SpeedLevel*/ 2,
335  /*SizeLevel*/ 1};
337  /*SpeedLevel*/ 2,
338  /*SizeLevel*/ 2};
339 
340 namespace {
341 
342 // The following passes/analyses have custom names, otherwise their name will
343 // include `(anonymous namespace)`. These are special since they are only for
344 // testing purposes and don't live in a header file.
345 
346 /// No-op module pass which does nothing.
347 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
349  return PreservedAnalyses::all();
350  }
351 
352  static StringRef name() { return "NoOpModulePass"; }
353 };
354 
355 /// No-op module analysis.
356 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
358  static AnalysisKey Key;
359 
360 public:
361  struct Result {};
362  Result run(Module &, ModuleAnalysisManager &) { return Result(); }
363  static StringRef name() { return "NoOpModuleAnalysis"; }
364 };
365 
366 /// No-op CGSCC pass which does nothing.
367 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
370  return PreservedAnalyses::all();
371  }
372  static StringRef name() { return "NoOpCGSCCPass"; }
373 };
374 
375 /// No-op CGSCC analysis.
376 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
378  static AnalysisKey Key;
379 
380 public:
381  struct Result {};
383  return Result();
384  }
385  static StringRef name() { return "NoOpCGSCCAnalysis"; }
386 };
387 
388 /// No-op function pass which does nothing.
389 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
391  return PreservedAnalyses::all();
392  }
393  static StringRef name() { return "NoOpFunctionPass"; }
394 };
395 
396 /// No-op function analysis.
397 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
399  static AnalysisKey Key;
400 
401 public:
402  struct Result {};
403  Result run(Function &, FunctionAnalysisManager &) { return Result(); }
404  static StringRef name() { return "NoOpFunctionAnalysis"; }
405 };
406 
407 /// No-op loop pass which does nothing.
408 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
411  return PreservedAnalyses::all();
412  }
413  static StringRef name() { return "NoOpLoopPass"; }
414 };
415 
416 /// No-op loop analysis.
417 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
419  static AnalysisKey Key;
420 
421 public:
422  struct Result {};
424  return Result();
425  }
426  static StringRef name() { return "NoOpLoopAnalysis"; }
427 };
428 
433 
434 /// Whether or not we should populate a PassInstrumentationCallbacks's class to
435 /// pass name map.
436 ///
437 /// This is for optimization purposes so we don't populate it if we never use
438 /// it. This should be updated if new pass instrumentation wants to use the map.
439 /// We currently only use this for --print-before/after.
440 bool shouldPopulateClassToPassNames() {
441  return !printBeforePasses().empty() || !printAfterPasses().empty();
442 }
443 
444 } // namespace
445 
447  Optional<PGOOptions> PGOOpt,
449  : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
450  if (TM)
451  TM->registerPassBuilderCallbacks(*this);
452  if (PIC && shouldPopulateClassToPassNames()) {
453 #define MODULE_PASS(NAME, CREATE_PASS) \
454  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
455 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
456  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
457 #define FUNCTION_PASS(NAME, CREATE_PASS) \
458  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
459 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
460  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
461 #define LOOP_PASS(NAME, CREATE_PASS) \
462  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
463 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
464  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
465 #define CGSCC_PASS(NAME, CREATE_PASS) \
466  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
467 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
468  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
469 #include "PassRegistry.def"
470  }
471 }
472 
473 void PassBuilder::invokePeepholeEPCallbacks(
475  for (auto &C : PeepholeEPCallbacks)
476  C(FPM, Level);
477 }
478 
480 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
481  MAM.registerPass([&] { return CREATE_PASS; });
482 #include "PassRegistry.def"
483 
484  for (auto &C : ModuleAnalysisRegistrationCallbacks)
485  C(MAM);
486 }
487 
489 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
490  CGAM.registerPass([&] { return CREATE_PASS; });
491 #include "PassRegistry.def"
492 
493  for (auto &C : CGSCCAnalysisRegistrationCallbacks)
494  C(CGAM);
495 }
496 
498 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
499  FAM.registerPass([&] { return CREATE_PASS; });
500 #include "PassRegistry.def"
501 
502  for (auto &C : FunctionAnalysisRegistrationCallbacks)
503  C(FAM);
504 }
505 
507 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
508  LAM.registerPass([&] { return CREATE_PASS; });
509 #include "PassRegistry.def"
510 
511  for (auto &C : LoopAnalysisRegistrationCallbacks)
512  C(LAM);
513 }
514 
515 // Helper to add AnnotationRemarksPass.
520 }
521 
522 // Helper to check if the current compilation phase is preparing for LTO
526 }
527 
528 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
530 PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
532 
534 
535  // Form SSA out of local memory accesses after breaking apart aggregates into
536  // scalars.
537  FPM.addPass(SROA());
538 
539  // Catch trivial redundancies
540  FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
541 
542  // Hoisting of scalars and load expressions.
543  FPM.addPass(SimplifyCFGPass());
544  FPM.addPass(InstCombinePass());
545 
547 
548  invokePeepholeEPCallbacks(FPM, Level);
549 
550  FPM.addPass(SimplifyCFGPass());
551 
552  // Form canonically associated expression trees, and simplify the trees using
553  // basic mathematical properties. For example, this will form (nearly)
554  // minimal multiplication trees.
555  FPM.addPass(ReassociatePass());
556 
557  // Add the primary loop simplification pipeline.
558  // FIXME: Currently this is split into two loop pass pipelines because we run
559  // some function passes in between them. These can and should be removed
560  // and/or replaced by scheduling the loop pass equivalents in the correct
561  // positions. But those equivalent passes aren't powerful enough yet.
562  // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
563  // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
564  // fully replace `SimplifyCFGPass`, and the closest to the other we have is
565  // `LoopInstSimplify`.
566  LoopPassManager LPM1, LPM2;
567 
568  // Simplify the loop body. We do this initially to clean up after other loop
569  // passes run, either when iterating on a loop or on inner loops with
570  // implications on the outer loop.
573 
574  // Try to remove as much code from the loop header as possible,
575  // to reduce amount of IR that will have to be duplicated.
576  // TODO: Investigate promotion cap for O1.
578 
579  LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,
580  isLTOPreLink(Phase)));
581  // TODO: Investigate promotion cap for O1.
584 
586  LPM2.addPass(IndVarSimplifyPass());
587 
588  for (auto &C : LateLoopOptimizationsEPCallbacks)
589  C(LPM2, Level);
590 
591  LPM2.addPass(LoopDeletionPass());
592 
595 
596  // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
597  // because it changes IR to makes profile annotation in back compile
598  // inaccurate. The normal unroller doesn't pay attention to forced full unroll
599  // attributes so we need to make sure and allow the full unroll pass to pay
600  // attention to it.
601  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
602  PGOOpt->Action != PGOOptions::SampleUse)
603  LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
604  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
606 
607  for (auto &C : LoopOptimizerEndEPCallbacks)
608  C(LPM2, Level);
609 
610  // We provide the opt remark emitter pass for LICM to use. We only need to do
611  // this once as it is immutable.
612  FPM.addPass(
616  /*UseBlockFrequencyInfo=*/true));
617  FPM.addPass(SimplifyCFGPass());
618  FPM.addPass(InstCombinePass());
619  if (EnableLoopFlatten)
621  // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
622  // *All* loop passes must preserve it, in order to be able to use it.
624  /*UseMemorySSA=*/false,
625  /*UseBlockFrequencyInfo=*/false));
626 
627  // Delete small array after loop unroll.
628  FPM.addPass(SROA());
629 
630  // Specially optimize memory movement as it doesn't look like dataflow in SSA.
631  FPM.addPass(MemCpyOptPass());
632 
633  // Sparse conditional constant propagation.
634  // FIXME: It isn't clear why we do this *after* loop passes rather than
635  // before...
636  FPM.addPass(SCCPPass());
637 
638  // Delete dead bit computations (instcombine runs after to fold away the dead
639  // computations, and then ADCE will run later to exploit any new DCE
640  // opportunities that creates).
641  FPM.addPass(BDCEPass());
642 
643  // Run instcombine after redundancy and dead bit elimination to exploit
644  // opportunities opened up by them.
645  FPM.addPass(InstCombinePass());
646  invokePeepholeEPCallbacks(FPM, Level);
647 
648  if (PTO.Coroutines)
649  FPM.addPass(CoroElidePass());
650 
651  for (auto &C : ScalarOptimizerLateEPCallbacks)
652  C(FPM, Level);
653 
654  // Finally, do an expensive DCE pass to catch all the dead code exposed by
655  // the simplifications and basic cleanup after all the simplifications.
656  // TODO: Investigate if this is too expensive.
657  FPM.addPass(ADCEPass());
658  FPM.addPass(SimplifyCFGPass());
659  FPM.addPass(InstCombinePass());
660  invokePeepholeEPCallbacks(FPM, Level);
661 
662  return FPM;
663 }
664 
668  assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
669 
670  // The O1 pipeline has a separate pipeline creation function to simplify
671  // construction readability.
672  if (Level.getSpeedupLevel() == 1)
673  return buildO1FunctionSimplificationPipeline(Level, Phase);
674 
676 
677  // Form SSA out of local memory accesses after breaking apart aggregates into
678  // scalars.
679  FPM.addPass(SROA());
680 
681  // Catch trivial redundancies
682  FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
685 
686  // Hoisting of scalars and load expressions.
687  if (EnableGVNHoist)
688  FPM.addPass(GVNHoistPass());
689 
690  // Global value numbering based sinking.
691  if (EnableGVNSink) {
692  FPM.addPass(GVNSinkPass());
693  FPM.addPass(SimplifyCFGPass());
694  }
695 
698 
699  // Speculative execution if the target has divergent branches; otherwise nop.
700  FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
701 
702  // Optimize based on known information about branches, and cleanup afterward.
703  FPM.addPass(JumpThreadingPass());
705 
706  FPM.addPass(SimplifyCFGPass());
709  FPM.addPass(InstCombinePass());
710 
711  if (!Level.isOptimizingForSize())
713 
714  invokePeepholeEPCallbacks(FPM, Level);
715 
716  // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
717  // using the size value profile. Don't perform this when optimizing for size.
718  if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
719  !Level.isOptimizingForSize())
720  FPM.addPass(PGOMemOPSizeOpt());
721 
722  FPM.addPass(TailCallElimPass());
723  FPM.addPass(SimplifyCFGPass());
724 
725  // Form canonically associated expression trees, and simplify the trees using
726  // basic mathematical properties. For example, this will form (nearly)
727  // minimal multiplication trees.
728  FPM.addPass(ReassociatePass());
729 
730  // Add the primary loop simplification pipeline.
731  // FIXME: Currently this is split into two loop pass pipelines because we run
732  // some function passes in between them. These can and should be removed
733  // and/or replaced by scheduling the loop pass equivalents in the correct
734  // positions. But those equivalent passes aren't powerful enough yet.
735  // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
736  // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
737  // fully replace `SimplifyCFGPass`, and the closest to the other we have is
738  // `LoopInstSimplify`.
739  LoopPassManager LPM1, LPM2;
740 
741  // Simplify the loop body. We do this initially to clean up after other loop
742  // passes run, either when iterating on a loop or on inner loops with
743  // implications on the outer loop.
746 
747  // Try to remove as much code from the loop header as possible,
748  // to reduce amount of IR that will have to be duplicated.
749  // TODO: Investigate promotion cap for O1.
751 
752  // Disable header duplication in loop rotation at -Oz.
753  LPM1.addPass(
755  // TODO: Investigate promotion cap for O1.
757  LPM1.addPass(
758  SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3 &&
761  LPM2.addPass(IndVarSimplifyPass());
762 
763  for (auto &C : LateLoopOptimizationsEPCallbacks)
764  C(LPM2, Level);
765 
766  LPM2.addPass(LoopDeletionPass());
767 
770 
771  // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
772  // because it changes IR to makes profile annotation in back compile
773  // inaccurate. The normal unroller doesn't pay attention to forced full unroll
774  // attributes so we need to make sure and allow the full unroll pass to pay
775  // attention to it.
776  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
777  PGOOpt->Action != PGOOptions::SampleUse)
778  LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
779  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
781 
782  for (auto &C : LoopOptimizerEndEPCallbacks)
783  C(LPM2, Level);
784 
785  // We provide the opt remark emitter pass for LICM to use. We only need to do
786  // this once as it is immutable.
787  FPM.addPass(
791  /*UseBlockFrequencyInfo=*/true));
792  FPM.addPass(SimplifyCFGPass());
793  FPM.addPass(InstCombinePass());
794  if (EnableLoopFlatten)
796  // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
797  // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
798  // *All* loop passes must preserve it, in order to be able to use it.
800  /*UseMemorySSA=*/false,
801  /*UseBlockFrequencyInfo=*/false));
802 
803  // Delete small array after loop unroll.
804  FPM.addPass(SROA());
805 
806  // Eliminate redundancies.
808  if (RunNewGVN)
809  FPM.addPass(NewGVNPass());
810  else
811  FPM.addPass(GVN());
812 
813  // Sparse conditional constant propagation.
814  // FIXME: It isn't clear why we do this *after* loop passes rather than
815  // before...
816  FPM.addPass(SCCPPass());
817 
818  // Delete dead bit computations (instcombine runs after to fold away the dead
819  // computations, and then ADCE will run later to exploit any new DCE
820  // opportunities that creates).
821  FPM.addPass(BDCEPass());
822 
823  // Run instcombine after redundancy and dead bit elimination to exploit
824  // opportunities opened up by them.
825  FPM.addPass(InstCombinePass());
826  invokePeepholeEPCallbacks(FPM, Level);
827 
828  // Re-consider control flow based optimizations after redundancy elimination,
829  // redo DCE, etc.
830  FPM.addPass(JumpThreadingPass());
832 
833  // Finally, do an expensive DCE pass to catch all the dead code exposed by
834  // the simplifications and basic cleanup after all the simplifications.
835  // TODO: Investigate if this is too expensive.
836  FPM.addPass(ADCEPass());
837 
838  // Specially optimize memory movement as it doesn't look like dataflow in SSA.
839  FPM.addPass(MemCpyOptPass());
840 
841  FPM.addPass(DSEPass());
844  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
845 
846  if (PTO.Coroutines)
847  FPM.addPass(CoroElidePass());
848 
849  for (auto &C : ScalarOptimizerLateEPCallbacks)
850  C(FPM, Level);
851 
853  SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true)));
854  FPM.addPass(InstCombinePass());
855  invokePeepholeEPCallbacks(FPM, Level);
856 
857  if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
858  (PGOOpt->Action == PGOOptions::IRUse ||
859  PGOOpt->Action == PGOOptions::SampleUse))
861 
862  return FPM;
863 }
864 
865 void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
868 }
869 
870 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
872  bool RunProfileGen, bool IsCS,
873  std::string ProfileFile,
874  std::string ProfileRemappingFile) {
875  assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
876  if (!IsCS && !DisablePreInliner) {
878 
879  IP.DefaultThreshold = PreInlineThreshold;
880 
881  // FIXME: The hint threshold has the same value used by the regular inliner
882  // when not optimzing for size. This should probably be lowered after
883  // performance testing.
884  // FIXME: this comment is cargo culted from the old pass manager, revisit).
885  IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
887  CGSCCPassManager &CGPipeline = MIWP.getPM();
888 
890  FPM.addPass(SROA());
891  FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
892  FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
893  FPM.addPass(InstCombinePass()); // Combine silly sequences.
894  invokePeepholeEPCallbacks(FPM, Level);
895 
897 
898  MPM.addPass(std::move(MIWP));
899 
900  // Delete anything that is now dead to make sure that we don't instrument
901  // dead code. Instrumentation can end up keeping dead code around and
902  // dramatically increase code size.
904  }
905 
906  if (!RunProfileGen) {
907  assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
908  MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
909  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
910  // RequireAnalysisPass for PSI before subsequent non-module passes.
912  return;
913  }
914 
915  // Perform PGO instrumentation.
917 
919  // Disable header duplication in loop rotation at -Oz.
922  /*UseBlockFrequencyInfo=*/false));
924 
925  // Add the profile lowering pass.
927  if (!ProfileFile.empty())
928  Options.InstrProfileOutput = ProfileFile;
929  // Do counter promotion at Level greater than O0.
930  Options.DoCounterPromotion = true;
931  Options.UseBFIInPromotion = IsCS;
933 }
934 
936  bool RunProfileGen, bool IsCS,
937  std::string ProfileFile,
938  std::string ProfileRemappingFile) {
939  if (!RunProfileGen) {
940  assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
941  MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
942  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
943  // RequireAnalysisPass for PSI before subsequent non-module passes.
945  return;
946  }
947 
948  // Perform PGO instrumentation.
950  // Add the profile lowering pass.
952  if (!ProfileFile.empty())
953  Options.InstrProfileOutput = ProfileFile;
954  // Do not do counter promotion at O0.
955  Options.DoCounterPromotion = false;
956  Options.UseBFIInPromotion = IsCS;
958 }
959 
960 static InlineParams
962  return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
963 }
964 
969  if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
970  PGOOpt->Action == PGOOptions::SampleUse)
971  IP.HotCallSiteThreshold = 0;
972 
973  if (PGOOpt)
974  IP.EnableDeferral = EnablePGOInlineDeferral;
975 
978 
979  // Require the GlobalsAA analysis for the module so we can query it within
980  // the CGSCC pipeline.
982  // Invalidate AAManager so it can be recreated and pick up the newly available
983  // GlobalsAA.
984  MIWP.addModulePass(
986 
987  // Require the ProfileSummaryAnalysis for the module so we can query it within
988  // the inliner pass.
990 
991  // Now begin the main postorder CGSCC pipeline.
992  // FIXME: The current CGSCC pipeline has its origins in the legacy pass
993  // manager and trying to emulate its precise behavior. Much of this doesn't
994  // make a lot of sense and we should revisit the core CGSCC structure.
995  CGSCCPassManager &MainCGPipeline = MIWP.getPM();
996 
997  // Note: historically, the PruneEH pass was run first to deduce nounwind and
998  // generally clean up exception handling overhead. It isn't clear this is
999  // valuable as the inliner doesn't currently care whether it is inlining an
1000  // invoke or a call.
1001 
1003  MainCGPipeline.addPass(AttributorCGSCCPass());
1004 
1005  if (PTO.Coroutines)
1006  MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
1007 
1008  // Now deduce any function attributes based in the current code.
1009  MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
1010 
1011  // When at O3 add argument promotion to the pass pipeline.
1012  // FIXME: It isn't at all clear why this should be limited to O3.
1014  MainCGPipeline.addPass(ArgumentPromotionPass());
1015 
1016  // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1017  // there are no OpenMP runtime calls present in the module.
1019  MainCGPipeline.addPass(OpenMPOptCGSCCPass());
1020 
1021  for (auto &C : CGSCCOptimizerLateEPCallbacks)
1022  C(MainCGPipeline, Level);
1023 
1024  // Lastly, add the core function simplification pipeline nested inside the
1025  // CGSCC walk.
1028 
1029  return MIWP;
1030 }
1031 
1036 
1037  // Place pseudo probe instrumentation as the first pass of the pipeline to
1038  // minimize the impact of optimization changes.
1039  if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1042 
1043  bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1044 
1045  // In ThinLTO mode, when flattened profile is used, all the available
1046  // profile information will be annotated in PreLink phase so there is
1047  // no need to load the profile again in PostLink.
1048  bool LoadSampleProfile =
1049  HasSampleProfile &&
1051 
1052  // During the ThinLTO backend phase we perform early indirect call promotion
1053  // here, before globalopt. Otherwise imported available_externally functions
1054  // look unreferenced and are removed. If we are going to load the sample
1055  // profile then defer until later.
1056  // TODO: See if we can move later and consolidate with the location where
1057  // we perform ICP when we are loading a sample profile.
1058  // TODO: We pass HasSampleProfile (whether there was a sample profile file
1059  // passed to the compile) to the SamplePGO flag of ICP. This is used to
1060  // determine whether the new direct calls are annotated with prof metadata.
1061  // Ideally this should be determined from whether the IR is annotated with
1062  // sample profile, and not whether the a sample profile was provided on the
1063  // command line. E.g. for flattened profiles where we will not be reloading
1064  // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1065  // provide the sample profile file.
1066  if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
1067  MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1068 
1069  // Do basic inference of function attributes from known properties of system
1070  // libraries and other oracles.
1072 
1073  // Create an early function pass manager to cleanup the output of the
1074  // frontend.
1075  FunctionPassManager EarlyFPM;
1076  // Lower llvm.expect to metadata before attempting transforms.
1077  // Compare/branch metadata may alter the behavior of passes like SimplifyCFG.
1078  EarlyFPM.addPass(LowerExpectIntrinsicPass());
1079  EarlyFPM.addPass(SimplifyCFGPass());
1080  EarlyFPM.addPass(SROA());
1081  EarlyFPM.addPass(EarlyCSEPass());
1082  if (PTO.Coroutines)
1083  EarlyFPM.addPass(CoroEarlyPass());
1085  EarlyFPM.addPass(CallSiteSplittingPass());
1086 
1087  // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
1088  // to convert bitcast to direct calls so that they can be inlined during the
1089  // profile annotation prepration step.
1090  // More details about SamplePGO design can be found in:
1091  // https://research.google.com/pubs/pub45290.html
1092  // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
1093  if (LoadSampleProfile)
1094  EarlyFPM.addPass(InstCombinePass());
1096 
1097  if (LoadSampleProfile) {
1098  // Annotate sample profile right after early FPM to ensure freshness of
1099  // the debug info.
1100  MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1101  PGOOpt->ProfileRemappingFile, Phase));
1102  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1103  // RequireAnalysisPass for PSI before subsequent non-module passes.
1105  // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1106  // for the profile annotation to be accurate in the LTO backend.
1109  // We perform early indirect call promotion here, before globalopt.
1110  // This is important for the ThinLTO backend phase because otherwise
1111  // imported available_externally functions look unreferenced and are
1112  // removed.
1113  MPM.addPass(
1114  PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1115  }
1116 
1117  // Try to perform OpenMP specific optimizations on the module. This is a
1118  // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1121 
1124 
1125  // Lower type metadata and the type.test intrinsic in the ThinLTO
1126  // post link pipeline after ICP. This is to enable usage of the type
1127  // tests in ICP sequences.
1129  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1130 
1131  for (auto &C : PipelineEarlySimplificationEPCallbacks)
1132  C(MPM, Level);
1133 
1134  // Interprocedural constant propagation now that basic cleanup has occurred
1135  // and prior to optimizing globals.
1136  // FIXME: This position in the pipeline hasn't been carefully considered in
1137  // years, it should be re-analyzed.
1138  MPM.addPass(IPSCCPPass());
1139 
1140  // Attach metadata to indirect call sites indicating the set of functions
1141  // they may target at run-time. This should follow IPSCCP.
1143 
1144  // Optimize globals to try and fold them into constants.
1146 
1147  // Promote any localized globals to SSA registers.
1148  // FIXME: Should this instead by a run of SROA?
1149  // FIXME: We should probably run instcombine and simplify-cfg afterward to
1150  // delete control flows that are dead once globals have been folded to
1151  // constants.
1153 
1154  // Remove any dead arguments exposed by cleanups and constant folding
1155  // globals.
1157 
1158  // Create a small function pass pipeline to cleanup after all the global
1159  // optimizations.
1160  FunctionPassManager GlobalCleanupPM;
1161  GlobalCleanupPM.addPass(InstCombinePass());
1162  invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1163 
1164  GlobalCleanupPM.addPass(SimplifyCFGPass());
1166 
1167  // Add all the requested passes for instrumentation PGO, if requested.
1168  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1169  (PGOOpt->Action == PGOOptions::IRInstr ||
1170  PGOOpt->Action == PGOOptions::IRUse)) {
1171  addPGOInstrPasses(MPM, Level,
1172  /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1173  /* IsCS */ false, PGOOpt->ProfileFile,
1174  PGOOpt->ProfileRemappingFile);
1175  MPM.addPass(PGOIndirectCallPromotion(false, false));
1176  }
1177  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1178  PGOOpt->CSAction == PGOOptions::CSIRInstr)
1179  MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1180 
1181  // Synthesize function entry counts for non-PGO compilation.
1182  if (EnableSyntheticCounts && !PGOOpt)
1184 
1186 
1190  }
1191 
1192  return MPM;
1193 }
1194 
1195 /// TODO: Should LTO cause any differences to this set of passes?
1196 void PassBuilder::addVectorPasses(OptimizationLevel Level,
1197  FunctionPassManager &FPM, bool IsLTO) {
1200 
1201  if (IsLTO) {
1202  // The vectorizer may have significantly shortened a loop body; unroll
1203  // again. Unroll small loops to hide loop backedge latency and saturate any
1204  // parallel execution resources of an out-of-order processor. We also then
1205  // need to clean up redundancies and loop invariant code.
1206  // FIXME: It would be really good to use a loop-integrated instruction
1207  // combiner for cleanup here so that the unrolling and LICM can be pipelined
1208  // across the loop nests.
1209  // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1210  if (EnableUnrollAndJam && PTO.LoopUnrolling)
1211  FPM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1213  Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1216  }
1217 
1218  if (!IsLTO) {
1219  // Eliminate loads by forwarding stores from the previous iteration to loads
1220  // of the current iteration.
1222  }
1223  // Cleanup after the loop optimization passes.
1224  FPM.addPass(InstCombinePass());
1225 
1226  if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1227  // At higher optimization levels, try to clean up any runtime overlap and
1228  // alignment checks inserted by the vectorizer. We want to track correlated
1229  // runtime checks for two inner loops in the same outer loop, fold any
1230  // common computations, hoist loop-invariant aspects out of any outer loop,
1231  // and unswitch the runtime checks if possible. Once hoisted, we may have
1232  // dead (or speculatable) control flows or more combining opportunities.
1233  FPM.addPass(EarlyCSEPass());
1235  FPM.addPass(InstCombinePass());
1236  LoopPassManager LPM;
1238  LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1240  FPM.addPass(
1244  /*UseBlockFrequencyInfo=*/true));
1245  FPM.addPass(SimplifyCFGPass());
1246  FPM.addPass(InstCombinePass());
1247  }
1248 
1249  // Now that we've formed fast to execute loop structures, we do further
1250  // optimizations. These are run afterward as they might block doing complex
1251  // analyses and transforms such as what are needed for loop vectorization.
1252 
1253  // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1254  // GVN, loop transforms, and others have already run, so it's now better to
1255  // convert to more optimized IR using more aggressive simplify CFG options.
1256  // The extra sinking transform can create larger basic blocks, so do this
1257  // before SLP vectorization.
1259  .forwardSwitchCondToPhi(true)
1260  .convertSwitchToLookupTable(true)
1261  .needCanonicalLoops(false)
1262  .hoistCommonInsts(true)
1263  .sinkCommonInsts(true)));
1264 
1265  if (IsLTO) {
1266  FPM.addPass(SCCPPass());
1267  FPM.addPass(InstCombinePass());
1268  FPM.addPass(BDCEPass());
1269  }
1270 
1271  // Optimize parallel scalar instruction chains into SIMD instructions.
1272  if (PTO.SLPVectorization) {
1273  FPM.addPass(SLPVectorizerPass());
1274  if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1275  FPM.addPass(EarlyCSEPass());
1276  }
1277  }
1278  // Enhance/cleanup vector code.
1279  FPM.addPass(VectorCombinePass());
1280 
1281  if (!IsLTO) {
1282  FPM.addPass(InstCombinePass());
1283  // Unroll small loops to hide loop backedge latency and saturate any
1284  // parallel execution resources of an out-of-order processor. We also then
1285  // need to clean up redundancies and loop invariant code.
1286  // FIXME: It would be really good to use a loop-integrated instruction
1287  // combiner for cleanup here so that the unrolling and LICM can be pipelined
1288  // across the loop nests.
1289  // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1290  if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1291  FPM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1292  }
1294  Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1297  FPM.addPass(InstCombinePass());
1298  FPM.addPass(
1302  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
1303  }
1304 
1305  // Now that we've vectorized and unrolled loops, we may have more refined
1306  // alignment information, try to re-derive it here.
1308 
1309  if (IsLTO)
1310  FPM.addPass(InstCombinePass());
1311 }
1312 
1315  bool LTOPreLink) {
1317 
1318  // Optimize globals now that the module is fully simplified.
1321 
1322  // Run partial inlining pass to partially inline functions that have
1323  // large bodies.
1324  if (RunPartialInlining)
1326 
1327  // Remove avail extern fns and globals definitions since we aren't compiling
1328  // an object file for later LTO. For LTO we want to preserve these so they
1329  // are eligible for inlining at link-time. Note if they are unreferenced they
1330  // will be removed by GlobalDCE later, so this only impacts referenced
1331  // available externally globals. Eventually they will be suppressed during
1332  // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1333  // may make globals referenced by available external functions dead and saves
1334  // running remaining passes on the eliminated functions. These should be
1335  // preserved during prelinking for link-time inlining decisions.
1336  if (!LTOPreLink)
1338 
1341 
1342  // Do RPO function attribute inference across the module to forward-propagate
1343  // attributes where applicable.
1344  // FIXME: Is this really an optimization rather than a canonicalization?
1346 
1347  // Do a post inline PGO instrumentation and use pass. This is a context
1348  // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1349  // cross-module inline has not been done yet. The context sensitive
1350  // instrumentation is after all the inlines are done.
1351  if (!LTOPreLink && PGOOpt) {
1352  if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1353  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1354  /* IsCS */ true, PGOOpt->CSProfileGenFile,
1355  PGOOpt->ProfileRemappingFile);
1356  else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1357  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1358  /* IsCS */ true, PGOOpt->ProfileFile,
1359  PGOOpt->ProfileRemappingFile);
1360  }
1361 
1362  // Re-require GloblasAA here prior to function passes. This is particularly
1363  // useful as the above will have inlined, DCE'ed, and function-attr
1364  // propagated everything. We should at this point have a reasonably minimal
1365  // and richly annotated call graph. By computing aliasing and mod/ref
1366  // information for all local globals here, the late loop passes and notably
1367  // the vectorizer will be able to use them to help recognize vectorizable
1368  // memory operations.
1370 
1371  FunctionPassManager OptimizePM;
1372  OptimizePM.addPass(Float2IntPass());
1373  OptimizePM.addPass(LowerConstantIntrinsicsPass());
1374 
1375  if (EnableMatrix) {
1376  OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1377  OptimizePM.addPass(EarlyCSEPass());
1378  }
1379 
1380  // FIXME: We need to run some loop optimizations to re-rotate loops after
1381  // simplify-cfg and others undo their rotation.
1382 
1383  // Optimize the loop execution. These passes operate on entire loop nests
1384  // rather than on each loop in an inside-out manner, and so they are actually
1385  // function passes.
1386 
1387  for (auto &C : VectorizerStartEPCallbacks)
1388  C(OptimizePM, Level);
1389 
1390  // First rotate loops that may have been un-rotated by prior passes.
1391  // Disable header duplication at -Oz.
1393  LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink),
1395  /*UseBlockFrequencyInfo=*/false));
1396 
1397  // Distribute loops to allow partial vectorization. I.e. isolate dependences
1398  // into separate loop that would otherwise inhibit vectorization. This is
1399  // currently only performed for loops marked with the metadata
1400  // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1401  OptimizePM.addPass(LoopDistributePass());
1402 
1403  // Populates the VFABI attribute with the scalar-to-vector mappings
1404  // from the TargetLibraryInfo.
1405  OptimizePM.addPass(InjectTLIMappings());
1406 
1407  addVectorPasses(Level, OptimizePM, /* IsLTO */ false);
1408 
1409  // Split out cold code. Splitting is done late to avoid hiding context from
1410  // other optimizations and inadvertently regressing performance. The tradeoff
1411  // is that this has a higher code size cost than splitting early.
1412  if (EnableHotColdSplit && !LTOPreLink)
1414 
1415  // Search the code for similar regions of code. If enough similar regions can
1416  // be found where extracting the regions into their own function will decrease
1417  // the size of the program, we extract the regions, a deduplicate the
1418  // structurally similar regions.
1419  if (EnableIROutliner)
1421 
1422  // Merge functions if requested.
1423  if (PTO.MergeFunctions)
1425 
1426  // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1427  // canonicalization pass that enables other optimizations. As a result,
1428  // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1429  // result too early.
1430  OptimizePM.addPass(LoopSinkPass());
1431 
1432  // And finally clean up LCSSA form before generating code.
1433  OptimizePM.addPass(InstSimplifyPass());
1434 
1435  // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1436  // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1437  // flattening of blocks.
1438  OptimizePM.addPass(DivRemPairsPass());
1439 
1440  // LoopSink (and other loop passes since the last simplifyCFG) might have
1441  // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1442  OptimizePM.addPass(SimplifyCFGPass());
1443 
1444  // Optimize PHIs by speculating around them when profitable. Note that this
1445  // pass needs to be run after any PRE or similar pass as it is essentially
1446  // inserting redundancies into the program. This even includes SimplifyCFG.
1447  OptimizePM.addPass(SpeculateAroundPHIsPass());
1448 
1449  if (PTO.Coroutines)
1450  OptimizePM.addPass(CoroCleanupPass());
1451 
1452  // Add the core optimizing pipeline.
1454 
1455  for (auto &C : OptimizerLastEPCallbacks)
1456  C(MPM, Level);
1457 
1458  if (PTO.CallGraphProfile)
1460 
1461  // Now we need to do some global optimization transforms.
1462  // FIXME: It would seem like these should come first in the optimization
1463  // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1464  // ordering here.
1467 
1468  // TODO: Relative look table converter pass caused an issue when full lto is
1469  // enabled. See https://reviews.llvm.org/D94355 for more details.
1470  // Until the issue fixed, disable this pass during pre-linking phase.
1471  if (!LTOPreLink)
1473 
1474  return MPM;
1475 }
1476 
1479  bool LTOPreLink) {
1481  "Must request optimizations for the default pipeline!");
1482 
1484 
1485  // Convert @llvm.global.annotations to !annotation metadata.
1487 
1488  // Force any function attributes we want the rest of the pipeline to observe.
1490 
1491  // Apply module pipeline start EP callback.
1492  for (auto &C : PipelineStartEPCallbacks)
1493  C(MPM, Level);
1494 
1495  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1497 
1498  // Add the core simplification pipeline.
1502 
1503  // Now add the optimization pipeline.
1505 
1506  if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1508 
1509  // Emit annotation remarks.
1511 
1512  if (LTOPreLink)
1513  addRequiredLTOPreLinkPasses(MPM);
1514 
1515  return MPM;
1516 }
1517 
1521  "Must request optimizations for the default pipeline!");
1522 
1524 
1525  // Convert @llvm.global.annotations to !annotation metadata.
1527 
1528  // Force any function attributes we want the rest of the pipeline to observe.
1530 
1531  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1533 
1534  // Apply module pipeline start EP callback.
1535  for (auto &C : PipelineStartEPCallbacks)
1536  C(MPM, Level);
1537 
1538  // If we are planning to perform ThinLTO later, we don't bloat the code with
1539  // unrolling/vectorization/... now. Just simplify the module as much as we
1540  // can.
1543 
1544  // Run partial inlining pass to partially inline functions that have
1545  // large bodies.
1546  // FIXME: It isn't clear whether this is really the right place to run this
1547  // in ThinLTO. Because there is another canonicalization and simplification
1548  // phase that will run after the thin link, running this here ends up with
1549  // less information than will be available later and it may grow functions in
1550  // ways that aren't beneficial.
1551  if (RunPartialInlining)
1553 
1554  // Reduce the size of the IR as much as possible.
1556 
1557  // Module simplification splits coroutines, but does not fully clean up
1558  // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1559  // on these, we schedule the cleanup here.
1560  if (PTO.Coroutines)
1562 
1563  if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1565 
1566  // Handle OptimizerLastEPCallbacks added by clang on PreLink. Actual
1567  // optimization is going to be done in PostLink stage, but clang can't
1568  // add callbacks there in case of in-process ThinLTO called by linker.
1569  for (auto &C : OptimizerLastEPCallbacks)
1570  C(MPM, Level);
1571 
1572  // Emit annotation remarks.
1574 
1575  addRequiredLTOPreLinkPasses(MPM);
1576 
1577  return MPM;
1578 }
1579 
1581  OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1583 
1584  // Convert @llvm.global.annotations to !annotation metadata.
1586 
1587  if (ImportSummary) {
1588  // These passes import type identifier resolutions for whole-program
1589  // devirtualization and CFI. They must run early because other passes may
1590  // disturb the specific instruction patterns that these passes look for,
1591  // creating dependencies on resolutions that may not appear in the summary.
1592  //
1593  // For example, GVN may transform the pattern assume(type.test) appearing in
1594  // two basic blocks into assume(phi(type.test, type.test)), which would
1595  // transform a dependency on a WPD resolution into a dependency on a type
1596  // identifier resolution for CFI.
1597  //
1598  // Also, WPD has access to more precise information than ICP and can
1599  // devirtualize more effectively, so it should operate on the IR first.
1600  //
1601  // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1602  // metadata and intrinsics.
1603  MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1604  MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1605  }
1606 
1607  if (Level == OptimizationLevel::O0) {
1608  // Run a second time to clean up any type tests left behind by WPD for use
1609  // in ICP.
1610  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1611  // Drop available_externally and unreferenced globals. This is necessary
1612  // with ThinLTO in order to avoid leaving undefined references to dead
1613  // globals in the object file.
1616  return MPM;
1617  }
1618 
1619  // Force any function attributes we want the rest of the pipeline to observe.
1621 
1622  // Add the core simplification pipeline.
1625 
1626  // Now add the optimization pipeline.
1628 
1629  // Emit annotation remarks.
1631 
1632  return MPM;
1633 }
1634 
1638  "Must request optimizations for the default pipeline!");
1639  // FIXME: We should use a customized pre-link pipeline!
1641  /* LTOPreLink */ true);
1642 }
1643 
1646  ModuleSummaryIndex *ExportSummary) {
1648 
1649  // Convert @llvm.global.annotations to !annotation metadata.
1651 
1652  if (Level == OptimizationLevel::O0) {
1653  // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1654  // metadata and intrinsics.
1655  MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1656  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1657  // Run a second time to clean up any type tests left behind by WPD for use
1658  // in ICP.
1659  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1660 
1661  // Emit annotation remarks.
1663 
1664  return MPM;
1665  }
1666 
1667  if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1668  // Load sample profile before running the LTO optimization pipeline.
1669  MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1670  PGOOpt->ProfileRemappingFile,
1672  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1673  // RequireAnalysisPass for PSI before subsequent non-module passes.
1675  }
1676 
1677  // Remove unused virtual tables to improve the quality of code generated by
1678  // whole-program devirtualization and bitset lowering.
1680 
1681  // Force any function attributes we want the rest of the pipeline to observe.
1683 
1684  // Do basic inference of function attributes from known properties of system
1685  // libraries and other oracles.
1687 
1688  if (Level.getSpeedupLevel() > 1) {
1689  FunctionPassManager EarlyFPM;
1690  EarlyFPM.addPass(CallSiteSplittingPass());
1692 
1693  // Indirect call promotion. This should promote all the targets that are
1694  // left by the earlier promotion pass that promotes intra-module targets.
1695  // This two-step promotion is to save the compile time. For LTO, it should
1696  // produce the same result as if we only do promotion here.
1698  true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1699  // Propagate constants at call sites into the functions they call. This
1700  // opens opportunities for globalopt (and inlining) by substituting function
1701  // pointers passed as arguments to direct uses of functions.
1702  MPM.addPass(IPSCCPPass());
1703 
1704  // Attach metadata to indirect call sites indicating the set of functions
1705  // they may target at run-time. This should follow IPSCCP.
1707  }
1708 
1709  // Now deduce any function attributes based in the current code.
1712 
1713  // Do RPO function attribute inference across the module to forward-propagate
1714  // attributes where applicable.
1715  // FIXME: Is this really an optimization rather than a canonicalization?
1717 
1718  // Use in-range annotations on GEP indices to split globals where beneficial.
1720 
1721  // Run whole program optimization of virtual call when the list of callees
1722  // is fixed.
1723  MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1724 
1725  // Stop here at -O1.
1726  if (Level == OptimizationLevel::O1) {
1727  // The LowerTypeTestsPass needs to run to lower type metadata and the
1728  // type.test intrinsics. The pass does nothing if CFI is disabled.
1729  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1730  // Run a second time to clean up any type tests left behind by WPD for use
1731  // in ICP (which is performed earlier than this in the regular LTO
1732  // pipeline).
1733  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1734 
1735  // Emit annotation remarks.
1737 
1738  return MPM;
1739  }
1740 
1741  // Optimize globals to try and fold them into constants.
1743 
1744  // Promote any localized globals to SSA registers.
1746 
1747  // Linking modules together can lead to duplicate global constant, only
1748  // keep one copy of each constant.
1750 
1751  // Remove unused arguments from functions.
1753 
1754  // Reduce the code after globalopt and ipsccp. Both can open up significant
1755  // simplification opportunities, and both can propagate functions through
1756  // function pointers. When this happens, we often have to resolve varargs
1757  // calls, etc, so let instcombine do this.
1758  FunctionPassManager PeepholeFPM;
1760  PeepholeFPM.addPass(AggressiveInstCombinePass());
1761  PeepholeFPM.addPass(InstCombinePass());
1762  invokePeepholeEPCallbacks(PeepholeFPM, Level);
1763 
1765 
1766  // Note: historically, the PruneEH pass was run first to deduce nounwind and
1767  // generally clean up exception handling overhead. It isn't clear this is
1768  // valuable as the inliner doesn't currently care whether it is inlining an
1769  // invoke or a call.
1770  // Run the inliner now.
1772 
1773  // Optimize globals again after we ran the inliner.
1775 
1776  // Garbage collect dead functions.
1777  // FIXME: Add ArgumentPromotion pass after once it's ported.
1779 
1780  FunctionPassManager FPM;
1781  // The IPO Passes may leave cruft around. Clean up after them.
1782  FPM.addPass(InstCombinePass());
1783  invokePeepholeEPCallbacks(FPM, Level);
1784 
1785  FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1786 
1787  // Do a post inline PGO instrumentation and use pass. This is a context
1788  // sensitive PGO pass.
1789  if (PGOOpt) {
1790  if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1791  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1792  /* IsCS */ true, PGOOpt->CSProfileGenFile,
1793  PGOOpt->ProfileRemappingFile);
1794  else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1795  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1796  /* IsCS */ true, PGOOpt->ProfileFile,
1797  PGOOpt->ProfileRemappingFile);
1798  }
1799 
1800  // Break up allocas
1801  FPM.addPass(SROA());
1802 
1803  // LTO provides additional opportunities for tailcall elimination due to
1804  // link-time inlining, and visibility of nocapture attribute.
1805  FPM.addPass(TailCallElimPass());
1806 
1807  // Run a few AA driver optimizations here and now to cleanup the code.
1809 
1810  MPM.addPass(
1812 
1813  // Require the GlobalsAA analysis for the module so we can query it within
1814  // MainFPM.
1816  // Invalidate AAManager so it can be recreated and pick up the newly available
1817  // GlobalsAA.
1818  MPM.addPass(
1820 
1821  FunctionPassManager MainFPM;
1824  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true));
1825 
1826  if (RunNewGVN)
1827  MainFPM.addPass(NewGVNPass());
1828  else
1829  MainFPM.addPass(GVN());
1830 
1831  // Remove dead memcpy()'s.
1832  MainFPM.addPass(MemCpyOptPass());
1833 
1834  // Nuke dead stores.
1835  MainFPM.addPass(DSEPass());
1837 
1838  // More loops are countable; try to optimize them.
1839  if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
1841 
1844 
1845  LoopPassManager LPM;
1846  LPM.addPass(IndVarSimplifyPass());
1847  LPM.addPass(LoopDeletionPass());
1848  // FIXME: Add loop interchange.
1849 
1850  // Unroll small loops and perform peeling.
1851  LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
1852  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
1854  // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
1855  // *All* loop passes must preserve it, in order to be able to use it.
1857  std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/true));
1858 
1859  MainFPM.addPass(LoopDistributePass());
1860 
1861  addVectorPasses(Level, MainFPM, /* IsLTO */ true);
1862 
1863  invokePeepholeEPCallbacks(MainFPM, Level);
1864  MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1866 
1867  // Create a function that performs CFI checks for cross-DSO calls with
1868  // targets in the current module.
1870 
1871  // Lower type metadata and the type.test intrinsic. This pass supports
1872  // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1873  // to be run at link time if CFI is enabled. This pass does nothing if
1874  // CFI is disabled.
1875  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1876  // Run a second time to clean up any type tests left behind by WPD for use
1877  // in ICP (which is performed earlier than this in the regular LTO pipeline).
1878  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1879 
1880  // Enable splitting late in the FullLTO post-link pipeline. This is done in
1881  // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1882  if (EnableHotColdSplit)
1884 
1885  // Add late LTO optimization passes.
1886  // Delete basic blocks, which optimization passes may have killed.
1888  SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))));
1889 
1890  // Drop bodies of available eternally objects to improve GlobalDCE.
1892 
1893  // Now that we have optimized the program, discard unreachable functions.
1895 
1896  if (PTO.MergeFunctions)
1898 
1899  // Emit annotation remarks.
1901 
1902  return MPM;
1903 }
1904 
1906  bool LTOPreLink) {
1908  "buildO0DefaultPipeline should only be used with O0");
1909 
1911 
1912  if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
1913  PGOOpt->Action == PGOOptions::IRUse))
1915  MPM,
1916  /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1917  /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1918 
1919  for (auto &C : PipelineStartEPCallbacks)
1920  C(MPM, Level);
1921  for (auto &C : PipelineEarlySimplificationEPCallbacks)
1922  C(MPM, Level);
1923 
1924  // Build a minimal pipeline based on the semantics required by LLVM,
1925  // which is just that always inlining occurs. Further, disable generating
1926  // lifetime intrinsics to avoid enabling further optimizations during
1927  // code generation.
1928  // However, we need to insert lifetime intrinsics to avoid invalid access
1929  // caused by multithreaded coroutines.
1931  /*InsertLifetimeIntrinsics=*/PTO.Coroutines));
1932 
1933  if (PTO.MergeFunctions)
1935 
1936  if (EnableMatrix)
1937  MPM.addPass(
1939 
1940  if (!CGSCCOptimizerLateEPCallbacks.empty()) {
1941  CGSCCPassManager CGPM;
1942  for (auto &C : CGSCCOptimizerLateEPCallbacks)
1943  C(CGPM, Level);
1944  if (!CGPM.isEmpty())
1946  }
1947  if (!LateLoopOptimizationsEPCallbacks.empty()) {
1948  LoopPassManager LPM;
1949  for (auto &C : LateLoopOptimizationsEPCallbacks)
1950  C(LPM, Level);
1951  if (!LPM.isEmpty()) {
1954  }
1955  }
1956  if (!LoopOptimizerEndEPCallbacks.empty()) {
1957  LoopPassManager LPM;
1958  for (auto &C : LoopOptimizerEndEPCallbacks)
1959  C(LPM, Level);
1960  if (!LPM.isEmpty()) {
1963  }
1964  }
1965  if (!ScalarOptimizerLateEPCallbacks.empty()) {
1966  FunctionPassManager FPM;
1967  for (auto &C : ScalarOptimizerLateEPCallbacks)
1968  C(FPM, Level);
1969  if (!FPM.isEmpty())
1971  }
1972  if (!VectorizerStartEPCallbacks.empty()) {
1973  FunctionPassManager FPM;
1974  for (auto &C : VectorizerStartEPCallbacks)
1975  C(FPM, Level);
1976  if (!FPM.isEmpty())
1978  }
1979 
1980  if (PTO.Coroutines) {
1982 
1983  CGSCCPassManager CGPM;
1984  CGPM.addPass(CoroSplitPass());
1987 
1989  }
1990 
1991  for (auto &C : OptimizerLastEPCallbacks)
1992  C(MPM, Level);
1993 
1994  if (LTOPreLink)
1995  addRequiredLTOPreLinkPasses(MPM);
1996 
1997  return MPM;
1998 }
1999 
2001  AAManager AA;
2002 
2003  // The order in which these are registered determines their priority when
2004  // being queried.
2005 
2006  // First we register the basic alias analysis that provides the majority of
2007  // per-function local AA logic. This is a stateless, on-demand local set of
2008  // AA techniques.
2010 
2011  // Next we query fast, specialized alias analyses that wrap IR-embedded
2012  // information about aliasing.
2015 
2016  // Add support for querying global aliasing information when available.
2017  // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2018  // analysis, all that the `AAManager` can do is query for any *cached*
2019  // results from `GlobalsAA` through a readonly proxy.
2021 
2022  // Add target-specific alias analyses.
2023  if (TM)
2025 
2026  return AA;
2027 }
2028 
2030  if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
2031  return None;
2032  int Count;
2033  if (Name.getAsInteger(0, Count) || Count <= 0)
2034  return None;
2035  return Count;
2036 }
2037 
2039  if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
2040  return None;
2041  int Count;
2042  if (Name.getAsInteger(0, Count) || Count < 0)
2043  return None;
2044  return Count;
2045 }
2046 
2048  if (!Name.consume_front(PassName))
2049  return false;
2050  // normal pass name w/o parameters == default parameters
2051  if (Name.empty())
2052  return true;
2053  return Name.startswith("<") && Name.endswith(">");
2054 }
2055 
2056 namespace {
2057 
2058 /// This performs customized parsing of pass name with parameters.
2059 ///
2060 /// We do not need parametrization of passes in textual pipeline very often,
2061 /// yet on a rare occasion ability to specify parameters right there can be
2062 /// useful.
2063 ///
2064 /// \p Name - parameterized specification of a pass from a textual pipeline
2065 /// is a string in a form of :
2066 /// PassName '<' parameter-list '>'
2067 ///
2068 /// Parameter list is being parsed by the parser callable argument, \p Parser,
2069 /// It takes a string-ref of parameters and returns either StringError or a
2070 /// parameter list in a form of a custom parameters type, all wrapped into
2071 /// Expected<> template class.
2072 ///
2073 template <typename ParametersParseCallableT>
2074 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
2075  StringRef PassName) -> decltype(Parser(StringRef{})) {
2076  using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
2077 
2078  StringRef Params = Name;
2079  if (!Params.consume_front(PassName)) {
2080  assert(false &&
2081  "unable to strip pass name from parametrized pass specification");
2082  }
2083  if (Params.empty())
2084  return ParametersT{};
2085  if (!Params.consume_front("<") || !Params.consume_back(">")) {
2086  assert(false && "invalid format for parametrized pass name");
2087  }
2088 
2089  Expected<ParametersT> Result = Parser(Params);
2090  assert((Result || Result.template errorIsA<StringError>()) &&
2091  "Pass parameter parser can only return StringErrors.");
2092  return Result;
2093 }
2094 
2095 /// Parser of parameters for LoopUnroll pass.
2096 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
2097  LoopUnrollOptions UnrollOpts;
2098  while (!Params.empty()) {
2099  StringRef ParamName;
2100  std::tie(ParamName, Params) = Params.split(';');
2101  int OptLevel = StringSwitch<int>(ParamName)
2102  .Case("O0", 0)
2103  .Case("O1", 1)
2104  .Case("O2", 2)
2105  .Case("O3", 3)
2106  .Default(-1);
2107  if (OptLevel >= 0) {
2108  UnrollOpts.setOptLevel(OptLevel);
2109  continue;
2110  }
2111  if (ParamName.consume_front("full-unroll-max=")) {
2112  int Count;
2113  if (ParamName.getAsInteger(0, Count))
2114  return make_error<StringError>(
2115  formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2117  UnrollOpts.setFullUnrollMaxCount(Count);
2118  continue;
2119  }
2120 
2121  bool Enable = !ParamName.consume_front("no-");
2122  if (ParamName == "partial") {
2123  UnrollOpts.setPartial(Enable);
2124  } else if (ParamName == "peeling") {
2125  UnrollOpts.setPeeling(Enable);
2126  } else if (ParamName == "profile-peeling") {
2127  UnrollOpts.setProfileBasedPeeling(Enable);
2128  } else if (ParamName == "runtime") {
2129  UnrollOpts.setRuntime(Enable);
2130  } else if (ParamName == "upperbound") {
2131  UnrollOpts.setUpperBound(Enable);
2132  } else {
2133  return make_error<StringError>(
2134  formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2136  }
2137  }
2138  return UnrollOpts;
2139 }
2140 
2141 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
2143  while (!Params.empty()) {
2144  StringRef ParamName;
2145  std::tie(ParamName, Params) = Params.split(';');
2146 
2147  if (ParamName == "recover") {
2148  Result.Recover = true;
2149  } else if (ParamName == "kernel") {
2150  Result.Kernel = true;
2151  } else if (ParamName.consume_front("track-origins=")) {
2152  if (ParamName.getAsInteger(0, Result.TrackOrigins))
2153  return make_error<StringError>(
2154  formatv("invalid argument to MemorySanitizer pass track-origins "
2155  "parameter: '{0}' ",
2156  ParamName)
2157  .str(),
2159  } else {
2160  return make_error<StringError>(
2161  formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
2162  .str(),
2164  }
2165  }
2166  return Result;
2167 }
2168 
2169 /// Parser of parameters for SimplifyCFG pass.
2170 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
2172  while (!Params.empty()) {
2173  StringRef ParamName;
2174  std::tie(ParamName, Params) = Params.split(';');
2175 
2176  bool Enable = !ParamName.consume_front("no-");
2177  if (ParamName == "forward-switch-cond") {
2178  Result.forwardSwitchCondToPhi(Enable);
2179  } else if (ParamName == "switch-to-lookup") {
2180  Result.convertSwitchToLookupTable(Enable);
2181  } else if (ParamName == "keep-loops") {
2182  Result.needCanonicalLoops(Enable);
2183  } else if (ParamName == "hoist-common-insts") {
2184  Result.hoistCommonInsts(Enable);
2185  } else if (ParamName == "sink-common-insts") {
2186  Result.sinkCommonInsts(Enable);
2187  } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
2188  APInt BonusInstThreshold;
2189  if (ParamName.getAsInteger(0, BonusInstThreshold))
2190  return make_error<StringError>(
2191  formatv("invalid argument to SimplifyCFG pass bonus-threshold "
2192  "parameter: '{0}' ",
2193  ParamName).str(),
2195  Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
2196  } else {
2197  return make_error<StringError>(
2198  formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
2200  }
2201  }
2202  return Result;
2203 }
2204 
2205 /// Parser of parameters for LoopVectorize pass.
2206 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
2207  LoopVectorizeOptions Opts;
2208  while (!Params.empty()) {
2209  StringRef ParamName;
2210  std::tie(ParamName, Params) = Params.split(';');
2211 
2212  bool Enable = !ParamName.consume_front("no-");
2213  if (ParamName == "interleave-forced-only") {
2214  Opts.setInterleaveOnlyWhenForced(Enable);
2215  } else if (ParamName == "vectorize-forced-only") {
2216  Opts.setVectorizeOnlyWhenForced(Enable);
2217  } else {
2218  return make_error<StringError>(
2219  formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
2221  }
2222  }
2223  return Opts;
2224 }
2225 
2226 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
2227  bool Result = false;
2228  while (!Params.empty()) {
2229  StringRef ParamName;
2230  std::tie(ParamName, Params) = Params.split(';');
2231 
2232  bool Enable = !ParamName.consume_front("no-");
2233  if (ParamName == "nontrivial") {
2234  Result = Enable;
2235  } else {
2236  return make_error<StringError>(
2237  formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
2238  .str(),
2240  }
2241  }
2242  return Result;
2243 }
2244 
2245 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
2246  bool Result = false;
2247  while (!Params.empty()) {
2248  StringRef ParamName;
2249  std::tie(ParamName, Params) = Params.split(';');
2250 
2251  bool Enable = !ParamName.consume_front("no-");
2252  if (ParamName == "split-footer-bb") {
2253  Result = Enable;
2254  } else {
2255  return make_error<StringError>(
2256  formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
2257  ParamName)
2258  .str(),
2260  }
2261  }
2262  return Result;
2263 }
2264 
2265 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
2267  while (!Params.empty()) {
2268  StringRef ParamName;
2269  std::tie(ParamName, Params) = Params.split(';');
2270 
2271  bool Enable = !ParamName.consume_front("no-");
2272  if (ParamName == "pre") {
2273  Result.setPRE(Enable);
2274  } else if (ParamName == "load-pre") {
2275  Result.setLoadPRE(Enable);
2276  } else if (ParamName == "split-backedge-load-pre") {
2277  Result.setLoadPRESplitBackedge(Enable);
2278  } else if (ParamName == "memdep") {
2279  Result.setMemDep(Enable);
2280  } else {
2281  return make_error<StringError>(
2282  formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
2284  }
2285  }
2286  return Result;
2287 }
2288 
2290 parseStackLifetimeOptions(StringRef Params) {
2292  while (!Params.empty()) {
2293  StringRef ParamName;
2294  std::tie(ParamName, Params) = Params.split(';');
2295 
2296  if (ParamName == "may") {
2298  } else if (ParamName == "must") {
2300  } else {
2301  return make_error<StringError>(
2302  formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
2304  }
2305  }
2306  return Result;
2307 }
2308 
2309 } // namespace
2310 
2311 /// Tests whether a pass name starts with a valid prefix for a default pipeline
2312 /// alias.
2314  return Name.startswith("default") || Name.startswith("thinlto") ||
2315  Name.startswith("lto");
2316 }
2317 
2318 /// Tests whether registered callbacks will accept a given pass name.
2319 ///
2320 /// When parsing a pipeline text, the type of the outermost pipeline may be
2321 /// omitted, in which case the type is automatically determined from the first
2322 /// pass name in the text. This may be a name that is handled through one of the
2323 /// callbacks. We check this through the oridinary parsing callbacks by setting
2324 /// up a dummy PassManager in order to not force the client to also handle this
2325 /// type of query.
2326 template <typename PassManagerT, typename CallbacksT>
2327 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
2328  if (!Callbacks.empty()) {
2329  PassManagerT DummyPM;
2330  for (auto &CB : Callbacks)
2331  if (CB(Name, DummyPM, {}))
2332  return true;
2333  }
2334  return false;
2335 }
2336 
2337 template <typename CallbacksT>
2338 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
2339  // Manually handle aliases for pre-configured pipeline fragments.
2341  return DefaultAliasRegex.match(Name);
2342 
2343  // Explicitly handle pass manager names.
2344  if (Name == "module")
2345  return true;
2346  if (Name == "cgscc")
2347  return true;
2348  if (Name == "function")
2349  return true;
2350 
2351  // Explicitly handle custom-parsed pass names.
2353  return true;
2354 
2355 #define MODULE_PASS(NAME, CREATE_PASS) \
2356  if (Name == NAME) \
2357  return true;
2358 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2359  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2360  return true;
2361 #include "PassRegistry.def"
2362 
2363  return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
2364 }
2365 
2366 template <typename CallbacksT>
2367 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2368  // Explicitly handle pass manager names.
2369  if (Name == "cgscc")
2370  return true;
2371  if (Name == "function")
2372  return true;
2373 
2374  // Explicitly handle custom-parsed pass names.
2376  return true;
2378  return true;
2379 
2380 #define CGSCC_PASS(NAME, CREATE_PASS) \
2381  if (Name == NAME) \
2382  return true;
2383 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2384  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2385  return true;
2386 #include "PassRegistry.def"
2387 
2388  return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2389 }
2390 
2391 template <typename CallbacksT>
2392 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2393  // Explicitly handle pass manager names.
2394  if (Name == "function")
2395  return true;
2396  if (Name == "loop" || Name == "loop-mssa")
2397  return true;
2398 
2399  // Explicitly handle custom-parsed pass names.
2401  return true;
2402 
2403 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2404  if (Name == NAME) \
2405  return true;
2406 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2407  if (checkParametrizedPassName(Name, NAME)) \
2408  return true;
2409 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2410  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2411  return true;
2412 #include "PassRegistry.def"
2413 
2414  return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2415 }
2416 
2417 template <typename CallbacksT>
2418 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2419  // Explicitly handle pass manager names.
2420  if (Name == "loop" || Name == "loop-mssa")
2421  return true;
2422 
2423  // Explicitly handle custom-parsed pass names.
2425  return true;
2426 
2427 #define LOOP_PASS(NAME, CREATE_PASS) \
2428  if (Name == NAME) \
2429  return true;
2430 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2431  if (checkParametrizedPassName(Name, NAME)) \
2432  return true;
2433 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2434  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2435  return true;
2436 #include "PassRegistry.def"
2437 
2438  return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2439 }
2440 
2442 PassBuilder::parsePipelineText(StringRef Text) {
2443  std::vector<PipelineElement> ResultPipeline;
2444 
2445  SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2446  &ResultPipeline};
2447  for (;;) {
2448  std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2449  size_t Pos = Text.find_first_of(",()");
2450  Pipeline.push_back({Text.substr(0, Pos), {}});
2451 
2452  // If we have a single terminating name, we're done.
2453  if (Pos == Text.npos)
2454  break;
2455 
2456  char Sep = Text[Pos];
2457  Text = Text.substr(Pos + 1);
2458  if (Sep == ',')
2459  // Just a name ending in a comma, continue.
2460  continue;
2461 
2462  if (Sep == '(') {
2463  // Push the inner pipeline onto the stack to continue processing.
2464  PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2465  continue;
2466  }
2467 
2468  assert(Sep == ')' && "Bogus separator!");
2469  // When handling the close parenthesis, we greedily consume them to avoid
2470  // empty strings in the pipeline.
2471  do {
2472  // If we try to pop the outer pipeline we have unbalanced parentheses.
2473  if (PipelineStack.size() == 1)
2474  return None;
2475 
2476  PipelineStack.pop_back();
2477  } while (Text.consume_front(")"));
2478 
2479  // Check if we've finished parsing.
2480  if (Text.empty())
2481  break;
2482 
2483  // Otherwise, the end of an inner pipeline always has to be followed by
2484  // a comma, and then we can continue.
2485  if (!Text.consume_front(","))
2486  return None;
2487  }
2488 
2489  if (PipelineStack.size() > 1)
2490  // Unbalanced paretheses.
2491  return None;
2492 
2493  assert(PipelineStack.back() == &ResultPipeline &&
2494  "Wrong pipeline at the bottom of the stack!");
2495  return {std::move(ResultPipeline)};
2496 }
2497 
2498 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2499  const PipelineElement &E) {
2500  auto &Name = E.Name;
2501  auto &InnerPipeline = E.InnerPipeline;
2502 
2503  // First handle complex passes like the pass managers which carry pipelines.
2504  if (!InnerPipeline.empty()) {
2505  if (Name == "module") {
2506  ModulePassManager NestedMPM;
2507  if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2508  return Err;
2509  MPM.addPass(std::move(NestedMPM));
2510  return Error::success();
2511  }
2512  if (Name == "cgscc") {
2513  CGSCCPassManager CGPM;
2514  if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
2515  return Err;
2517  return Error::success();
2518  }
2519  if (Name == "function") {
2520  FunctionPassManager FPM;
2521  if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2522  return Err;
2524  return Error::success();
2525  }
2526  if (auto Count = parseRepeatPassName(Name)) {
2527  ModulePassManager NestedMPM;
2528  if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2529  return Err;
2530  MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2531  return Error::success();
2532  }
2533 
2534  for (auto &C : ModulePipelineParsingCallbacks)
2535  if (C(Name, MPM, InnerPipeline))
2536  return Error::success();
2537 
2538  // Normal passes can't have pipelines.
2539  return make_error<StringError>(
2540  formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2542  ;
2543  }
2544 
2545  // Manually handle aliases for pre-configured pipeline fragments.
2547  SmallVector<StringRef, 3> Matches;
2548  if (!DefaultAliasRegex.match(Name, &Matches))
2549  return make_error<StringError>(
2550  formatv("unknown default pipeline alias '{0}'", Name).str(),
2552 
2553  assert(Matches.size() == 3 && "Must capture two matched strings!");
2554 
2555  OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2556  .Case("O0", OptimizationLevel::O0)
2557  .Case("O1", OptimizationLevel::O1)
2558  .Case("O2", OptimizationLevel::O2)
2559  .Case("O3", OptimizationLevel::O3)
2560  .Case("Os", OptimizationLevel::Os)
2561  .Case("Oz", OptimizationLevel::Oz);
2562  if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" &&
2563  Matches[1] != "lto") {
2564  MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" ||
2565  Matches[1] == "lto-pre-link"));
2566  return Error::success();
2567  }
2568 
2569  // This is consistent with old pass manager invoked via opt, but
2570  // inconsistent with clang. Clang doesn't enable loop vectorization
2571  // but does enable slp vectorization at Oz.
2572  PTO.LoopVectorization =
2573  L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2574  PTO.SLPVectorization =
2575  L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2576 
2577  if (Matches[1] == "default") {
2579  } else if (Matches[1] == "thinlto-pre-link") {
2581  } else if (Matches[1] == "thinlto") {
2583  } else if (Matches[1] == "lto-pre-link") {
2585  } else {
2586  assert(Matches[1] == "lto" && "Not one of the matched options!");
2587  MPM.addPass(buildLTODefaultPipeline(L, nullptr));
2588  }
2589  return Error::success();
2590  }
2591 
2592  // Finally expand the basic registered passes from the .inc file.
2593 #define MODULE_PASS(NAME, CREATE_PASS) \
2594  if (Name == NAME) { \
2595  MPM.addPass(CREATE_PASS); \
2596  return Error::success(); \
2597  }
2598 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2599  if (Name == "require<" NAME ">") { \
2600  MPM.addPass( \
2601  RequireAnalysisPass< \
2602  std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
2603  return Error::success(); \
2604  } \
2605  if (Name == "invalidate<" NAME ">") { \
2606  MPM.addPass(InvalidateAnalysisPass< \
2607  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2608  return Error::success(); \
2609  }
2610 #define CGSCC_PASS(NAME, CREATE_PASS) \
2611  if (Name == NAME) { \
2612  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
2613  return Error::success(); \
2614  }
2615 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2616  if (Name == NAME) { \
2617  MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
2618  return Error::success(); \
2619  }
2620 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2621  if (checkParametrizedPassName(Name, NAME)) { \
2622  auto Params = parsePassParameters(PARSER, Name, NAME); \
2623  if (!Params) \
2624  return Params.takeError(); \
2625  MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2626  return Error::success(); \
2627  }
2628 #define LOOP_PASS(NAME, CREATE_PASS) \
2629  if (Name == NAME) { \
2630  MPM.addPass(createModuleToFunctionPassAdaptor( \
2631  createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
2632  return Error::success(); \
2633  }
2634 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2635  if (checkParametrizedPassName(Name, NAME)) { \
2636  auto Params = parsePassParameters(PARSER, Name, NAME); \
2637  if (!Params) \
2638  return Params.takeError(); \
2639  MPM.addPass( \
2640  createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2641  CREATE_PASS(Params.get()), false, false))); \
2642  return Error::success(); \
2643  }
2644 #include "PassRegistry.def"
2645 
2646  for (auto &C : ModulePipelineParsingCallbacks)
2647  if (C(Name, MPM, InnerPipeline))
2648  return Error::success();
2649  return make_error<StringError>(
2650  formatv("unknown module pass '{0}'", Name).str(),
2652 }
2653 
2654 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2655  const PipelineElement &E) {
2656  auto &Name = E.Name;
2657  auto &InnerPipeline = E.InnerPipeline;
2658 
2659  // First handle complex passes like the pass managers which carry pipelines.
2660  if (!InnerPipeline.empty()) {
2661  if (Name == "cgscc") {
2662  CGSCCPassManager NestedCGPM;
2663  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2664  return Err;
2665  // Add the nested pass manager with the appropriate adaptor.
2666  CGPM.addPass(std::move(NestedCGPM));
2667  return Error::success();
2668  }
2669  if (Name == "function") {
2670  FunctionPassManager FPM;
2671  if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2672  return Err;
2673  // Add the nested pass manager with the appropriate adaptor.
2675  return Error::success();
2676  }
2677  if (auto Count = parseRepeatPassName(Name)) {
2678  CGSCCPassManager NestedCGPM;
2679  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2680  return Err;
2681  CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2682  return Error::success();
2683  }
2684  if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2685  CGSCCPassManager NestedCGPM;
2686  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2687  return Err;
2688  CGPM.addPass(
2689  createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2690  return Error::success();
2691  }
2692 
2693  for (auto &C : CGSCCPipelineParsingCallbacks)
2694  if (C(Name, CGPM, InnerPipeline))
2695  return Error::success();
2696 
2697  // Normal passes can't have pipelines.
2698  return make_error<StringError>(
2699  formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2701  }
2702 
2703 // Now expand the basic registered passes from the .inc file.
2704 #define CGSCC_PASS(NAME, CREATE_PASS) \
2705  if (Name == NAME) { \
2706  CGPM.addPass(CREATE_PASS); \
2707  return Error::success(); \
2708  }
2709 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2710  if (Name == "require<" NAME ">") { \
2711  CGPM.addPass(RequireAnalysisPass< \
2712  std::remove_reference<decltype(CREATE_PASS)>::type, \
2713  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
2714  CGSCCUpdateResult &>()); \
2715  return Error::success(); \
2716  } \
2717  if (Name == "invalidate<" NAME ">") { \
2718  CGPM.addPass(InvalidateAnalysisPass< \
2719  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2720  return Error::success(); \
2721  }
2722 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2723  if (Name == NAME) { \
2724  CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
2725  return Error::success(); \
2726  }
2727 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2728  if (checkParametrizedPassName(Name, NAME)) { \
2729  auto Params = parsePassParameters(PARSER, Name, NAME); \
2730  if (!Params) \
2731  return Params.takeError(); \
2732  CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2733  return Error::success(); \
2734  }
2735 #define LOOP_PASS(NAME, CREATE_PASS) \
2736  if (Name == NAME) { \
2737  CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
2738  createFunctionToLoopPassAdaptor(CREATE_PASS, false, false))); \
2739  return Error::success(); \
2740  }
2741 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2742  if (checkParametrizedPassName(Name, NAME)) { \
2743  auto Params = parsePassParameters(PARSER, Name, NAME); \
2744  if (!Params) \
2745  return Params.takeError(); \
2746  CGPM.addPass( \
2747  createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2748  CREATE_PASS(Params.get()), false, false))); \
2749  return Error::success(); \
2750  }
2751 #include "PassRegistry.def"
2752 
2753  for (auto &C : CGSCCPipelineParsingCallbacks)
2754  if (C(Name, CGPM, InnerPipeline))
2755  return Error::success();
2756  return make_error<StringError>(
2757  formatv("unknown cgscc pass '{0}'", Name).str(),
2759 }
2760 
2761 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2762  const PipelineElement &E) {
2763  auto &Name = E.Name;
2764  auto &InnerPipeline = E.InnerPipeline;
2765 
2766  // First handle complex passes like the pass managers which carry pipelines.
2767  if (!InnerPipeline.empty()) {
2768  if (Name == "function") {
2769  FunctionPassManager NestedFPM;
2770  if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2771  return Err;
2772  // Add the nested pass manager with the appropriate adaptor.
2773  FPM.addPass(std::move(NestedFPM));
2774  return Error::success();
2775  }
2776  if (Name == "loop" || Name == "loop-mssa") {
2777  LoopPassManager LPM;
2778  if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
2779  return Err;
2780  // Add the nested pass manager with the appropriate adaptor.
2781  bool UseMemorySSA = (Name == "loop-mssa");
2782  bool UseBFI = llvm::any_of(
2783  InnerPipeline, [](auto Pipeline) { return Pipeline.Name == "licm"; });
2784  FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2785  UseBFI));
2786  return Error::success();
2787  }
2788  if (auto Count = parseRepeatPassName(Name)) {
2789  FunctionPassManager NestedFPM;
2790  if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2791  return Err;
2792  FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2793  return Error::success();
2794  }
2795 
2796  for (auto &C : FunctionPipelineParsingCallbacks)
2797  if (C(Name, FPM, InnerPipeline))
2798  return Error::success();
2799 
2800  // Normal passes can't have pipelines.
2801  return make_error<StringError>(
2802  formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2804  }
2805 
2806 // Now expand the basic registered passes from the .inc file.
2807 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2808  if (Name == NAME) { \
2809  FPM.addPass(CREATE_PASS); \
2810  return Error::success(); \
2811  }
2812 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2813  if (checkParametrizedPassName(Name, NAME)) { \
2814  auto Params = parsePassParameters(PARSER, Name, NAME); \
2815  if (!Params) \
2816  return Params.takeError(); \
2817  FPM.addPass(CREATE_PASS(Params.get())); \
2818  return Error::success(); \
2819  }
2820 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2821  if (Name == "require<" NAME ">") { \
2822  FPM.addPass( \
2823  RequireAnalysisPass< \
2824  std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2825  return Error::success(); \
2826  } \
2827  if (Name == "invalidate<" NAME ">") { \
2828  FPM.addPass(InvalidateAnalysisPass< \
2829  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2830  return Error::success(); \
2831  }
2832 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2833 // bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2834 // "guard-widening");
2835 // The risk is that it may become obsolete if we're not careful.
2836 #define LOOP_PASS(NAME, CREATE_PASS) \
2837  if (Name == NAME) { \
2838  FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false)); \
2839  return Error::success(); \
2840  }
2841 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2842  if (checkParametrizedPassName(Name, NAME)) { \
2843  auto Params = parsePassParameters(PARSER, Name, NAME); \
2844  if (!Params) \
2845  return Params.takeError(); \
2846  FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
2847  false, false)); \
2848  return Error::success(); \
2849  }
2850 #include "PassRegistry.def"
2851 
2852  for (auto &C : FunctionPipelineParsingCallbacks)
2853  if (C(Name, FPM, InnerPipeline))
2854  return Error::success();
2855  return make_error<StringError>(
2856  formatv("unknown function pass '{0}'", Name).str(),
2858 }
2859 
2860 Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
2861  const PipelineElement &E) {
2862  StringRef Name = E.Name;
2863  auto &InnerPipeline = E.InnerPipeline;
2864 
2865  // First handle complex passes like the pass managers which carry pipelines.
2866  if (!InnerPipeline.empty()) {
2867  if (Name == "loop") {
2868  LoopPassManager NestedLPM;
2869  if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2870  return Err;
2871  // Add the nested pass manager with the appropriate adaptor.
2872  LPM.addPass(std::move(NestedLPM));
2873  return Error::success();
2874  }
2875  if (auto Count = parseRepeatPassName(Name)) {
2876  LoopPassManager NestedLPM;
2877  if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2878  return Err;
2879  LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2880  return Error::success();
2881  }
2882 
2883  for (auto &C : LoopPipelineParsingCallbacks)
2884  if (C(Name, LPM, InnerPipeline))
2885  return Error::success();
2886 
2887  // Normal passes can't have pipelines.
2888  return make_error<StringError>(
2889  formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2891  }
2892 
2893 // Now expand the basic registered passes from the .inc file.
2894 #define LOOP_PASS(NAME, CREATE_PASS) \
2895  if (Name == NAME) { \
2896  LPM.addPass(CREATE_PASS); \
2897  return Error::success(); \
2898  }
2899 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2900  if (checkParametrizedPassName(Name, NAME)) { \
2901  auto Params = parsePassParameters(PARSER, Name, NAME); \
2902  if (!Params) \
2903  return Params.takeError(); \
2904  LPM.addPass(CREATE_PASS(Params.get())); \
2905  return Error::success(); \
2906  }
2907 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2908  if (Name == "require<" NAME ">") { \
2909  LPM.addPass(RequireAnalysisPass< \
2910  std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2911  LoopAnalysisManager, LoopStandardAnalysisResults &, \
2912  LPMUpdater &>()); \
2913  return Error::success(); \
2914  } \
2915  if (Name == "invalidate<" NAME ">") { \
2916  LPM.addPass(InvalidateAnalysisPass< \
2917  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2918  return Error::success(); \
2919  }
2920 #include "PassRegistry.def"
2921 
2922  for (auto &C : LoopPipelineParsingCallbacks)
2923  if (C(Name, LPM, InnerPipeline))
2924  return Error::success();
2925  return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2927 }
2928 
2929 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2930 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2931  if (Name == NAME) { \
2932  AA.registerModuleAnalysis< \
2933  std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2934  return true; \
2935  }
2936 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2937  if (Name == NAME) { \
2938  AA.registerFunctionAnalysis< \
2939  std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2940  return true; \
2941  }
2942 #include "PassRegistry.def"
2943 
2944  for (auto &C : AAParsingCallbacks)
2945  if (C(Name, AA))
2946  return true;
2947  return false;
2948 }
2949 
2950 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2951  ArrayRef<PipelineElement> Pipeline) {
2952  for (const auto &Element : Pipeline) {
2953  if (auto Err = parseLoopPass(LPM, Element))
2954  return Err;
2955  }
2956  return Error::success();
2957 }
2958 
2959 Error PassBuilder::parseFunctionPassPipeline(
2961  for (const auto &Element : Pipeline) {
2962  if (auto Err = parseFunctionPass(FPM, Element))
2963  return Err;
2964  }
2965  return Error::success();
2966 }
2967 
2968 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2969  ArrayRef<PipelineElement> Pipeline) {
2970  for (const auto &Element : Pipeline) {
2971  if (auto Err = parseCGSCCPass(CGPM, Element))
2972  return Err;
2973  }
2974  return Error::success();
2975 }
2976 
2988 }
2989 
2990 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2991  ArrayRef<PipelineElement> Pipeline) {
2992  for (const auto &Element : Pipeline) {
2993  if (auto Err = parseModulePass(MPM, Element))
2994  return Err;
2995  }
2996  return Error::success();
2997 }
2998 
2999 // Primary pass pipeline description parsing routine for a \c ModulePassManager
3000 // FIXME: Should this routine accept a TargetMachine or require the caller to
3001 // pre-populate the analysis managers with target-specific stuff?
3003  StringRef PipelineText) {
3004  auto Pipeline = parsePipelineText(PipelineText);
3005  if (!Pipeline || Pipeline->empty())
3006  return make_error<StringError>(
3007  formatv("invalid pipeline '{0}'", PipelineText).str(),
3009 
3010  // If the first name isn't at the module layer, wrap the pipeline up
3011  // automatically.
3012  StringRef FirstName = Pipeline->front().Name;
3013 
3014  if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
3015  if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
3016  Pipeline = {{"cgscc", std::move(*Pipeline)}};
3017  } else if (isFunctionPassName(FirstName,
3018  FunctionPipelineParsingCallbacks)) {
3019  Pipeline = {{"function", std::move(*Pipeline)}};
3020  } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
3021  Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
3022  } else {
3023  for (auto &C : TopLevelPipelineParsingCallbacks)
3024  if (C(MPM, *Pipeline))
3025  return Error::success();
3026 
3027  // Unknown pass or pipeline name!
3028  auto &InnerPipeline = Pipeline->front().InnerPipeline;
3029  return make_error<StringError>(
3030  formatv("unknown {0} name '{1}'",
3031  (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
3032  .str(),
3034  }
3035  }
3036 
3037  if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
3038  return Err;
3039  return Error::success();
3040 }
3041 
3042 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
3044  StringRef PipelineText) {
3045  auto Pipeline = parsePipelineText(PipelineText);
3046  if (!Pipeline || Pipeline->empty())
3047  return make_error<StringError>(
3048  formatv("invalid pipeline '{0}'", PipelineText).str(),
3050 
3051  StringRef FirstName = Pipeline->front().Name;
3052  if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
3053  return make_error<StringError>(
3054  formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
3055  PipelineText)
3056  .str(),
3058 
3059  if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
3060  return Err;
3061  return Error::success();
3062 }
3063 
3064 // Primary pass pipeline description parsing routine for a \c
3065 // FunctionPassManager
3067  StringRef PipelineText) {
3068  auto Pipeline = parsePipelineText(PipelineText);
3069  if (!Pipeline || Pipeline->empty())
3070  return make_error<StringError>(
3071  formatv("invalid pipeline '{0}'", PipelineText).str(),
3073 
3074  StringRef FirstName = Pipeline->front().Name;
3075  if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
3076  return make_error<StringError>(
3077  formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
3078  PipelineText)
3079  .str(),
3081 
3082  if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
3083  return Err;
3084  return Error::success();
3085 }
3086 
3087 // Primary pass pipeline description parsing routine for a \c LoopPassManager
3089  StringRef PipelineText) {
3090  auto Pipeline = parsePipelineText(PipelineText);
3091  if (!Pipeline || Pipeline->empty())
3092  return make_error<StringError>(
3093  formatv("invalid pipeline '{0}'", PipelineText).str(),
3095 
3096  if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
3097  return Err;
3098 
3099  return Error::success();
3100 }
3101 
3103  // If the pipeline just consists of the word 'default' just replace the AA
3104  // manager with our default one.
3105  if (PipelineText == "default") {
3106  AA = buildDefaultAAPipeline();
3107  return Error::success();
3108  }
3109 
3110  while (!PipelineText.empty()) {
3111  StringRef Name;
3112  std::tie(Name, PipelineText) = PipelineText.split(',');
3113  if (!parseAAPassName(AA, Name))
3114  return make_error<StringError>(
3115  formatv("unknown alias analysis name '{0}'", Name).str(),
3117  }
3118 
3119  return Error::success();
3120 }
3121 
3123 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3124  if (PassName == NAME) \
3125  return true;
3126 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3127  if (PassName == NAME) \
3128  return true;
3129 #include "PassRegistry.def"
3130  return false;
3131 }
3132 
3134 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
3135  if (PassName == NAME) \
3136  return true;
3137 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
3138  if (PassName == NAME) \
3139  return true;
3140 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
3141  if (PassName == NAME) \
3142  return true;
3143 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
3144  if (PassName == NAME) \
3145  return true;
3146 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3147  if (PassName == NAME) \
3148  return true;
3149 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3150  if (PassName == NAME) \
3151  return true;
3152 #include "PassRegistry.def"
3153  return false;
3154 }
3155 
3157  OS << " " << PassName << "\n";
3158 }
3159 
3161  // TODO: print pass descriptions when they are available
3162 
3163  OS << "Module passes:\n";
3164 #define MODULE_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3165 #include "PassRegistry.def"
3166 
3167  OS << "Module analyses:\n";
3168 #define MODULE_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3169 #include "PassRegistry.def"
3170 
3171  OS << "Module alias analyses:\n";
3172 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3173 #include "PassRegistry.def"
3174 
3175  OS << "CGSCC passes:\n";
3176 #define CGSCC_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3177 #include "PassRegistry.def"
3178 
3179  OS << "CGSCC analyses:\n";
3180 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3181 #include "PassRegistry.def"
3182 
3183  OS << "Function passes:\n";
3184 #define FUNCTION_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3185 #include "PassRegistry.def"
3186 
3187  OS << "Function analyses:\n";
3188 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3189 #include "PassRegistry.def"
3190 
3191  OS << "Function alias analyses:\n";
3192 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3193 #include "PassRegistry.def"
3194 
3195  OS << "Loop passes:\n";
3196 #define LOOP_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3197 #include "PassRegistry.def"
3198 
3199  OS << "Loop analyses:\n";
3200 #define LOOP_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3201 #include "PassRegistry.def"
3202 }
3203 
3206  &C) {
3207  TopLevelPipelineParsingCallbacks.push_back(C);
3208 }
llvm::GVNOptions
A set of parameters to control various transforms performed by GVN pass.
Definition: GVN.h:75
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
llvm::GlobalsAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: GlobalsModRef.h:132
AddressSanitizer.h
RegionInfo.h
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::PipelineTuningOptions::Coroutines
bool Coroutines
Tuning option to enable/disable coroutine intrinsic lowering.
Definition: PassBuilder.h:114
TypeBasedAliasAnalysis.h
llvm::PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & >
Definition: LoopPassManager.h:70
Instrumentation.h
SymbolRewriter.h
llvm::BasicAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: BasicAliasAnalysis.h:233
llvm::IPSCCPPass
Pass to perform interprocedural constant propagation.
Definition: SCCP.h:30
llvm::ScopedNoAliasAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: ScopedNoAliasAA.h:53
AssumptionCache.h
StackSafetyAnalysis.h
llvm::AAManager
A manager for alias analyses.
Definition: AliasAnalysis.h:1233
startsWithDefaultPipelineAliasPrefix
static bool startsWithDefaultPipelineAliasPrefix(StringRef Name)
Tests whether a pass name starts with a valid prefix for a default pipeline alias.
Definition: PassBuilder.cpp:2313
IROutliner.h
EnableSyntheticCounts
static cl::opt< bool > EnableSyntheticCounts("enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run synthetic function entry count generation " "pass"))
llvm::PassBuilder::OptimizationLevel::O1
static const OptimizationLevel O1
Optimize quickly without destroying debuggability.
Definition: PassBuilder.h:198
Lint.h
llvm::RunNewGVN
cl::opt< bool > RunNewGVN
LowerWidenableCondition.h
DeadArgumentElimination.h
llvm
Definition: AllocatorList.h:23
LoopSimplify.h
DominanceFrontier.h
PassBuilder.h
StripSymbols.h
Phase
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
Definition: AArch64FalkorHWPFFix.cpp:231
llvm::SampleProfileProbePass
Definition: SampleProfileProbe.h:133
llvm::StringRef::empty
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:153
Mem2Reg.h
MemorySanitizer.h
llvm::LoopSinkPass
A pass that does profile-guided sinking of instructions into loops.
Definition: LoopSink.h:33
llvm::PassManager::isEmpty
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:563
OpenMPOpt.h
llvm::InferFunctionAttrsPass
A pass which infers function attributes from the names and signatures of function declarations in a m...
Definition: InferFunctionAttrs.h:25
llvm::SampleProfileLoaderPass
The sample profiler data loader pass.
Definition: SampleProfile.h:25
CalledValuePropagation.h
Annotation2Metadata.h
llvm::LoopInterchangePass
Definition: LoopInterchange.h:17
llvm::AlignmentFromAssumptionsPass
Definition: AlignmentFromAssumptions.h:29
llvm::EnableHotColdSplit
cl::opt< bool > EnableHotColdSplit
PartialInlining.h
Inliner.h
BreakCriticalEdges.h
llvm::Annotation2MetadataPass
Pass to convert @llvm.global.annotations to !annotation metadata.
Definition: Annotation2Metadata.h:24
ThreadSanitizer.h
llvm::ThinOrFullLTOPhase::ThinLTOPostLink
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
llvm::PassInfoMixin
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:374
GlobalSplit.h
llvm::ModuleAnalysisManagerFunctionProxy
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Definition: PassManager.h:1163
llvm::GVNHoistPass
A simple and fast domtree-based GVN pass to hoist common expressions from sibling branches.
Definition: GVN.h:374
CorrelatedValuePropagation.h
llvm::LoopIdiomRecognizePass
Performs Loop Idiom Recognize Pass.
Definition: LoopIdiomRecognize.h:40
llvm::Function
Definition: Function.h:61
InstructionNamer.h
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:530
llvm::ExtraVectorizerPasses
cl::opt< bool > ExtraVectorizerPasses
NaryReassociate.h
llvm::EnableConstraintElimination
cl::opt< bool > EnableConstraintElimination
llvm::CrossDSOCFIPass
Definition: CrossDSOCFI.h:20
llvm::StringSwitch::Default
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
llvm::ReassociatePass
Reassociate commutative expressions.
Definition: Reassociate.h:71
InferAddressSpaces.h
LCSSA.h
ConstantMerge.h
llvm::SetLicmMssaNoAccForPromotionCap
cl::opt< unsigned > SetLicmMssaNoAccForPromotionCap
AlignmentFromAssumptions.h
llvm::StringRef::npos
static constexpr size_t npos
Definition: StringRef.h:60
LowerAtomic.h
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1167
FunctionImport.h
llvm::APInt::getSExtValue
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1643
IndVarSimplify.h
ErrorHandling.h
llvm::RunPartialInlining
cl::opt< bool > RunPartialInlining
LoopAccessAnalysis.h
InstCount.h
SCCP.h
llvm::PassBuilder::buildModuleOptimizationPipeline
ModulePassManager buildModuleOptimizationPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Construct the core LLVM module optimization pipeline.
Definition: PassBuilder.cpp:1314
UnifyFunctionExitNodes.h
llvm::Error::success
static ErrorSuccess success()
Create a success value.
Definition: Error.h:332
TailRecursionElimination.h
DivRemPairs.h
llvm::LoopRotatePass
A simple loop rotation transformation.
Definition: LoopRotation.h:23
llvm::BDCEPass
Definition: BDCE.h:25
llvm::PassBuilder::PassBuilder
PassBuilder(TargetMachine *TM=nullptr, PipelineTuningOptions PTO=PipelineTuningOptions(), Optional< PGOOptions > PGOOpt=None, PassInstrumentationCallbacks *PIC=nullptr)
Definition: PassBuilder.cpp:446
DeadStoreElimination.h
llvm::PipelineTuningOptions
Tunable parameters for passes in the default pipelines.
Definition: PassBuilder.h:86
llvm::StringRef::consume_front
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition: StringRef.h:680
OptimizationRemarkEmitter.h
CoroCleanup.h
isLoopPassName
static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2418
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
GlobalsModRef.h
VectorCombine.h
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
MergeFunctions.h
llvm::PseudoProbeUpdatePass
Definition: SampleProfileProbe.h:153
llvm::ModuleAnalysisManagerCGSCCProxy
OuterAnalysisManagerProxy< ModuleAnalysisManager, LazyCallGraph::SCC, LazyCallGraph & > ModuleAnalysisManagerCGSCCProxy
A proxy from a ModuleAnalysisManager to an SCC.
Definition: CGSCCPassManager.h:215
IVUsersPrinter.h
LoopFlatten.h
ScalarEvolution.h
LoopAccessAnalysisPrinter.h
ElimAvailExtern.h
llvm::PipelineTuningOptions::ForgetAllSCEVInLoopUnroll
bool ForgetAllSCEVInLoopUnroll
Tuning option to forget all SCEV loops in LoopUnroll.
Definition: PassBuilder.h:108
llvm::LoopUnrollOptions
A set of parameters used to control various transforms performed by the LoopUnroll pass.
Definition: LoopUnrollPass.h:61
InjectTLIMappings.h
ConstraintElimination.h
Float2Int.h
llvm::LoopStandardAnalysisResults
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
Definition: LoopAnalysisManager.h:52
CGProfile.h
isModulePassName
static bool isModulePassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2338
PIC
PassInstrumentationCallbacks PIC
Definition: PassBuilderBindings.cpp:55
MetaRenamer.h
llvm::LoopDistributePass
Definition: LoopDistribute.h:25
JumpThreading.h
HotColdSplitting.h
llvm::createCGSCCToFunctionPassAdaptor
CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:494
llvm::createModuleToPostOrderCGSCCPassAdaptor
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:376
llvm::EnableKnowledgeRetention
cl::opt< bool > EnableKnowledgeRetention
enable preservation of attributes in assume like: call void @llvm.assume(i1 true) [ "nonnull"(i32* PT...
Definition: InstCombineCalls.cpp:97
llvm::PGOInstrumentationUse
The profile annotation (profile-instr-use) pass for IR based PGO.
Definition: PGOInstrumentation.h:58
llvm::Optional
Definition: APInt.h:33
llvm::InlineParams
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:160
LoopUnrollAndJamPass.h
PartiallyInlineLibCalls.h
DivergenceAnalysis.h
llvm::LoopLoadEliminationPass
Pass to forward loads in a loop around the backedge to subsequent iterations.
Definition: LoopLoadElimination.h:27
llvm::StackLifetime::LivenessType::Must
@ Must
llvm::EnableOrderFileInstrumentation
cl::opt< bool > EnableOrderFileInstrumentation
FixIrreducible.h
llvm::ThinOrFullLTOPhase::ThinLTOPreLink
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
llvm::TailCallElimPass
Definition: TailRecursionElimination.h:60
llvm::CoroCleanupPass
Definition: CoroCleanup.h:23
LazyValueInfo.h
llvm::EnableMatrix
cl::opt< bool > EnableMatrix
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
llvm::EnableMSSALoopDependency
cl::opt< bool > EnableMSSALoopDependency
Enables memory ssa as a dependency for loop passes.
LoopInstSimplify.h
MemoryDependenceAnalysis.h
ModuleSummaryAnalysis.h
LoopBoundSplit.h
StripDeadPrototypes.h
llvm::PassBuilder::OptimizationLevel::Oz
static const OptimizationLevel Oz
A very specialized mode that will optimize for code size at any and all costs.
Definition: PassBuilder.h:249
llvm::SLPVectorizerPass
Definition: SLPVectorizer.h:58
llvm::ReversePostOrderFunctionAttrsPass
A pass to do RPO deduction and propagation of function attributes.
Definition: FunctionAttrs.h:69
llvm::PartialInlinerPass
Pass to remove unused function declarations.
Definition: PartialInlining.h:24
llvm::PGOOptions::SampleUse
@ SampleUse
Definition: PassBuilder.h:36
llvm::DSEPass
This class implements a trivial dead store elimination.
Definition: DeadStoreElimination.h:28
isLTOPreLink
static bool isLTOPreLink(ThinOrFullLTOPhase Phase)
Definition: PassBuilder.cpp:523
llvm::PassBuilder::addPGOInstrPassesForO0
void addPGOInstrPassesForO0(ModulePassManager &MPM, bool RunProfileGen, bool IsCS, std::string ProfileFile, std::string ProfileRemappingFile)
Add PGOInstrumenation passes for O0 only.
Definition: PassBuilder.cpp:935
BasicAliasAnalysis.h
llvm::PassBuilder::OptimizationLevel::O2
static const OptimizationLevel O2
Optimize for fast execution as much as possible without triggering significant incremental compile ti...
Definition: PassBuilder.h:215
DataFlowSanitizer.h
llvm::Regex::match
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition: Regex.cpp:86
CoroElide.h
llvm::LoopUnrollOptions::setPartial
LoopUnrollOptions & setPartial(bool Partial)
Enables or disables partial unrolling.
Definition: LoopUnrollPass.h:87
Internalize.h
llvm::PipelineTuningOptions::LicmMssaNoAccForPromotionCap
unsigned LicmMssaNoAccForPromotionCap
Tuning option to disable promotion to scalars in LICM with MemorySSA, if the number of access is too ...
Definition: PassBuilder.h:122
MergedLoadStoreMotion.h
llvm::LoopVectorizeOptions::setInterleaveOnlyWhenForced
LoopVectorizeOptions & setInterleaveOnlyWhenForced(bool Value)
Definition: LoopVectorize.h:108
MustExecute.h
llvm::AggressiveInstCombinePass
Definition: AggressiveInstCombine.h:24
llvm::InstrProfiling
Instrumentation based profiling lowering pass.
Definition: InstrProfiling.h:35
llvm::VectorCombinePass
Optimize scalar/vector interactions in IR using target cost models.
Definition: VectorCombine.h:23
llvm::ConstraintEliminationPass
Definition: ConstraintElimination.h:16
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::HotColdSplittingPass
Pass to outline cold regions.
Definition: HotColdSplitting.h:61
llvm::PassBuilder::OptimizationLevel::O0
static const OptimizationLevel O0
Disable as many optimizations as possible.
Definition: PassBuilder.h:182
llvm::TypeBasedAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: TypeBasedAliasAnalysis.h:59
Scalarizer.h
llvm::PassBuilder::parsePassPipeline
Error parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText)
Parse a textual pass pipeline description into a ModulePassManager.
Definition: PassBuilder.cpp:3002
llvm::StringRef::substr
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:609
llvm::ThinOrFullLTOPhase
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:73
CommandLine.h
LoopRotation.h
llvm::formatv
auto formatv(const char *Fmt, Ts &&... Vals) -> formatv_object< decltype(std::make_tuple(detail::build_format_adapter(std::forward< Ts >(Vals))...))>
Definition: FormatVariadic.h:250
llvm::MODULE
@ MODULE
Definition: Attributor.h:3861
Reg2Mem.h
llvm::ModuleInlinerWrapperPass
Module pass, wrapping the inliner pass.
Definition: Inliner.h:118
llvm::CoroElidePass
Definition: CoroElide.h:25
llvm::ControlHeightReductionPass
Definition: ControlHeightReduction.h:22
llvm::CorrelatedValuePropagationPass
Definition: CorrelatedValuePropagation.h:18
llvm::FlattenedProfileUsed
cl::opt< bool > FlattenedProfileUsed
EnableO3NonTrivialUnswitching
static cl::opt< bool > EnableO3NonTrivialUnswitching("enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"))
StackLifetime.h
TargetMachine.h
llvm::AttributorPass
}
Definition: Attributor.h:2440
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:421
llvm::LoopUnrollOptions::setPeeling
LoopUnrollOptions & setPeeling(bool Peeling)
Enables or disables loop peeling.
Definition: LoopUnrollPass.h:99
AlwaysInliner.h
InstrProfiling.h
PostDominators.h
LoopIdiomRecognize.h
HWAddressSanitizer.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::LICMPass
Performs Loop Invariant Code Motion Pass.
Definition: LICM.h:46
HelloWorld.h
llvm::StringRef::split
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
ArgumentPromotion.h
EnableMemProfiler
static cl::opt< bool > EnableMemProfiler("enable-mem-prof", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable memory profiler"))
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::PostOrderFunctionAttrsPass
Computes function attributes in post-order over the call graph.
Definition: FunctionAttrs.h:50
BlockExtractor.h
MAM
ModuleAnalysisManager MAM
Definition: PassBuilderBindings.cpp:61
llvm::CGSCCAnalysisManagerModuleProxy
InnerAnalysisManagerProxy< CGSCCAnalysisManager, Module > CGSCCAnalysisManagerModuleProxy
A proxy from a CGSCCAnalysisManager to a Module.
Definition: CGSCCPassManager.h:168
LowerInvoke.h
llvm::ModuleInlinerWrapperPass::getPM
CGSCCPassManager & getPM()
Allow adding more CGSCC passes, besides inlining.
Definition: Inliner.h:131
isFunctionPassName
static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2392
llvm::PassBuilder::buildFunctionSimplificationPipeline
FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM function canonicalization and simplification pipeline.
Definition: PassBuilder.cpp:666
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::AMDGPU::PALMD::Key
Key
PAL metadata keys.
Definition: AMDGPUMetadata.h:481
llvm::PGOOptions::IRUse
@ IRUse
Definition: PassBuilder.h:36
llvm::CoroEarlyPass
Definition: CoroEarly.h:26
IP
Definition: NVPTXLowerArgs.cpp:166
TargetLibraryInfo.h
RewriteStatepointsForGC.h
AssumeBundleBuilder.h
llvm::DisablePreInliner
cl::opt< bool > DisablePreInliner
llvm::GVNSinkPass
Uses an "inverted" value numbering to decide the similarity of expressions and sinks similar expressi...
Definition: GVN.h:381
llvm::PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & >::addPass
std::enable_if_t< is_detected< HasRunOnLoopT, PassT >::value > addPass(PassT Pass)
Definition: LoopPassManager.h:105
LowerSwitch.h
InlineAdvisor.h
StructurizeCFG.h
llvm::OpenMPOptCGSCCPass
Definition: OpenMPOpt.h:73
llvm::PGOIndirectCallPromotion
The indirect function call promotion pass.
Definition: PGOInstrumentation.h:73
EnablePGOInlineDeferral
static cl::opt< bool > EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true), cl::Hidden, cl::desc("Enable inline deferral during PGO"))
Flag to enable inline deferral during PGO.
IRPrintingPasses.h
Options
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
Definition: PassBuilderBindings.cpp:48
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
MemCpyOptimizer.h
llvm::CanonicalizeAliasesPass
Simple pass that canonicalizes aliases.
Definition: CanonicalizeAliases.h:22
GVN.h
llvm::LoopDeletionPass
Definition: LoopDeletion.h:24
llvm::PromotePass
Definition: Mem2Reg.h:23
llvm::LoopFlattenPass
Definition: LoopFlatten.h:23
llvm::getInlineParams
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
Definition: InlineCost.cpp:2743
llvm::PGOOptions::CSIRInstr
@ CSIRInstr
Definition: PassBuilder.h:37
llvm::PreInlineThreshold
cl::opt< int > PreInlineThreshold
ConstantHoisting.h
llvm::PassBuilder::buildInlinerPipeline
ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the module pipeline that performs inlining as well as the inlining-driven cleanups.
Definition: PassBuilder.cpp:966
LazyCallGraph.h
CFLAndersAliasAnalysis.h
llvm::GlobalOptPass
Optimize globals that never have their address taken.
Definition: GlobalOpt.h:25
llvm::SyntheticCountsPropagation
Definition: SyntheticCountsPropagation.h:17
llvm::PGOMemOPSizeOpt
The profile size based optimization pass for memory intrinsics.
Definition: PGOInstrumentation.h:86
FormatVariadic.h
llvm::PassBuilder::buildThinLTOPreLinkDefaultPipeline
ModulePassManager buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, ThinLTO-targeting default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1519
PrintPasses.h
ScopedNoAliasAA.h
llvm::StringRef::getAsInteger
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:508
llvm::None
const NoneType None
Definition: None.h:23
CanonicalizeFreezeInLoops.h
llvm::MemProfilerPass
Public interface to the memory profiler pass for instrumenting code to profile memory accesses.
Definition: MemProfiler.h:29
llvm::LoopSimplifyCFGPass
Performs basic CFG simplifications to assist other loop passes.
Definition: LoopSimplifyCFG.h:26
llvm::PGOInstrumentationGenCreateVar
The instrumentation (profile-instr-gen) pass for IR based PGO.
Definition: PGOInstrumentation.h:35
llvm::MergedLoadStoreMotionPass
Definition: MergedLoadStoreMotion.h:41
ControlHeightReduction.h
InstSimplifyPass.h
llvm::LowerTypeTestsPass
Definition: LowerTypeTests.h:200
llvm::PassBuilder::isAnalysisPassName
bool isAnalysisPassName(StringRef PassName)
Returns true if the pass name is the name of a (non-alias) analysis pass.
Definition: PassBuilder.cpp:3133
llvm::PassBuilder::isAAPassName
bool isAAPassName(StringRef PassName)
Returns true if the pass name is the name of an alias analysis pass.
Definition: PassBuilder.cpp:3122
LoopInfo.h
llvm::AnnotationRemarksPass
Definition: AnnotationRemarks.h:23
llvm::SimpleLoopUnswitchPass
This pass transforms loops that contain branches or switches on loop- invariant conditions to have mu...
Definition: SimpleLoopUnswitch.h:62
llvm::cl::ZeroOrMore
@ ZeroOrMore
Definition: CommandLine.h:120
LoopFuse.h
SROA.h
SampleProfile.h
llvm::GlobalSplitPass
Pass to perform split of global variables.
Definition: GlobalSplit.h:26
MakeGuardsExplicit.h
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
llvm::StackLifetime::LivenessType::May
@ May
ForceFunctionAttrs.h
llvm::cl::opt
Definition: CommandLine.h:1422
Attributor.h
llvm::ModuleMemProfilerPass
Public interface to the memory profiler module pass for instrumenting code to profile memory allocati...
Definition: MemProfiler.h:38
llvm::createModuleToFunctionPassAdaptor
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: PassManager.h:1209
LoadStoreVectorizer.h
SpeculativeExecution.h
checkParametrizedPassName
static bool checkParametrizedPassName(StringRef Name, StringRef PassName)
Definition: PassBuilder.cpp:2047
InlineSizeEstimatorAnalysis.h
llvm::PassBuilder::registerFunctionAnalyses
void registerFunctionAnalyses(FunctionAnalysisManager &FAM)
Registers all available function analysis passes.
Definition: PassBuilder.cpp:497
llvm::cl::values
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:699
BranchProbabilityInfo.h
llvm::GVN
The core GVN pass object.
Definition: GVN.h:118
EarlyCSE.h
CrossDSOCFI.h
CFGPrinter.h
GCOVProfiler.h
ProfileSummaryInfo.h
DefaultAliasRegex
static const Regex DefaultAliasRegex("^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$")
llvm::WholeProgramDevirtPass
Definition: WholeProgramDevirt.h:223
llvm::LPMUpdater
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
Definition: LoopPassManager.h:240
CoroSplit.h
BoundsChecking.h
llvm::AssumeSimplifyPass
This pass attempts to minimize the number of assume without loosing any information.
Definition: AssumeBundleBuilder.h:54
StripGCRelocates.h
llvm::PassBuilder::buildLTOPreLinkDefaultPipeline
ModulePassManager buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, LTO-targeting default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1636
llvm::PassBuilder::registerModuleAnalyses
void registerModuleAnalyses(ModuleAnalysisManager &MAM)
Registers all available module analysis passes.
Definition: PassBuilder.cpp:479
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::AttributorCGSCCPass
Definition: Attributor.h:2443
PGOInstrumentation.h
CGSCCPassManager.h
llvm::AnalysisKey
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:72
llvm::StringRef::front
LLVM_NODISCARD char front() const
front - Get the first character in the string.
Definition: StringRef.h:161
llvm::PassBuilder::buildLTODefaultPipeline
ModulePassManager buildLTODefaultPipeline(OptimizationLevel Level, ModuleSummaryIndex *ExportSummary)
Build an LTO default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1645
llvm::ForceFunctionAttrsPass
Pass which forces specific function attributes into the IR, primarily as a debugging tool.
Definition: ForceFunctionAttrs.h:22
AggressiveInstCombine.h
llvm::InvalidateAnalysisPass
A no-op pass template which simply forces a specific analysis result to be invalidated.
Definition: PassManager.h:1252
UseBFI
UseBFI
Definition: MachineLICM.cpp:83
LowerExpectIntrinsic.h
llvm::PassBuilder::buildThinLTODefaultPipeline
ModulePassManager buildThinLTODefaultPipeline(OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary)
Build an ThinLTO default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1580
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::DeadArgumentEliminationPass
Eliminate dead arguments (and return values) from functions.
Definition: DeadArgumentElimination.h:38
ADCE.h
llvm::EnableGVNSink
cl::opt< bool > EnableGVNSink
llvm::PassBuilder::buildDefaultAAPipeline
AAManager buildDefaultAAPipeline()
Build the default AAManager with the default alias analysis pipeline registered.
Definition: PassBuilder.cpp:2000
InferFunctionAttrs.h
SimpleLoopUnswitch.h
llvm::PipelineTuningOptions::PipelineTuningOptions
PipelineTuningOptions()
Constructor sets pipeline tuning defaults based on cl::opts.
Definition: PassBuilder.cpp:281
llvm::InstrProfOptions
Options for the frontend instrumentation based profiling pass.
Definition: Instrumentation.h:121
Enable
@ Enable
Definition: DwarfDebug.cpp:88
llvm::SpeculativeExecutionPass
Definition: SpeculativeExecution.h:69
llvm::StackLifetime::LivenessType
LivenessType
Definition: StackLifetime.h:84
llvm::TargetMachine::registerDefaultAliasAnalyses
virtual void registerDefaultAliasAnalyses(AAManager &)
Allow the target to register alias analyses with the AAManager for use with the new pass manager.
Definition: TargetMachine.h:337
SeparateConstOffsetFromGEP.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::PassBuilder::printPassNames
void printPassNames(raw_ostream &OS)
Print pass names.
Definition: PassBuilder.cpp:3160
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::EliminateAvailableExternallyPass
A pass that transforms external global definitions into declarations.
Definition: ElimAvailExtern.h:24
Regex.h
llvm::ADCEPass
A DCE pass that assumes instructions are dead until proven otherwise.
Definition: ADCE.h:31
MemDerefPrinter.h
llvm::PipelineTuningOptions::SLPVectorization
bool SLPVectorization
Tuning option to enable/disable slp loop vectorization, set based on opt level.
Definition: PassBuilder.h:101
llvm::RelLookupTableConverterPass
Definition: RelLookupTableConverter.h:60
Reassociate.h
llvm::FunctionAnalysisManagerLoopProxy
OuterAnalysisManagerProxy< FunctionAnalysisManager, Loop, LoopStandardAnalysisResults & > FunctionAnalysisManagerLoopProxy
A proxy from a FunctionAnalysisManager to a Loop.
Definition: LoopAnalysisManager.h:156
llvm::createDevirtSCCRepeatedPass
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:539
llvm::JumpThreadingPass
This pass performs 'jump threading', which looks at blocks that have multiple predecessors and multip...
Definition: JumpThreading.h:79
SafepointIRVerifier.h
GuardWidening.h
llvm::LoopFullUnrollPass
Loop unroll pass that only does full loop unrolling and peeling.
Definition: LoopUnrollPass.h:26
parseDevirtPassName
static Optional< int > parseDevirtPassName(StringRef Name)
Definition: PassBuilder.cpp:2038
llvm::LowerExpectIntrinsicPass
Definition: LowerExpectIntrinsic.h:23
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
LoopNestAnalysis.h
llvm::printBeforePasses
std::vector< std::string > printBeforePasses()
Definition: PrintPasses.cpp:73
ObjCARCAliasAnalysis.h
llvm::EnableIROutliner
cl::opt< bool > EnableIROutliner
llvm::InliningAdvisorMode::Development
@ Development
llvm::PassBuilder::OptimizationLevel::Os
static const OptimizationLevel Os
Similar to O2 but tries to optimize for small code size instead of fast execution without triggering ...
Definition: PassBuilder.h:241
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
SampleProfileProbe.h
LoopPassManager.h
llvm::Float2IntPass
Definition: Float2Int.h:26
llvm::LibCallsShrinkWrapPass
Definition: LibCallsShrinkWrap.h:18
InductiveRangeCheckElimination.h
ModuleDebugInfoPrinter.h
llvm::AnalysisInfoMixin
A CRTP mix-in that provides informational APIs needed for analysis passes.
Definition: PassManager.h:391
NameAnonGlobals.h
llvm::CalledValuePropagationPass
Definition: CalledValuePropagation.h:26
llvm::WarnMissedTransformationsPass
Definition: WarnMissedTransforms.h:24
llvm::IndVarSimplifyPass
Definition: IndVarSimplify.h:25
LowerMatrixIntrinsics.h
llvm::GlobalDCEPass
Pass to remove unused function declarations.
Definition: GlobalDCE.h:29
llvm::printAfterPasses
std::vector< std::string > printAfterPasses()
Definition: PrintPasses.cpp:77
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:70
LoopInterchange.h
llvm::PipelineTuningOptions::CallGraphProfile
bool CallGraphProfile
Tuning option to enable/disable call graph profile.
Definition: PassBuilder.h:126
llvm::ThinOrFullLTOPhase::FullLTOPostLink
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
llvm::LowerMatrixIntrinsicsPass
Definition: LowerMatrixIntrinsics.h:19
llvm::NewGVNPass
Definition: NewGVN.h:23
LoopExtractor.h
SyntheticCountsPropagation.h
DDG.h
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::AddDiscriminatorsPass
Definition: AddDiscriminators.h:24
CanonicalizeAliases.h
llvm::CGSCC
@ CGSCC
Definition: Attributor.h:3862
LibCallsShrinkWrap.h
llvm::AAManager::registerModuleAnalysis
void registerModuleAnalysis()
Register a specific AA result.
Definition: AliasAnalysis.h:1243
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1489
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::InstCombinePass
Definition: InstCombine.h:25
SanitizerCoverage.h
llvm::LoopAnalysisManagerFunctionProxy
InnerAnalysisManagerProxy< LoopAnalysisManager, Function > LoopAnalysisManagerFunctionProxy
A proxy from a LoopAnalysisManager to a Function.
Definition: LoopAnalysisManager.h:79
GlobalDCE.h
llvm::EnableLoopFlatten
cl::opt< bool > EnableLoopFlatten
clEnumValN
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:674
SpeculateAroundPHIs.h
llvm::LoopVectorizeOptions::setVectorizeOnlyWhenForced
LoopVectorizeOptions & setVectorizeOnlyWhenForced(bool Value)
Definition: LoopVectorize.h:113
llvm::PipelineTuningOptions::LoopVectorization
bool LoopVectorization
Tuning option to enable/disable loop vectorization, set based on opt level.
Definition: PassBuilder.h:97
NewGVN.h
llvm::PICLevel::Level
Level
Definition: CodeGen.h:33
llvm::createFunctionToLoopPassAdaptor
std::enable_if_t< is_detected< HasRunOnLoopT, LoopPassT >::value, FunctionToLoopPassAdaptor > createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA=false, bool UseBlockFrequencyInfo=false)
A function to deduce a loop pass type and wrap it in the templated adaptor.
Definition: LoopPassManager.h:448
llvm::LoopUnrollOptions::setFullUnrollMaxCount
LoopUnrollOptions & setFullUnrollMaxCount(unsigned O)
Definition: LoopUnrollPass.h:124
LowerTypeTests.h
BlockFrequencyInfo.h
CallSiteSplitting.h
llvm::LoopVectorizeOptions
Definition: LoopVectorize.h:83
llvm::InstrOrderFilePass
The instrumentation pass for recording function order.
Definition: InstrOrderFile.h:21
LoopSimplifyCFG.h
llvm::CGProfilePass
Definition: CGProfile.h:19
AnnotationRemarks.h
LAM
LoopAnalysisManager LAM
Definition: PassBuilderBindings.cpp:58
llvm::PassBuilder::parseAAPipeline
Error parseAAPipeline(AAManager &AA, StringRef PipelineText)
Parse a textual alias analysis pipeline into the provided AA manager.
Definition: PassBuilder.cpp:3102
LoopVectorize.h
SCCP.h
DemandedBits.h
llvm::ModuleInlinerWrapperPass::addModulePass
void addModulePass(T Pass)
Allow adding module-level passes benefiting the contained CGSCC passes.
Definition: Inliner.h:134
llvm::PassManager< Function >
llvm::LoopUnrollAndJamPass
A simple loop rotation transformation.
Definition: LoopUnrollAndJamPass.h:18
MergeICmps.h
LoopVersioningLICM.h
callbacksAcceptPassName
static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks)
Tests whether registered callbacks will accept a given pass name.
Definition: PassBuilder.cpp:2327
FunctionPropertiesAnalysis.h
llvm::IROutlinerPass
Pass to outline similar regions.
Definition: IROutliner.h:351
PhiValues.h
llvm::PassBuilder::registerLoopAnalyses
void registerLoopAnalyses(LoopAnalysisManager &LAM)
Registers all available loop analysis passes.
Definition: PassBuilder.cpp:506
name
static const char * name
Definition: SVEIntrinsicOpts.cpp:84
StraightLineStrengthReduce.h
llvm::ForgetSCEVInLoopUnroll
cl::opt< bool > ForgetSCEVInLoopUnroll
SLPVectorizer.h
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
LoopLoadElimination.h
llvm::CGSCCAnalysisManagerFunctionProxy
OuterAnalysisManagerProxy< CGSCCAnalysisManager, Function > CGSCCAnalysisManagerFunctionProxy
A proxy from a CGSCCAnalysisManager to a Function.
Definition: CGSCCPassManager.h:427
Verifier.h
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
LoopStrengthReduce.h
llvm::PassBuilder::OptimizationLevel
LLVM-provided high-level optimization levels.
Definition: PassBuilder.h:163
llvm::inconvertibleErrorCode
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
llvm::PassBuilder::buildPerModuleDefaultPipeline
ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Build a per-module default optimization pipeline.
Definition: PassBuilder.cpp:1478
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::EnableCHR
cl::opt< bool > EnableCHR
LoopUnrollPass.h
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
llvm::SCCPPass
This pass performs function-level constant propagation and merging.
Definition: SCCP.h:37
llvm::PipelineTuningOptions::MergeFunctions
bool MergeFunctions
Tuning option to enable/disable function merging.
Definition: PassBuilder.h:130
IRSimilarityIdentifier.h
UnifyLoopExits.h
llvm::PassBuilder::registerCGSCCAnalyses
void registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM)
Registers all available CGSCC analysis passes.
Definition: PassBuilder.cpp:488
PerformMandatoryInliningsFirst
static cl::opt< bool > PerformMandatoryInliningsFirst("mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Perform mandatory inlinings module-wide, before performing " "inlining."))
llvm::PassBuilder::OptimizationLevel::O3
static const OptimizationLevel O3
Optimize for fast execution as much as possible.
Definition: PassBuilder.h:230
llvm::NameAnonGlobalPass
Simple pass that provides a name to every anonymous globals.
Definition: NameAnonGlobals.h:23
PassManager.h
LowerConstantIntrinsics.h
llvm::ThinOrFullLTOPhase::None
@ None
No LTO/ThinLTO behavior needed.
addAnnotationRemarksPass
static void addAnnotationRemarksPass(ModulePassManager &MPM)
Definition: PassBuilder.cpp:516
ScalarEvolutionAliasAnalysis.h
llvm::PGOOptions::CSIRUse
@ CSIRUse
Definition: PassBuilder.h:37
StringSwitch.h
llvm::CoroSplitPass
Definition: CoroSplit.h:24
ScalarizeMaskedMemIntrin.h
llvm::PassBuilder::buildO0DefaultPipeline
ModulePassManager buildO0DefaultPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Build an O0 pipeline with the minimal semantically required passes.
Definition: PassBuilder.cpp:1905
llvm::AnalysisManager::registerPass
bool registerPass(PassBuilderT &&PassBuilder)
Register an analysis pass with the manager.
Definition: PassManager.h:831
Sink.h
LoopVersioning.h
WholeProgramDevirt.h
LowerGuardIntrinsic.h
llvm::MemCpyOptPass
Definition: MemCpyOptimizer.h:42
llvm::AAManager::registerFunctionAnalysis
void registerFunctionAnalysis()
Register a specific AA result.
Definition: AliasAnalysis.h:1238
llvm::ArgumentPromotionPass
Argument promotion pass.
Definition: ArgumentPromotion.h:24
getInlineParamsFromOptLevel
static InlineParams getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level)
Definition: PassBuilder.cpp:961
llvm::DivRemPairsPass
Hoist/decompose integer division and remainder instructions to enable CFG improvements and better cod...
Definition: DivRemPairs.h:23
LoopDistribute.h
llvm::LoopUnrollOptions::setRuntime
LoopUnrollOptions & setRuntime(bool Runtime)
Enables or disables unrolling of loops with runtime trip count.
Definition: LoopUnrollPass.h:93
llvm::AlwaysInlinerPass
Inlines functions marked as "always_inline".
Definition: AlwaysInliner.h:29
CallGraph.h
WarnMissedTransforms.h
LoopReroll.h
DDGPrinter.h
llvm::ThinOrFullLTOPhase::FullLTOPreLink
@ FullLTOPreLink
Full LTO prelink phase.
llvm::SimplifyCFGOptions
Definition: SimplifyCFGOptions.h:23
MemorySSA.h
AliasAnalysisEvaluator.h
GlobalOpt.h
llvm::InjectTLIMappings
Definition: InjectTLIMappings.h:20
LoopSink.h
IVUsers.h
llvm::LowerConstantIntrinsicsPass
Definition: LowerConstantIntrinsics.h:23
ObjCARC.h
llvm::OpenMPOptPass
OpenMP optimizations pass.
Definition: OpenMPOpt.h:65
llvm::ModuleSummaryIndex
Class to hold module path string table and global value map, and encapsulate methods for operating on...
Definition: ModuleSummaryIndex.h:1026
llvm::PassInstrumentationCallbacks
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
Definition: PassInstrumentation.h:66
Dominators.h
llvm::StringRef::consume_back
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
Definition: StringRef.h:690
llvm::InliningAdvisorMode::Default
@ Default
StripNonLineTableDebugInfo.h
llvm::PassBuilder::crossRegisterProxies
void crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, ModuleAnalysisManager &MAM)
Cross register the analysis managers through their proxies.
Definition: PassBuilder.cpp:2977
llvm::LoopUnrollOptions::setOptLevel
LoopUnrollOptions & setOptLevel(int O)
Definition: LoopUnrollPass.h:112
llvm::PGOInstrumentationGen
The instrumentation (profile-instr-gen) pass for IR based PGO.
Definition: PGOInstrumentation.h:47
llvm::CallSiteSplittingPass
Definition: CallSiteSplitting.h:17
LoopDeletion.h
llvm::LoopInstSimplifyPass
Performs Loop Inst Simplify Pass.
Definition: LoopInstSimplify.h:25
MemProfiler.h
TargetTransformInfo.h
llvm::MemorySanitizerOptions
Definition: MemorySanitizer.h:21
Delinearization.h
LICM.h
LoopCacheAnalysis.h
llvm::EnableLoopInterchange
cl::opt< bool > EnableLoopInterchange
CoroEarly.h
llvm::LoopUnrollOptions::setProfileBasedPeeling
LoopUnrollOptions & setProfileBasedPeeling(int O)
Definition: LoopUnrollPass.h:118
llvm::EnableUnrollAndJam
cl::opt< bool > EnableUnrollAndJam
llvm::PGOOptions::IRInstr
@ IRInstr
Definition: PassBuilder.h:36
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
DCE.h
AliasSetTracker.h
llvm::InliningAdvisorMode::Release
@ Release
llvm::AttributorRun
cl::opt< AttributorRunOption > AttributorRun
llvm::StringRef::find_first_of
LLVM_NODISCARD size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:410
BDCE.h
InstrOrderFile.h
parseRepeatPassName
static Optional< int > parseRepeatPassName(StringRef Name)
Definition: PassBuilder.cpp:2029
InstCombine.h
DependenceAnalysis.h
llvm::InstSimplifyPass
Run instruction simplification across each instruction in the function.
Definition: InstSimplifyPass.h:34
llvm::RequireAnalysisPass
A utility pass template to force an analysis result to be available.
Definition: PassManager.h:1230
llvm::PassBuilder::buildModuleSimplificationPipeline
ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM module canonicalization and simplification pipeline.
Definition: PassBuilder.cpp:1033
llvm::PipelineTuningOptions::LoopUnrolling
bool LoopUnrolling
Tuning option to enable/disable loop unrolling. Its default value is true.
Definition: PassBuilder.h:104
llvm::cl::desc
Definition: CommandLine.h:414
llvm::Regex
Definition: Regex.h:28
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:112
llvm::PipelineTuningOptions::LicmMssaOptCap
unsigned LicmMssaOptCap
Tuning option to cap the number of calls to retrive clobbering accesses in MemorySSA,...
Definition: PassBuilder.h:118
LoopPredication.h
UseInlineAdvisor
static cl::opt< InliningAdvisorMode > UseInlineAdvisor("enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden, cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"), cl::values(clEnumValN(InliningAdvisorMode::Default, "default", "Heuristics-based inliner version."), clEnumValN(InliningAdvisorMode::Development, "development", "Use development mode (runtime-loadable model)."), clEnumValN(InliningAdvisorMode::Release, "release", "Use release mode (AOT-compiled model).")))
llvm::PipelineTuningOptions::LoopInterleaving
bool LoopInterleaving
Tuning option to set loop interleaving on/off, set based on opt level.
Definition: PassBuilder.h:93
EntryExitInstrumenter.h
llvm::MaxDevirtIterations
cl::opt< unsigned > MaxDevirtIterations("max-devirt-iterations", cl::ReallyHidden, cl::init(4))
Definition: PassBuilder.cpp:295
SimplifyCFG.h
llvm::SetLicmMssaOptCap
cl::opt< unsigned > SetLicmMssaOptCap
llvm::MergeFunctionsPass
Merge identical functions.
Definition: MergeFunctions.h:25
llvm::PassBuilder::registerParseTopLevelPipelineCallback
void registerParseTopLevelPipelineCallback(const std::function< bool(ModulePassManager &, ArrayRef< PipelineElement >)> &C)
Register a callback for a top-level pipeline entry.
Definition: PassBuilder.cpp:3204
PoisonChecking.h
MPM
ModulePassManager MPM
Definition: PassBuilderBindings.cpp:70
llvm::PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & >::isEmpty
bool isEmpty() const
Definition: LoopPassManager.h:150
llvm::LoopUnrollPass
Loop unroll pass that will support both full and partial unrolling.
Definition: LoopUnrollPass.h:133
CGAM
CGSCCAnalysisManager CGAM
Definition: PassBuilderBindings.cpp:60
llvm::FunctionAnalysisManagerModuleProxy
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
Definition: PassManager.h:1009
FunctionAttrs.h
Debug.h
llvm::SimplifyCFGPass
A pass to simplify and canonicalize the CFG of a function.
Definition: SimplifyCFG.h:29
isCGSCCPassName
static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2367
llvm::LoopVectorizePass
The LoopVectorize Pass.
Definition: LoopVectorize.h:129
RelLookupTableConverter.h
PassName
static const char PassName[]
Definition: X86LowerAMXIntrinsics.cpp:669
llvm::PassManager::addPass
std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT Pass)
Definition: PassManager.h:542
CFLSteensAliasAnalysis.h
llvm::LoopUnrollOptions::setUpperBound
LoopUnrollOptions & setUpperBound(bool UpperBound)
Enables or disables the use of trip count upper bound in loop unrolling.
Definition: LoopUnrollPass.h:106
LoopDataPrefetch.h
llvm::SROA
An optimization pass providing Scalar Replacement of Aggregates.
Definition: SROA.h:65
llvm::createRepeatedPass
RepeatedPass< PassT > createRepeatedPass(int Count, PassT P)
Definition: PassManager.h:1320
printPassName
static void printPassName(StringRef PassName, raw_ostream &OS)
Definition: PassBuilder.cpp:3156
llvm::EarlyCSEPass
A simple and fast domtree-based CSE pass.
Definition: EarlyCSE.h:30
llvm::SpeculateAroundPHIsPass
This pass handles simple speculating of instructions around PHIs when doing so is profitable for a pa...
Definition: SpeculateAroundPHIs.h:102
llvm::ConstantMergePass
A pass that merges duplicate global constants into a single constant.
Definition: ConstantMerge.h:29
AddDiscriminators.h
llvm::EnableGVNHoist
cl::opt< bool > EnableGVNHoist