LLVM  13.0.0git
PassBuilder.cpp
Go to the documentation of this file.
1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16 
18 #include "llvm/ADT/StringSwitch.h"
30 #include "llvm/Analysis/DDG.h"
40 #include "llvm/Analysis/IVUsers.h"
46 #include "llvm/Analysis/Lint.h"
49 #include "llvm/Analysis/LoopInfo.h"
71 #include "llvm/IR/Dominators.h"
73 #include "llvm/IR/PassManager.h"
74 #include "llvm/IR/PrintPasses.h"
76 #include "llvm/IR/Verifier.h"
78 #include "llvm/Support/Debug.h"
81 #include "llvm/Support/Regex.h"
138 #include "llvm/Transforms/ObjCARC.h"
239 
240 using namespace llvm;
241 
243 
245  "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
246  cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
248  "Heuristics-based inliner version."),
250  "Use development mode (runtime-loadable model)."),
252  "Use release mode (AOT-compiled model).")));
253 
255  "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
256  cl::desc("Run synthetic function entry count generation "
257  "pass"));
258 
259 static const Regex DefaultAliasRegex(
260  "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
261 
262 /// Flag to enable inline deferral during PGO.
263 static cl::opt<bool>
264  EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
265  cl::Hidden,
266  cl::desc("Enable inline deferral during PGO"));
267 
268 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
270  cl::desc("Enable memory profiler"));
271 
273  "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
274  cl::desc("Perform mandatory inlinings module-wide, before performing "
275  "inlining."));
276 
278  "enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden,
279  cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"));
280 
282  LoopInterleaving = true;
283  LoopVectorization = true;
284  SLPVectorization = false;
285  LoopUnrolling = true;
287  Coroutines = false;
290  CallGraphProfile = true;
291  MergeFunctions = false;
292 }
294 
301 extern cl::opt<bool> EnableCHR;
305 extern cl::opt<bool> RunNewGVN;
307 
309 
312 
314 
317 
319  /*SpeedLevel*/ 0,
320  /*SizeLevel*/ 0};
322  /*SpeedLevel*/ 1,
323  /*SizeLevel*/ 0};
325  /*SpeedLevel*/ 2,
326  /*SizeLevel*/ 0};
328  /*SpeedLevel*/ 3,
329  /*SizeLevel*/ 0};
331  /*SpeedLevel*/ 2,
332  /*SizeLevel*/ 1};
334  /*SpeedLevel*/ 2,
335  /*SizeLevel*/ 2};
336 
337 namespace {
338 
339 // The following passes/analyses have custom names, otherwise their name will
340 // include `(anonymous namespace)`. These are special since they are only for
341 // testing purposes and don't live in a header file.
342 
343 /// No-op module pass which does nothing.
344 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
346  return PreservedAnalyses::all();
347  }
348 
349  static StringRef name() { return "NoOpModulePass"; }
350 };
351 
352 /// No-op module analysis.
353 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
355  static AnalysisKey Key;
356 
357 public:
358  struct Result {};
359  Result run(Module &, ModuleAnalysisManager &) { return Result(); }
360  static StringRef name() { return "NoOpModuleAnalysis"; }
361 };
362 
363 /// No-op CGSCC pass which does nothing.
364 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
367  return PreservedAnalyses::all();
368  }
369  static StringRef name() { return "NoOpCGSCCPass"; }
370 };
371 
372 /// No-op CGSCC analysis.
373 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
375  static AnalysisKey Key;
376 
377 public:
378  struct Result {};
380  return Result();
381  }
382  static StringRef name() { return "NoOpCGSCCAnalysis"; }
383 };
384 
385 /// No-op function pass which does nothing.
386 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
388  return PreservedAnalyses::all();
389  }
390  static StringRef name() { return "NoOpFunctionPass"; }
391 };
392 
393 /// No-op function analysis.
394 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
396  static AnalysisKey Key;
397 
398 public:
399  struct Result {};
400  Result run(Function &, FunctionAnalysisManager &) { return Result(); }
401  static StringRef name() { return "NoOpFunctionAnalysis"; }
402 };
403 
404 /// No-op loop pass which does nothing.
405 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
408  return PreservedAnalyses::all();
409  }
410  static StringRef name() { return "NoOpLoopPass"; }
411 };
412 
413 /// No-op loop analysis.
414 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
416  static AnalysisKey Key;
417 
418 public:
419  struct Result {};
421  return Result();
422  }
423  static StringRef name() { return "NoOpLoopAnalysis"; }
424 };
425 
430 
431 /// Whether or not we should populate a PassInstrumentationCallbacks's class to
432 /// pass name map.
433 ///
434 /// This is for optimization purposes so we don't populate it if we never use
435 /// it. This should be updated if new pass instrumentation wants to use the map.
436 /// We currently only use this for --print-before/after.
437 bool shouldPopulateClassToPassNames() {
438  return !printBeforePasses().empty() || !printAfterPasses().empty();
439 }
440 
441 } // namespace
442 
446  : DebugLogging(DebugLogging), TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
447  if (TM)
448  TM->registerPassBuilderCallbacks(*this, DebugLogging);
449  if (PIC && shouldPopulateClassToPassNames()) {
450 #define MODULE_PASS(NAME, CREATE_PASS) \
451  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
452 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
453  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
454 #define FUNCTION_PASS(NAME, CREATE_PASS) \
455  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
456 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
457  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
458 #define LOOP_PASS(NAME, CREATE_PASS) \
459  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
460 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
461  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
462 #define CGSCC_PASS(NAME, CREATE_PASS) \
463  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
464 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
465  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
466 #include "PassRegistry.def"
467  }
468 }
469 
470 void PassBuilder::invokePeepholeEPCallbacks(
472  for (auto &C : PeepholeEPCallbacks)
473  C(FPM, Level);
474 }
475 
477 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
478  MAM.registerPass([&] { return CREATE_PASS; });
479 #include "PassRegistry.def"
480 
481  for (auto &C : ModuleAnalysisRegistrationCallbacks)
482  C(MAM);
483 }
484 
486 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
487  CGAM.registerPass([&] { return CREATE_PASS; });
488 #include "PassRegistry.def"
489 
490  for (auto &C : CGSCCAnalysisRegistrationCallbacks)
491  C(CGAM);
492 }
493 
495 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
496  FAM.registerPass([&] { return CREATE_PASS; });
497 #include "PassRegistry.def"
498 
499  for (auto &C : FunctionAnalysisRegistrationCallbacks)
500  C(FAM);
501 }
502 
504 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
505  LAM.registerPass([&] { return CREATE_PASS; });
506 #include "PassRegistry.def"
507 
508  for (auto &C : LoopAnalysisRegistrationCallbacks)
509  C(LAM);
510 }
511 
512 // Helper to add AnnotationRemarksPass.
517 }
518 
519 // Helper to check if the current compilation phase is preparing for LTO
523 }
524 
525 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
527 PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
529 
530  FunctionPassManager FPM(DebugLogging);
531 
532  // Form SSA out of local memory accesses after breaking apart aggregates into
533  // scalars.
534  FPM.addPass(SROA());
535 
536  // Catch trivial redundancies
537  FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
538 
539  // Hoisting of scalars and load expressions.
540  FPM.addPass(SimplifyCFGPass());
541  FPM.addPass(InstCombinePass());
542 
544 
545  invokePeepholeEPCallbacks(FPM, Level);
546 
547  FPM.addPass(SimplifyCFGPass());
548 
549  // Form canonically associated expression trees, and simplify the trees using
550  // basic mathematical properties. For example, this will form (nearly)
551  // minimal multiplication trees.
552  FPM.addPass(ReassociatePass());
553 
554  // Add the primary loop simplification pipeline.
555  // FIXME: Currently this is split into two loop pass pipelines because we run
556  // some function passes in between them. These can and should be removed
557  // and/or replaced by scheduling the loop pass equivalents in the correct
558  // positions. But those equivalent passes aren't powerful enough yet.
559  // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
560  // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
561  // fully replace `SimplifyCFGPass`, and the closest to the other we have is
562  // `LoopInstSimplify`.
563  LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
564 
565  // Simplify the loop body. We do this initially to clean up after other loop
566  // passes run, either when iterating on a loop or on inner loops with
567  // implications on the outer loop.
568  LPM1.addPass(LoopInstSimplifyPass());
569  LPM1.addPass(LoopSimplifyCFGPass());
570 
571  // Try to remove as much code from the loop header as possible,
572  // to reduce amount of IR that will have to be duplicated.
573  // TODO: Investigate promotion cap for O1.
574  LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
575 
576  LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true,
577  isLTOPreLink(Phase)));
578  // TODO: Investigate promotion cap for O1.
579  LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
580  LPM1.addPass(SimpleLoopUnswitchPass());
581 
582  LPM2.addPass(LoopIdiomRecognizePass());
583  LPM2.addPass(IndVarSimplifyPass());
584 
585  for (auto &C : LateLoopOptimizationsEPCallbacks)
586  C(LPM2, Level);
587 
588  LPM2.addPass(LoopDeletionPass());
589 
591  LPM2.addPass(LoopInterchangePass());
592 
593  // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
594  // because it changes IR to makes profile annotation in back compile
595  // inaccurate. The normal unroller doesn't pay attention to forced full unroll
596  // attributes so we need to make sure and allow the full unroll pass to pay
597  // attention to it.
598  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
599  PGOOpt->Action != PGOOptions::SampleUse)
600  LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
601  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
603 
604  for (auto &C : LoopOptimizerEndEPCallbacks)
605  C(LPM2, Level);
606 
607  // We provide the opt remark emitter pass for LICM to use. We only need to do
608  // this once as it is immutable.
609  FPM.addPass(
612  std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
613  DebugLogging));
614  FPM.addPass(SimplifyCFGPass());
615  FPM.addPass(InstCombinePass());
616  if (EnableLoopFlatten)
617  FPM.addPass(LoopFlattenPass());
618  // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
619  // *All* loop passes must preserve it, in order to be able to use it.
621  std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
622  DebugLogging));
623 
624  // Delete small array after loop unroll.
625  FPM.addPass(SROA());
626 
627  // Specially optimize memory movement as it doesn't look like dataflow in SSA.
628  FPM.addPass(MemCpyOptPass());
629 
630  // Sparse conditional constant propagation.
631  // FIXME: It isn't clear why we do this *after* loop passes rather than
632  // before...
633  FPM.addPass(SCCPPass());
634 
635  // Delete dead bit computations (instcombine runs after to fold away the dead
636  // computations, and then ADCE will run later to exploit any new DCE
637  // opportunities that creates).
638  FPM.addPass(BDCEPass());
639 
640  // Run instcombine after redundancy and dead bit elimination to exploit
641  // opportunities opened up by them.
642  FPM.addPass(InstCombinePass());
643  invokePeepholeEPCallbacks(FPM, Level);
644 
645  if (PTO.Coroutines)
646  FPM.addPass(CoroElidePass());
647 
648  for (auto &C : ScalarOptimizerLateEPCallbacks)
649  C(FPM, Level);
650 
651  // Finally, do an expensive DCE pass to catch all the dead code exposed by
652  // the simplifications and basic cleanup after all the simplifications.
653  // TODO: Investigate if this is too expensive.
654  FPM.addPass(ADCEPass());
655  FPM.addPass(SimplifyCFGPass());
656  FPM.addPass(InstCombinePass());
657  invokePeepholeEPCallbacks(FPM, Level);
658 
659  return FPM;
660 }
661 
665  assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
666 
667  // The O1 pipeline has a separate pipeline creation function to simplify
668  // construction readability.
669  if (Level.getSpeedupLevel() == 1)
670  return buildO1FunctionSimplificationPipeline(Level, Phase);
671 
672  FunctionPassManager FPM(DebugLogging);
673 
674  // Form SSA out of local memory accesses after breaking apart aggregates into
675  // scalars.
676  FPM.addPass(SROA());
677 
678  // Catch trivial redundancies
679  FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
682 
683  // Hoisting of scalars and load expressions.
684  if (EnableGVNHoist)
685  FPM.addPass(GVNHoistPass());
686 
687  // Global value numbering based sinking.
688  if (EnableGVNSink) {
689  FPM.addPass(GVNSinkPass());
690  FPM.addPass(SimplifyCFGPass());
691  }
692 
695 
696  // Speculative execution if the target has divergent branches; otherwise nop.
697  FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
698 
699  // Optimize based on known information about branches, and cleanup afterward.
700  FPM.addPass(JumpThreadingPass());
702 
703  FPM.addPass(SimplifyCFGPass());
706  FPM.addPass(InstCombinePass());
707 
708  if (!Level.isOptimizingForSize())
710 
711  invokePeepholeEPCallbacks(FPM, Level);
712 
713  // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
714  // using the size value profile. Don't perform this when optimizing for size.
715  if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
716  !Level.isOptimizingForSize())
717  FPM.addPass(PGOMemOPSizeOpt());
718 
719  FPM.addPass(TailCallElimPass());
720  FPM.addPass(SimplifyCFGPass());
721 
722  // Form canonically associated expression trees, and simplify the trees using
723  // basic mathematical properties. For example, this will form (nearly)
724  // minimal multiplication trees.
725  FPM.addPass(ReassociatePass());
726 
727  // Add the primary loop simplification pipeline.
728  // FIXME: Currently this is split into two loop pass pipelines because we run
729  // some function passes in between them. These can and should be removed
730  // and/or replaced by scheduling the loop pass equivalents in the correct
731  // positions. But those equivalent passes aren't powerful enough yet.
732  // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
733  // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
734  // fully replace `SimplifyCFGPass`, and the closest to the other we have is
735  // `LoopInstSimplify`.
736  LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
737 
738  // Simplify the loop body. We do this initially to clean up after other loop
739  // passes run, either when iterating on a loop or on inner loops with
740  // implications on the outer loop.
741  LPM1.addPass(LoopInstSimplifyPass());
742  LPM1.addPass(LoopSimplifyCFGPass());
743 
744  // Try to remove as much code from the loop header as possible,
745  // to reduce amount of IR that will have to be duplicated.
746  // TODO: Investigate promotion cap for O1.
747  LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
748 
749  // Disable header duplication in loop rotation at -Oz.
750  LPM1.addPass(
752  // TODO: Investigate promotion cap for O1.
753  LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
754  LPM1.addPass(
755  SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3 &&
758  LPM2.addPass(IndVarSimplifyPass());
759 
760  for (auto &C : LateLoopOptimizationsEPCallbacks)
761  C(LPM2, Level);
762 
763  LPM2.addPass(LoopDeletionPass());
764 
767 
768  // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
769  // because it changes IR to makes profile annotation in back compile
770  // inaccurate. The normal unroller doesn't pay attention to forced full unroll
771  // attributes so we need to make sure and allow the full unroll pass to pay
772  // attention to it.
773  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
774  PGOOpt->Action != PGOOptions::SampleUse)
775  LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
776  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
778 
779  for (auto &C : LoopOptimizerEndEPCallbacks)
780  C(LPM2, Level);
781 
782  // We provide the opt remark emitter pass for LICM to use. We only need to do
783  // this once as it is immutable.
784  FPM.addPass(
787  std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
788  DebugLogging));
789  FPM.addPass(SimplifyCFGPass());
790  FPM.addPass(InstCombinePass());
791  if (EnableLoopFlatten)
792  FPM.addPass(LoopFlattenPass());
793  // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
794  // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
795  // *All* loop passes must preserve it, in order to be able to use it.
797  std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
798  DebugLogging));
799 
800  // Delete small array after loop unroll.
801  FPM.addPass(SROA());
802 
803  // Eliminate redundancies.
805  if (RunNewGVN)
806  FPM.addPass(NewGVNPass());
807  else
808  FPM.addPass(GVN());
809 
810  // Sparse conditional constant propagation.
811  // FIXME: It isn't clear why we do this *after* loop passes rather than
812  // before...
813  FPM.addPass(SCCPPass());
814 
815  // Delete dead bit computations (instcombine runs after to fold away the dead
816  // computations, and then ADCE will run later to exploit any new DCE
817  // opportunities that creates).
818  FPM.addPass(BDCEPass());
819 
820  // Run instcombine after redundancy and dead bit elimination to exploit
821  // opportunities opened up by them.
822  FPM.addPass(InstCombinePass());
823  invokePeepholeEPCallbacks(FPM, Level);
824 
825  // Re-consider control flow based optimizations after redundancy elimination,
826  // redo DCE, etc.
827  FPM.addPass(JumpThreadingPass());
829 
830  // Finally, do an expensive DCE pass to catch all the dead code exposed by
831  // the simplifications and basic cleanup after all the simplifications.
832  // TODO: Investigate if this is too expensive.
833  FPM.addPass(ADCEPass());
834 
835  // Specially optimize memory movement as it doesn't look like dataflow in SSA.
836  FPM.addPass(MemCpyOptPass());
837 
838  FPM.addPass(DSEPass());
841  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
842 
843  if (PTO.Coroutines)
844  FPM.addPass(CoroElidePass());
845 
846  for (auto &C : ScalarOptimizerLateEPCallbacks)
847  C(FPM, Level);
848 
849  FPM.addPass(SimplifyCFGPass());
850  FPM.addPass(InstCombinePass());
851  invokePeepholeEPCallbacks(FPM, Level);
852 
853  if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
854  (PGOOpt->Action == PGOOptions::IRUse ||
855  PGOOpt->Action == PGOOptions::SampleUse))
857 
858  return FPM;
859 }
860 
861 void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
864 }
865 
866 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
868  bool RunProfileGen, bool IsCS,
869  std::string ProfileFile,
870  std::string ProfileRemappingFile) {
871  assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
872  if (!IsCS && !DisablePreInliner) {
874 
875  IP.DefaultThreshold = PreInlineThreshold;
876 
877  // FIXME: The hint threshold has the same value used by the regular inliner
878  // when not optimzing for size. This should probably be lowered after
879  // performance testing.
880  // FIXME: this comment is cargo culted from the old pass manager, revisit).
881  IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
882  ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
883  CGSCCPassManager &CGPipeline = MIWP.getPM();
884 
886  FPM.addPass(SROA());
887  FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
888  FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
889  FPM.addPass(InstCombinePass()); // Combine silly sequences.
890  invokePeepholeEPCallbacks(FPM, Level);
891 
893 
894  MPM.addPass(std::move(MIWP));
895 
896  // Delete anything that is now dead to make sure that we don't instrument
897  // dead code. Instrumentation can end up keeping dead code around and
898  // dramatically increase code size.
899  MPM.addPass(GlobalDCEPass());
900  }
901 
902  if (!RunProfileGen) {
903  assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
904  MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
905  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
906  // RequireAnalysisPass for PSI before subsequent non-module passes.
908  return;
909  }
910 
911  // Perform PGO instrumentation.
912  MPM.addPass(PGOInstrumentationGen(IsCS));
913 
915  // Disable header duplication in loop rotation at -Oz.
918  /*UseBlockFrequencyInfo=*/false, DebugLogging));
920 
921  // Add the profile lowering pass.
922  InstrProfOptions Options;
923  if (!ProfileFile.empty())
924  Options.InstrProfileOutput = ProfileFile;
925  // Do counter promotion at Level greater than O0.
926  Options.DoCounterPromotion = true;
927  Options.UseBFIInPromotion = IsCS;
928  MPM.addPass(InstrProfiling(Options, IsCS));
929 }
930 
932  bool RunProfileGen, bool IsCS,
933  std::string ProfileFile,
934  std::string ProfileRemappingFile) {
935  if (!RunProfileGen) {
936  assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
937  MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
938  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
939  // RequireAnalysisPass for PSI before subsequent non-module passes.
941  return;
942  }
943 
944  // Perform PGO instrumentation.
945  MPM.addPass(PGOInstrumentationGen(IsCS));
946  // Add the profile lowering pass.
947  InstrProfOptions Options;
948  if (!ProfileFile.empty())
949  Options.InstrProfileOutput = ProfileFile;
950  // Do not do counter promotion at O0.
951  Options.DoCounterPromotion = false;
952  Options.UseBFIInPromotion = IsCS;
953  MPM.addPass(InstrProfiling(Options, IsCS));
954 }
955 
956 static InlineParams
958  return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
959 }
960 
965  if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
966  PGOOpt->Action == PGOOptions::SampleUse)
967  IP.HotCallSiteThreshold = 0;
968 
969  if (PGOOpt)
970  IP.EnableDeferral = EnablePGOInlineDeferral;
971 
972  ModuleInlinerWrapperPass MIWP(IP, DebugLogging,
975 
976  // Require the GlobalsAA analysis for the module so we can query it within
977  // the CGSCC pipeline.
979 
980  // Require the ProfileSummaryAnalysis for the module so we can query it within
981  // the inliner pass.
983 
984  // Now begin the main postorder CGSCC pipeline.
985  // FIXME: The current CGSCC pipeline has its origins in the legacy pass
986  // manager and trying to emulate its precise behavior. Much of this doesn't
987  // make a lot of sense and we should revisit the core CGSCC structure.
988  CGSCCPassManager &MainCGPipeline = MIWP.getPM();
989 
990  // Note: historically, the PruneEH pass was run first to deduce nounwind and
991  // generally clean up exception handling overhead. It isn't clear this is
992  // valuable as the inliner doesn't currently care whether it is inlining an
993  // invoke or a call.
994 
996  MainCGPipeline.addPass(AttributorCGSCCPass());
997 
998  if (PTO.Coroutines)
999  MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
1000 
1001  // Now deduce any function attributes based in the current code.
1002  MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
1003 
1004  // When at O3 add argument promotion to the pass pipeline.
1005  // FIXME: It isn't at all clear why this should be limited to O3.
1007  MainCGPipeline.addPass(ArgumentPromotionPass());
1008 
1009  // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
1010  // there are no OpenMP runtime calls present in the module.
1012  MainCGPipeline.addPass(OpenMPOptPass());
1013 
1014  for (auto &C : CGSCCOptimizerLateEPCallbacks)
1015  C(MainCGPipeline, Level);
1016 
1017  // Lastly, add the core function simplification pipeline nested inside the
1018  // CGSCC walk.
1019  MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
1021 
1022  return MIWP;
1023 }
1024 
1028  ModulePassManager MPM(DebugLogging);
1029 
1030  // Place pseudo probe instrumentation as the first pass of the pipeline to
1031  // minimize the impact of optimization changes.
1032  if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1035 
1036  bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1037 
1038  // In ThinLTO mode, when flattened profile is used, all the available
1039  // profile information will be annotated in PreLink phase so there is
1040  // no need to load the profile again in PostLink.
1041  bool LoadSampleProfile =
1042  HasSampleProfile &&
1044 
1045  // During the ThinLTO backend phase we perform early indirect call promotion
1046  // here, before globalopt. Otherwise imported available_externally functions
1047  // look unreferenced and are removed. If we are going to load the sample
1048  // profile then defer until later.
1049  // TODO: See if we can move later and consolidate with the location where
1050  // we perform ICP when we are loading a sample profile.
1051  // TODO: We pass HasSampleProfile (whether there was a sample profile file
1052  // passed to the compile) to the SamplePGO flag of ICP. This is used to
1053  // determine whether the new direct calls are annotated with prof metadata.
1054  // Ideally this should be determined from whether the IR is annotated with
1055  // sample profile, and not whether the a sample profile was provided on the
1056  // command line. E.g. for flattened profiles where we will not be reloading
1057  // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1058  // provide the sample profile file.
1059  if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
1060  MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1061 
1062  // Do basic inference of function attributes from known properties of system
1063  // libraries and other oracles.
1065 
1066  // Create an early function pass manager to cleanup the output of the
1067  // frontend.
1068  FunctionPassManager EarlyFPM(DebugLogging);
1069  EarlyFPM.addPass(SimplifyCFGPass());
1070  EarlyFPM.addPass(SROA());
1071  EarlyFPM.addPass(EarlyCSEPass());
1072  EarlyFPM.addPass(LowerExpectIntrinsicPass());
1073  if (PTO.Coroutines)
1074  EarlyFPM.addPass(CoroEarlyPass());
1076  EarlyFPM.addPass(CallSiteSplittingPass());
1077 
1078  // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
1079  // to convert bitcast to direct calls so that they can be inlined during the
1080  // profile annotation prepration step.
1081  // More details about SamplePGO design can be found in:
1082  // https://research.google.com/pubs/pub45290.html
1083  // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
1084  if (LoadSampleProfile)
1085  EarlyFPM.addPass(InstCombinePass());
1087 
1088  if (LoadSampleProfile) {
1089  // Annotate sample profile right after early FPM to ensure freshness of
1090  // the debug info.
1091  MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1092  PGOOpt->ProfileRemappingFile, Phase));
1093  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1094  // RequireAnalysisPass for PSI before subsequent non-module passes.
1096  // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1097  // for the profile annotation to be accurate in the LTO backend.
1100  // We perform early indirect call promotion here, before globalopt.
1101  // This is important for the ThinLTO backend phase because otherwise
1102  // imported available_externally functions look unreferenced and are
1103  // removed.
1104  MPM.addPass(
1105  PGOIndirectCallPromotion(true /* IsInLTO */, true /* SamplePGO */));
1106  }
1107 
1109  MPM.addPass(AttributorPass());
1110 
1111  // Lower type metadata and the type.test intrinsic in the ThinLTO
1112  // post link pipeline after ICP. This is to enable usage of the type
1113  // tests in ICP sequences.
1115  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1116 
1117  for (auto &C : PipelineEarlySimplificationEPCallbacks)
1118  C(MPM, Level);
1119 
1120  // Interprocedural constant propagation now that basic cleanup has occurred
1121  // and prior to optimizing globals.
1122  // FIXME: This position in the pipeline hasn't been carefully considered in
1123  // years, it should be re-analyzed.
1124  MPM.addPass(IPSCCPPass());
1125 
1126  // Attach metadata to indirect call sites indicating the set of functions
1127  // they may target at run-time. This should follow IPSCCP.
1129 
1130  // Optimize globals to try and fold them into constants.
1131  MPM.addPass(GlobalOptPass());
1132 
1133  // Promote any localized globals to SSA registers.
1134  // FIXME: Should this instead by a run of SROA?
1135  // FIXME: We should probably run instcombine and simplify-cfg afterward to
1136  // delete control flows that are dead once globals have been folded to
1137  // constants.
1139 
1140  // Remove any dead arguments exposed by cleanups and constant folding
1141  // globals.
1143 
1144  // Create a small function pass pipeline to cleanup after all the global
1145  // optimizations.
1146  FunctionPassManager GlobalCleanupPM(DebugLogging);
1147  GlobalCleanupPM.addPass(InstCombinePass());
1148  invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1149 
1150  GlobalCleanupPM.addPass(SimplifyCFGPass());
1151  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1152 
1153  // Add all the requested passes for instrumentation PGO, if requested.
1154  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1155  (PGOOpt->Action == PGOOptions::IRInstr ||
1156  PGOOpt->Action == PGOOptions::IRUse)) {
1157  addPGOInstrPasses(MPM, Level,
1158  /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1159  /* IsCS */ false, PGOOpt->ProfileFile,
1160  PGOOpt->ProfileRemappingFile);
1161  MPM.addPass(PGOIndirectCallPromotion(false, false));
1162  }
1163  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1164  PGOOpt->CSAction == PGOOptions::CSIRInstr)
1165  MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1166 
1167  // Synthesize function entry counts for non-PGO compilation.
1168  if (EnableSyntheticCounts && !PGOOpt)
1170 
1172 
1176  }
1177 
1178  return MPM;
1179 }
1180 
1183  bool LTOPreLink) {
1184  ModulePassManager MPM(DebugLogging);
1185 
1186  // Optimize globals now that the module is fully simplified.
1187  MPM.addPass(GlobalOptPass());
1188  MPM.addPass(GlobalDCEPass());
1189 
1190  // Run partial inlining pass to partially inline functions that have
1191  // large bodies.
1192  if (RunPartialInlining)
1193  MPM.addPass(PartialInlinerPass());
1194 
1195  // Remove avail extern fns and globals definitions since we aren't compiling
1196  // an object file for later LTO. For LTO we want to preserve these so they
1197  // are eligible for inlining at link-time. Note if they are unreferenced they
1198  // will be removed by GlobalDCE later, so this only impacts referenced
1199  // available externally globals. Eventually they will be suppressed during
1200  // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1201  // may make globals referenced by available external functions dead and saves
1202  // running remaining passes on the eliminated functions. These should be
1203  // preserved during prelinking for link-time inlining decisions.
1204  if (!LTOPreLink)
1206 
1208  MPM.addPass(InstrOrderFilePass());
1209 
1210  // Do RPO function attribute inference across the module to forward-propagate
1211  // attributes where applicable.
1212  // FIXME: Is this really an optimization rather than a canonicalization?
1214 
1215  // Do a post inline PGO instrumentation and use pass. This is a context
1216  // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1217  // cross-module inline has not been done yet. The context sensitive
1218  // instrumentation is after all the inlines are done.
1219  if (!LTOPreLink && PGOOpt) {
1220  if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1221  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1222  /* IsCS */ true, PGOOpt->CSProfileGenFile,
1223  PGOOpt->ProfileRemappingFile);
1224  else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1225  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1226  /* IsCS */ true, PGOOpt->ProfileFile,
1227  PGOOpt->ProfileRemappingFile);
1228  }
1229 
1230  // Re-require GloblasAA here prior to function passes. This is particularly
1231  // useful as the above will have inlined, DCE'ed, and function-attr
1232  // propagated everything. We should at this point have a reasonably minimal
1233  // and richly annotated call graph. By computing aliasing and mod/ref
1234  // information for all local globals here, the late loop passes and notably
1235  // the vectorizer will be able to use them to help recognize vectorizable
1236  // memory operations.
1238 
1239  FunctionPassManager OptimizePM(DebugLogging);
1240  OptimizePM.addPass(Float2IntPass());
1241  OptimizePM.addPass(LowerConstantIntrinsicsPass());
1242 
1243  if (EnableMatrix) {
1244  OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1245  OptimizePM.addPass(EarlyCSEPass());
1246  }
1247 
1248  // FIXME: We need to run some loop optimizations to re-rotate loops after
1249  // simplify-cfg and others undo their rotation.
1250 
1251  // Optimize the loop execution. These passes operate on entire loop nests
1252  // rather than on each loop in an inside-out manner, and so they are actually
1253  // function passes.
1254 
1255  for (auto &C : VectorizerStartEPCallbacks)
1256  C(OptimizePM, Level);
1257 
1258  // First rotate loops that may have been un-rotated by prior passes.
1259  // Disable header duplication at -Oz.
1261  LoopRotatePass(Level != OptimizationLevel::Oz, LTOPreLink),
1263  /*UseBlockFrequencyInfo=*/false, DebugLogging));
1264 
1265  // Distribute loops to allow partial vectorization. I.e. isolate dependences
1266  // into separate loop that would otherwise inhibit vectorization. This is
1267  // currently only performed for loops marked with the metadata
1268  // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1269  OptimizePM.addPass(LoopDistributePass());
1270 
1271  // Populates the VFABI attribute with the scalar-to-vector mappings
1272  // from the TargetLibraryInfo.
1273  OptimizePM.addPass(InjectTLIMappings());
1274 
1275  // Now run the core loop vectorizer.
1276  OptimizePM.addPass(LoopVectorizePass(
1278 
1279  // Eliminate loads by forwarding stores from the previous iteration to loads
1280  // of the current iteration.
1281  OptimizePM.addPass(LoopLoadEliminationPass());
1282 
1283  // Cleanup after the loop optimization passes.
1284  OptimizePM.addPass(InstCombinePass());
1285 
1286  if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1287  // At higher optimization levels, try to clean up any runtime overlap and
1288  // alignment checks inserted by the vectorizer. We want to track correlated
1289  // runtime checks for two inner loops in the same outer loop, fold any
1290  // common computations, hoist loop-invariant aspects out of any outer loop,
1291  // and unswitch the runtime checks if possible. Once hoisted, we may have
1292  // dead (or speculatable) control flows or more combining opportunities.
1293  OptimizePM.addPass(EarlyCSEPass());
1295  OptimizePM.addPass(InstCombinePass());
1296  LoopPassManager LPM(DebugLogging);
1298  LPM.addPass(
1299  SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
1302  std::move(LPM), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
1303  DebugLogging));
1304  OptimizePM.addPass(SimplifyCFGPass());
1305  OptimizePM.addPass(InstCombinePass());
1306  }
1307 
1308  // Now that we've formed fast to execute loop structures, we do further
1309  // optimizations. These are run afterward as they might block doing complex
1310  // analyses and transforms such as what are needed for loop vectorization.
1311 
1312  // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1313  // GVN, loop transforms, and others have already run, so it's now better to
1314  // convert to more optimized IR using more aggressive simplify CFG options.
1315  // The extra sinking transform can create larger basic blocks, so do this
1316  // before SLP vectorization.
1317  // FIXME: study whether hoisting and/or sinking of common instructions should
1318  // be delayed until after SLP vectorizer.
1320  .forwardSwitchCondToPhi(true)
1321  .convertSwitchToLookupTable(true)
1322  .needCanonicalLoops(false)
1323  .hoistCommonInsts(true)
1324  .sinkCommonInsts(true)));
1325 
1326  // Optimize parallel scalar instruction chains into SIMD instructions.
1327  if (PTO.SLPVectorization) {
1328  OptimizePM.addPass(SLPVectorizerPass());
1329  if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1330  OptimizePM.addPass(EarlyCSEPass());
1331  }
1332  }
1333 
1334  // Enhance/cleanup vector code.
1335  OptimizePM.addPass(VectorCombinePass());
1336  OptimizePM.addPass(InstCombinePass());
1337 
1338  // Unroll small loops to hide loop backedge latency and saturate any parallel
1339  // execution resources of an out-of-order processor. We also then need to
1340  // clean up redundancies and loop invariant code.
1341  // FIXME: It would be really good to use a loop-integrated instruction
1342  // combiner for cleanup here so that the unrolling and LICM can be pipelined
1343  // across the loop nests.
1344  // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1345  if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1346  OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1347  }
1349  Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1351  OptimizePM.addPass(WarnMissedTransformationsPass());
1352  OptimizePM.addPass(InstCombinePass());
1356  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
1357 
1358  // Now that we've vectorized and unrolled loops, we may have more refined
1359  // alignment information, try to re-derive it here.
1360  OptimizePM.addPass(AlignmentFromAssumptionsPass());
1361 
1362  // Split out cold code. Splitting is done late to avoid hiding context from
1363  // other optimizations and inadvertently regressing performance. The tradeoff
1364  // is that this has a higher code size cost than splitting early.
1365  if (EnableHotColdSplit && !LTOPreLink)
1367 
1368  // Search the code for similar regions of code. If enough similar regions can
1369  // be found where extracting the regions into their own function will decrease
1370  // the size of the program, we extract the regions, a deduplicate the
1371  // structurally similar regions.
1372  if (EnableIROutliner)
1373  MPM.addPass(IROutlinerPass());
1374 
1375  // Merge functions if requested.
1376  if (PTO.MergeFunctions)
1377  MPM.addPass(MergeFunctionsPass());
1378 
1379  // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1380  // canonicalization pass that enables other optimizations. As a result,
1381  // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1382  // result too early.
1383  OptimizePM.addPass(LoopSinkPass());
1384 
1385  // And finally clean up LCSSA form before generating code.
1386  OptimizePM.addPass(InstSimplifyPass());
1387 
1388  // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1389  // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1390  // flattening of blocks.
1391  OptimizePM.addPass(DivRemPairsPass());
1392 
1393  // LoopSink (and other loop passes since the last simplifyCFG) might have
1394  // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1395  OptimizePM.addPass(SimplifyCFGPass());
1396 
1397  // Optimize PHIs by speculating around them when profitable. Note that this
1398  // pass needs to be run after any PRE or similar pass as it is essentially
1399  // inserting redundancies into the program. This even includes SimplifyCFG.
1400  OptimizePM.addPass(SpeculateAroundPHIsPass());
1401 
1402  if (PTO.Coroutines)
1403  OptimizePM.addPass(CoroCleanupPass());
1404 
1405  // Add the core optimizing pipeline.
1407 
1408  for (auto &C : OptimizerLastEPCallbacks)
1409  C(MPM, Level);
1410 
1411  if (PTO.CallGraphProfile)
1412  MPM.addPass(CGProfilePass());
1413 
1414  // Now we need to do some global optimization transforms.
1415  // FIXME: It would seem like these should come first in the optimization
1416  // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1417  // ordering here.
1418  MPM.addPass(GlobalDCEPass());
1419  MPM.addPass(ConstantMergePass());
1420 
1421  return MPM;
1422 }
1423 
1426  bool LTOPreLink) {
1428  "Must request optimizations for the default pipeline!");
1429 
1430  ModulePassManager MPM(DebugLogging);
1431 
1432  // Convert @llvm.global.annotations to !annotation metadata.
1434 
1435  // Force any function attributes we want the rest of the pipeline to observe.
1437 
1438  // Apply module pipeline start EP callback.
1439  for (auto &C : PipelineStartEPCallbacks)
1440  C(MPM, Level);
1441 
1442  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1444 
1445  // Add the core simplification pipeline.
1449 
1450  // Now add the optimization pipeline.
1451  MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink));
1452 
1453  if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1455 
1456  // Emit annotation remarks.
1458 
1459  if (LTOPreLink)
1460  addRequiredLTOPreLinkPasses(MPM);
1461 
1462  return MPM;
1463 }
1464 
1468  "Must request optimizations for the default pipeline!");
1469 
1470  ModulePassManager MPM(DebugLogging);
1471 
1472  // Convert @llvm.global.annotations to !annotation metadata.
1474 
1475  // Force any function attributes we want the rest of the pipeline to observe.
1477 
1478  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1480 
1481  // Apply module pipeline start EP callback.
1482  for (auto &C : PipelineStartEPCallbacks)
1483  C(MPM, Level);
1484 
1485  // If we are planning to perform ThinLTO later, we don't bloat the code with
1486  // unrolling/vectorization/... now. Just simplify the module as much as we
1487  // can.
1490 
1491  // Run partial inlining pass to partially inline functions that have
1492  // large bodies.
1493  // FIXME: It isn't clear whether this is really the right place to run this
1494  // in ThinLTO. Because there is another canonicalization and simplification
1495  // phase that will run after the thin link, running this here ends up with
1496  // less information than will be available later and it may grow functions in
1497  // ways that aren't beneficial.
1498  if (RunPartialInlining)
1499  MPM.addPass(PartialInlinerPass());
1500 
1501  // Reduce the size of the IR as much as possible.
1502  MPM.addPass(GlobalOptPass());
1503 
1504  // Module simplification splits coroutines, but does not fully clean up
1505  // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1506  // on these, we schedule the cleanup here.
1507  if (PTO.Coroutines)
1509 
1510  if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
1512 
1513  // Handle OptimizerLastEPCallbacks added by clang on PreLink. Actual
1514  // optimization is going to be done in PostLink stage, but clang can't
1515  // add callbacks there in case of in-process ThinLTO called by linker.
1516  for (auto &C : OptimizerLastEPCallbacks)
1517  C(MPM, Level);
1518 
1519  // Emit annotation remarks.
1521 
1522  addRequiredLTOPreLinkPasses(MPM);
1523 
1524  return MPM;
1525 }
1526 
1528  OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1529  ModulePassManager MPM(DebugLogging);
1530 
1531  // Convert @llvm.global.annotations to !annotation metadata.
1533 
1534  if (ImportSummary) {
1535  // These passes import type identifier resolutions for whole-program
1536  // devirtualization and CFI. They must run early because other passes may
1537  // disturb the specific instruction patterns that these passes look for,
1538  // creating dependencies on resolutions that may not appear in the summary.
1539  //
1540  // For example, GVN may transform the pattern assume(type.test) appearing in
1541  // two basic blocks into assume(phi(type.test, type.test)), which would
1542  // transform a dependency on a WPD resolution into a dependency on a type
1543  // identifier resolution for CFI.
1544  //
1545  // Also, WPD has access to more precise information than ICP and can
1546  // devirtualize more effectively, so it should operate on the IR first.
1547  //
1548  // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1549  // metadata and intrinsics.
1550  MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1551  MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1552  }
1553 
1554  if (Level == OptimizationLevel::O0) {
1555  // Run a second time to clean up any type tests left behind by WPD for use
1556  // in ICP.
1557  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1558  // Drop available_externally and unreferenced globals. This is necessary
1559  // with ThinLTO in order to avoid leaving undefined references to dead
1560  // globals in the object file.
1562  MPM.addPass(GlobalDCEPass());
1563  return MPM;
1564  }
1565 
1566  // Force any function attributes we want the rest of the pipeline to observe.
1568 
1569  // Add the core simplification pipeline.
1572 
1573  // Now add the optimization pipeline.
1575 
1576  // Emit annotation remarks.
1578 
1579  return MPM;
1580 }
1581 
1585  "Must request optimizations for the default pipeline!");
1586  // FIXME: We should use a customized pre-link pipeline!
1588  /* LTOPreLink */ true);
1589 }
1590 
1593  ModuleSummaryIndex *ExportSummary) {
1594  ModulePassManager MPM(DebugLogging);
1595 
1596  // Convert @llvm.global.annotations to !annotation metadata.
1598 
1599  if (Level == OptimizationLevel::O0) {
1600  // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1601  // metadata and intrinsics.
1602  MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1603  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1604  // Run a second time to clean up any type tests left behind by WPD for use
1605  // in ICP.
1606  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1607 
1608  // Emit annotation remarks.
1610 
1611  return MPM;
1612  }
1613 
1614  if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1615  // Load sample profile before running the LTO optimization pipeline.
1616  MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1617  PGOOpt->ProfileRemappingFile,
1619  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1620  // RequireAnalysisPass for PSI before subsequent non-module passes.
1622  }
1623 
1624  // Remove unused virtual tables to improve the quality of code generated by
1625  // whole-program devirtualization and bitset lowering.
1626  MPM.addPass(GlobalDCEPass());
1627 
1628  // Force any function attributes we want the rest of the pipeline to observe.
1630 
1631  // Do basic inference of function attributes from known properties of system
1632  // libraries and other oracles.
1634 
1635  if (Level.getSpeedupLevel() > 1) {
1636  FunctionPassManager EarlyFPM(DebugLogging);
1637  EarlyFPM.addPass(CallSiteSplittingPass());
1639 
1640  // Indirect call promotion. This should promote all the targets that are
1641  // left by the earlier promotion pass that promotes intra-module targets.
1642  // This two-step promotion is to save the compile time. For LTO, it should
1643  // produce the same result as if we only do promotion here.
1645  true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1646  // Propagate constants at call sites into the functions they call. This
1647  // opens opportunities for globalopt (and inlining) by substituting function
1648  // pointers passed as arguments to direct uses of functions.
1649  MPM.addPass(IPSCCPPass());
1650 
1651  // Attach metadata to indirect call sites indicating the set of functions
1652  // they may target at run-time. This should follow IPSCCP.
1654  }
1655 
1656  // Now deduce any function attributes based in the current code.
1659 
1660  // Do RPO function attribute inference across the module to forward-propagate
1661  // attributes where applicable.
1662  // FIXME: Is this really an optimization rather than a canonicalization?
1664 
1665  // Use in-range annotations on GEP indices to split globals where beneficial.
1666  MPM.addPass(GlobalSplitPass());
1667 
1668  // Run whole program optimization of virtual call when the list of callees
1669  // is fixed.
1670  MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1671 
1672  // Stop here at -O1.
1673  if (Level == OptimizationLevel::O1) {
1674  // The LowerTypeTestsPass needs to run to lower type metadata and the
1675  // type.test intrinsics. The pass does nothing if CFI is disabled.
1676  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1677  // Run a second time to clean up any type tests left behind by WPD for use
1678  // in ICP (which is performed earlier than this in the regular LTO
1679  // pipeline).
1680  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1681 
1682  // Emit annotation remarks.
1684 
1685  return MPM;
1686  }
1687 
1688  // Optimize globals to try and fold them into constants.
1689  MPM.addPass(GlobalOptPass());
1690 
1691  // Promote any localized globals to SSA registers.
1693 
1694  // Linking modules together can lead to duplicate global constant, only
1695  // keep one copy of each constant.
1696  MPM.addPass(ConstantMergePass());
1697 
1698  // Remove unused arguments from functions.
1700 
1701  // Reduce the code after globalopt and ipsccp. Both can open up significant
1702  // simplification opportunities, and both can propagate functions through
1703  // function pointers. When this happens, we often have to resolve varargs
1704  // calls, etc, so let instcombine do this.
1705  FunctionPassManager PeepholeFPM(DebugLogging);
1707  PeepholeFPM.addPass(AggressiveInstCombinePass());
1708  PeepholeFPM.addPass(InstCombinePass());
1709  invokePeepholeEPCallbacks(PeepholeFPM, Level);
1710 
1712 
1713  // Note: historically, the PruneEH pass was run first to deduce nounwind and
1714  // generally clean up exception handling overhead. It isn't clear this is
1715  // valuable as the inliner doesn't currently care whether it is inlining an
1716  // invoke or a call.
1717  // Run the inliner now.
1719  DebugLogging));
1720 
1721  // Optimize globals again after we ran the inliner.
1722  MPM.addPass(GlobalOptPass());
1723 
1724  // Garbage collect dead functions.
1725  // FIXME: Add ArgumentPromotion pass after once it's ported.
1726  MPM.addPass(GlobalDCEPass());
1727 
1728  FunctionPassManager FPM(DebugLogging);
1729  // The IPO Passes may leave cruft around. Clean up after them.
1730  FPM.addPass(InstCombinePass());
1731  invokePeepholeEPCallbacks(FPM, Level);
1732 
1733  FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1734 
1735  // Do a post inline PGO instrumentation and use pass. This is a context
1736  // sensitive PGO pass.
1737  if (PGOOpt) {
1738  if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1739  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1740  /* IsCS */ true, PGOOpt->CSProfileGenFile,
1741  PGOOpt->ProfileRemappingFile);
1742  else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1743  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1744  /* IsCS */ true, PGOOpt->ProfileFile,
1745  PGOOpt->ProfileRemappingFile);
1746  }
1747 
1748  // Break up allocas
1749  FPM.addPass(SROA());
1750 
1751  // LTO provides additional opportunities for tailcall elimination due to
1752  // link-time inlining, and visibility of nocapture attribute.
1753  FPM.addPass(TailCallElimPass());
1754 
1755  // Run a few AA driver optimizations here and now to cleanup the code.
1757 
1758  MPM.addPass(
1760  // FIXME: here we run IP alias analysis in the legacy PM.
1761 
1762  FunctionPassManager MainFPM;
1763 
1766 
1767  if (RunNewGVN)
1768  MainFPM.addPass(NewGVNPass());
1769  else
1770  MainFPM.addPass(GVN());
1771 
1772  // Remove dead memcpy()'s.
1773  MainFPM.addPass(MemCpyOptPass());
1774 
1775  // Nuke dead stores.
1776  MainFPM.addPass(DSEPass());
1778 
1779  // More loops are countable; try to optimize them.
1780  if (EnableLoopFlatten && Level.getSpeedupLevel() > 1)
1781  MainFPM.addPass(LoopFlattenPass());
1782 
1785 
1786  LoopPassManager LPM(DebugLogging);
1787  LPM.addPass(IndVarSimplifyPass());
1788  LPM.addPass(LoopDeletionPass());
1789  // FIXME: Add loop interchange.
1790 
1791  // Unroll small loops and perform peeling.
1792  LPM.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
1793  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
1795  // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
1796  // *All* loop passes must preserve it, in order to be able to use it.
1798  std::move(LPM), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/true,
1799  DebugLogging));
1800 
1801  MainFPM.addPass(LoopDistributePass());
1802  MainFPM.addPass(LoopVectorizePass(
1804  // The vectorizer may have significantly shortened a loop body; unroll again.
1806  Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1808 
1810 
1811  MainFPM.addPass(InstCombinePass());
1812  MainFPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true)));
1813  MainFPM.addPass(SCCPPass());
1814  MainFPM.addPass(InstCombinePass());
1815  MainFPM.addPass(BDCEPass());
1816 
1817  // More scalar chains could be vectorized due to more alias information
1818  if (PTO.SLPVectorization) {
1819  MainFPM.addPass(SLPVectorizerPass());
1820  if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
1821  MainFPM.addPass(EarlyCSEPass());
1822  }
1823  }
1824 
1825  MainFPM.addPass(VectorCombinePass()); // Clean up partial vectorization.
1826 
1827  // After vectorization, assume intrinsics may tell us more about pointer
1828  // alignments.
1830 
1831  // FIXME: Conditionally run LoadCombine here, after it's ported
1832  // (in case we still have this pass, given its questionable usefulness).
1833 
1834  MainFPM.addPass(InstCombinePass());
1835  invokePeepholeEPCallbacks(MainFPM, Level);
1836  MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1838 
1839  // Create a function that performs CFI checks for cross-DSO calls with
1840  // targets in the current module.
1841  MPM.addPass(CrossDSOCFIPass());
1842 
1843  // Lower type metadata and the type.test intrinsic. This pass supports
1844  // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1845  // to be run at link time if CFI is enabled. This pass does nothing if
1846  // CFI is disabled.
1847  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1848  // Run a second time to clean up any type tests left behind by WPD for use
1849  // in ICP (which is performed earlier than this in the regular LTO pipeline).
1850  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1851 
1852  // Enable splitting late in the FullLTO post-link pipeline. This is done in
1853  // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1854  if (EnableHotColdSplit)
1856 
1857  // Add late LTO optimization passes.
1858  // Delete basic blocks, which optimization passes may have killed.
1860  SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))));
1861 
1862  // Drop bodies of available eternally objects to improve GlobalDCE.
1864 
1865  // Now that we have optimized the program, discard unreachable functions.
1866  MPM.addPass(GlobalDCEPass());
1867 
1868  if (PTO.MergeFunctions)
1869  MPM.addPass(MergeFunctionsPass());
1870 
1871  // Emit annotation remarks.
1873 
1874  return MPM;
1875 }
1876 
1878  bool LTOPreLink) {
1880  "buildO0DefaultPipeline should only be used with O0");
1881 
1882  ModulePassManager MPM(DebugLogging);
1883 
1884  if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
1885  PGOOpt->Action == PGOOptions::IRUse))
1887  MPM,
1888  /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1889  /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1890 
1891  for (auto &C : PipelineStartEPCallbacks)
1892  C(MPM, Level);
1893  for (auto &C : PipelineEarlySimplificationEPCallbacks)
1894  C(MPM, Level);
1895 
1896  // Build a minimal pipeline based on the semantics required by LLVM,
1897  // which is just that always inlining occurs. Further, disable generating
1898  // lifetime intrinsics to avoid enabling further optimizations during
1899  // code generation.
1900  // However, we need to insert lifetime intrinsics to avoid invalid access
1901  // caused by multithreaded coroutines.
1903  /*InsertLifetimeIntrinsics=*/PTO.Coroutines));
1904 
1905  if (PTO.MergeFunctions)
1906  MPM.addPass(MergeFunctionsPass());
1907 
1908  if (EnableMatrix)
1909  MPM.addPass(
1911 
1912  if (!CGSCCOptimizerLateEPCallbacks.empty()) {
1913  CGSCCPassManager CGPM(DebugLogging);
1914  for (auto &C : CGSCCOptimizerLateEPCallbacks)
1915  C(CGPM, Level);
1916  if (!CGPM.isEmpty())
1918  }
1919  if (!LateLoopOptimizationsEPCallbacks.empty()) {
1920  LoopPassManager LPM(DebugLogging);
1921  for (auto &C : LateLoopOptimizationsEPCallbacks)
1922  C(LPM, Level);
1923  if (!LPM.isEmpty()) {
1926  }
1927  }
1928  if (!LoopOptimizerEndEPCallbacks.empty()) {
1929  LoopPassManager LPM(DebugLogging);
1930  for (auto &C : LoopOptimizerEndEPCallbacks)
1931  C(LPM, Level);
1932  if (!LPM.isEmpty()) {
1935  }
1936  }
1937  if (!ScalarOptimizerLateEPCallbacks.empty()) {
1938  FunctionPassManager FPM(DebugLogging);
1939  for (auto &C : ScalarOptimizerLateEPCallbacks)
1940  C(FPM, Level);
1941  if (!FPM.isEmpty())
1943  }
1944  if (!VectorizerStartEPCallbacks.empty()) {
1945  FunctionPassManager FPM(DebugLogging);
1946  for (auto &C : VectorizerStartEPCallbacks)
1947  C(FPM, Level);
1948  if (!FPM.isEmpty())
1950  }
1951 
1952  if (PTO.Coroutines) {
1954 
1955  CGSCCPassManager CGPM(DebugLogging);
1956  CGPM.addPass(CoroSplitPass());
1959 
1961  }
1962 
1963  for (auto &C : OptimizerLastEPCallbacks)
1964  C(MPM, Level);
1965 
1966  if (LTOPreLink)
1967  addRequiredLTOPreLinkPasses(MPM);
1968 
1969  return MPM;
1970 }
1971 
1973  AAManager AA;
1974 
1975  // The order in which these are registered determines their priority when
1976  // being queried.
1977 
1978  // First we register the basic alias analysis that provides the majority of
1979  // per-function local AA logic. This is a stateless, on-demand local set of
1980  // AA techniques.
1982 
1983  // Next we query fast, specialized alias analyses that wrap IR-embedded
1984  // information about aliasing.
1987 
1988  // Add support for querying global aliasing information when available.
1989  // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1990  // analysis, all that the `AAManager` can do is query for any *cached*
1991  // results from `GlobalsAA` through a readonly proxy.
1993 
1994  // Add target-specific alias analyses.
1995  if (TM)
1997 
1998  return AA;
1999 }
2000 
2002  if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
2003  return None;
2004  int Count;
2005  if (Name.getAsInteger(0, Count) || Count <= 0)
2006  return None;
2007  return Count;
2008 }
2009 
2011  if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
2012  return None;
2013  int Count;
2014  if (Name.getAsInteger(0, Count) || Count < 0)
2015  return None;
2016  return Count;
2017 }
2018 
2020  if (!Name.consume_front(PassName))
2021  return false;
2022  // normal pass name w/o parameters == default parameters
2023  if (Name.empty())
2024  return true;
2025  return Name.startswith("<") && Name.endswith(">");
2026 }
2027 
2028 namespace {
2029 
2030 /// This performs customized parsing of pass name with parameters.
2031 ///
2032 /// We do not need parametrization of passes in textual pipeline very often,
2033 /// yet on a rare occasion ability to specify parameters right there can be
2034 /// useful.
2035 ///
2036 /// \p Name - parameterized specification of a pass from a textual pipeline
2037 /// is a string in a form of :
2038 /// PassName '<' parameter-list '>'
2039 ///
2040 /// Parameter list is being parsed by the parser callable argument, \p Parser,
2041 /// It takes a string-ref of parameters and returns either StringError or a
2042 /// parameter list in a form of a custom parameters type, all wrapped into
2043 /// Expected<> template class.
2044 ///
2045 template <typename ParametersParseCallableT>
2046 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
2047  StringRef PassName) -> decltype(Parser(StringRef{})) {
2048  using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
2049 
2050  StringRef Params = Name;
2051  if (!Params.consume_front(PassName)) {
2052  assert(false &&
2053  "unable to strip pass name from parametrized pass specification");
2054  }
2055  if (Params.empty())
2056  return ParametersT{};
2057  if (!Params.consume_front("<") || !Params.consume_back(">")) {
2058  assert(false && "invalid format for parametrized pass name");
2059  }
2060 
2061  Expected<ParametersT> Result = Parser(Params);
2062  assert((Result || Result.template errorIsA<StringError>()) &&
2063  "Pass parameter parser can only return StringErrors.");
2064  return Result;
2065 }
2066 
2067 /// Parser of parameters for LoopUnroll pass.
2068 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
2069  LoopUnrollOptions UnrollOpts;
2070  while (!Params.empty()) {
2071  StringRef ParamName;
2072  std::tie(ParamName, Params) = Params.split(';');
2073  int OptLevel = StringSwitch<int>(ParamName)
2074  .Case("O0", 0)
2075  .Case("O1", 1)
2076  .Case("O2", 2)
2077  .Case("O3", 3)
2078  .Default(-1);
2079  if (OptLevel >= 0) {
2080  UnrollOpts.setOptLevel(OptLevel);
2081  continue;
2082  }
2083  if (ParamName.consume_front("full-unroll-max=")) {
2084  int Count;
2085  if (ParamName.getAsInteger(0, Count))
2086  return make_error<StringError>(
2087  formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2089  UnrollOpts.setFullUnrollMaxCount(Count);
2090  continue;
2091  }
2092 
2093  bool Enable = !ParamName.consume_front("no-");
2094  if (ParamName == "partial") {
2095  UnrollOpts.setPartial(Enable);
2096  } else if (ParamName == "peeling") {
2097  UnrollOpts.setPeeling(Enable);
2098  } else if (ParamName == "profile-peeling") {
2099  UnrollOpts.setProfileBasedPeeling(Enable);
2100  } else if (ParamName == "runtime") {
2101  UnrollOpts.setRuntime(Enable);
2102  } else if (ParamName == "upperbound") {
2103  UnrollOpts.setUpperBound(Enable);
2104  } else {
2105  return make_error<StringError>(
2106  formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2108  }
2109  }
2110  return UnrollOpts;
2111 }
2112 
2113 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
2115  while (!Params.empty()) {
2116  StringRef ParamName;
2117  std::tie(ParamName, Params) = Params.split(';');
2118 
2119  if (ParamName == "recover") {
2120  Result.Recover = true;
2121  } else if (ParamName == "kernel") {
2122  Result.Kernel = true;
2123  } else if (ParamName.consume_front("track-origins=")) {
2124  if (ParamName.getAsInteger(0, Result.TrackOrigins))
2125  return make_error<StringError>(
2126  formatv("invalid argument to MemorySanitizer pass track-origins "
2127  "parameter: '{0}' ",
2128  ParamName)
2129  .str(),
2131  } else {
2132  return make_error<StringError>(
2133  formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
2134  .str(),
2136  }
2137  }
2138  return Result;
2139 }
2140 
2141 /// Parser of parameters for SimplifyCFG pass.
2142 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
2144  while (!Params.empty()) {
2145  StringRef ParamName;
2146  std::tie(ParamName, Params) = Params.split(';');
2147 
2148  bool Enable = !ParamName.consume_front("no-");
2149  if (ParamName == "forward-switch-cond") {
2150  Result.forwardSwitchCondToPhi(Enable);
2151  } else if (ParamName == "switch-to-lookup") {
2152  Result.convertSwitchToLookupTable(Enable);
2153  } else if (ParamName == "keep-loops") {
2154  Result.needCanonicalLoops(Enable);
2155  } else if (ParamName == "hoist-common-insts") {
2156  Result.hoistCommonInsts(Enable);
2157  } else if (ParamName == "sink-common-insts") {
2158  Result.sinkCommonInsts(Enable);
2159  } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
2160  APInt BonusInstThreshold;
2161  if (ParamName.getAsInteger(0, BonusInstThreshold))
2162  return make_error<StringError>(
2163  formatv("invalid argument to SimplifyCFG pass bonus-threshold "
2164  "parameter: '{0}' ",
2165  ParamName).str(),
2167  Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
2168  } else {
2169  return make_error<StringError>(
2170  formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
2172  }
2173  }
2174  return Result;
2175 }
2176 
2177 /// Parser of parameters for LoopVectorize pass.
2178 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
2179  LoopVectorizeOptions Opts;
2180  while (!Params.empty()) {
2181  StringRef ParamName;
2182  std::tie(ParamName, Params) = Params.split(';');
2183 
2184  bool Enable = !ParamName.consume_front("no-");
2185  if (ParamName == "interleave-forced-only") {
2186  Opts.setInterleaveOnlyWhenForced(Enable);
2187  } else if (ParamName == "vectorize-forced-only") {
2188  Opts.setVectorizeOnlyWhenForced(Enable);
2189  } else {
2190  return make_error<StringError>(
2191  formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
2193  }
2194  }
2195  return Opts;
2196 }
2197 
2198 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
2199  bool Result = false;
2200  while (!Params.empty()) {
2201  StringRef ParamName;
2202  std::tie(ParamName, Params) = Params.split(';');
2203 
2204  bool Enable = !ParamName.consume_front("no-");
2205  if (ParamName == "nontrivial") {
2206  Result = Enable;
2207  } else {
2208  return make_error<StringError>(
2209  formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
2210  .str(),
2212  }
2213  }
2214  return Result;
2215 }
2216 
2217 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
2218  bool Result = false;
2219  while (!Params.empty()) {
2220  StringRef ParamName;
2221  std::tie(ParamName, Params) = Params.split(';');
2222 
2223  bool Enable = !ParamName.consume_front("no-");
2224  if (ParamName == "split-footer-bb") {
2225  Result = Enable;
2226  } else {
2227  return make_error<StringError>(
2228  formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
2229  ParamName)
2230  .str(),
2232  }
2233  }
2234  return Result;
2235 }
2236 
2237 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
2239  while (!Params.empty()) {
2240  StringRef ParamName;
2241  std::tie(ParamName, Params) = Params.split(';');
2242 
2243  bool Enable = !ParamName.consume_front("no-");
2244  if (ParamName == "pre") {
2245  Result.setPRE(Enable);
2246  } else if (ParamName == "load-pre") {
2247  Result.setLoadPRE(Enable);
2248  } else if (ParamName == "split-backedge-load-pre") {
2249  Result.setLoadPRESplitBackedge(Enable);
2250  } else if (ParamName == "memdep") {
2251  Result.setMemDep(Enable);
2252  } else {
2253  return make_error<StringError>(
2254  formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
2256  }
2257  }
2258  return Result;
2259 }
2260 
2262 parseStackLifetimeOptions(StringRef Params) {
2264  while (!Params.empty()) {
2265  StringRef ParamName;
2266  std::tie(ParamName, Params) = Params.split(';');
2267 
2268  if (ParamName == "may") {
2270  } else if (ParamName == "must") {
2272  } else {
2273  return make_error<StringError>(
2274  formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
2276  }
2277  }
2278  return Result;
2279 }
2280 
2281 } // namespace
2282 
2283 /// Tests whether a pass name starts with a valid prefix for a default pipeline
2284 /// alias.
2286  return Name.startswith("default") || Name.startswith("thinlto") ||
2287  Name.startswith("lto");
2288 }
2289 
2290 /// Tests whether registered callbacks will accept a given pass name.
2291 ///
2292 /// When parsing a pipeline text, the type of the outermost pipeline may be
2293 /// omitted, in which case the type is automatically determined from the first
2294 /// pass name in the text. This may be a name that is handled through one of the
2295 /// callbacks. We check this through the oridinary parsing callbacks by setting
2296 /// up a dummy PassManager in order to not force the client to also handle this
2297 /// type of query.
2298 template <typename PassManagerT, typename CallbacksT>
2299 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
2300  if (!Callbacks.empty()) {
2301  PassManagerT DummyPM;
2302  for (auto &CB : Callbacks)
2303  if (CB(Name, DummyPM, {}))
2304  return true;
2305  }
2306  return false;
2307 }
2308 
2309 template <typename CallbacksT>
2310 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
2311  // Manually handle aliases for pre-configured pipeline fragments.
2313  return DefaultAliasRegex.match(Name);
2314 
2315  // Explicitly handle pass manager names.
2316  if (Name == "module")
2317  return true;
2318  if (Name == "cgscc")
2319  return true;
2320  if (Name == "function")
2321  return true;
2322 
2323  // Explicitly handle custom-parsed pass names.
2325  return true;
2326 
2327 #define MODULE_PASS(NAME, CREATE_PASS) \
2328  if (Name == NAME) \
2329  return true;
2330 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2331  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2332  return true;
2333 #include "PassRegistry.def"
2334 
2335  return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
2336 }
2337 
2338 template <typename CallbacksT>
2339 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2340  // Explicitly handle pass manager names.
2341  if (Name == "cgscc")
2342  return true;
2343  if (Name == "function")
2344  return true;
2345 
2346  // Explicitly handle custom-parsed pass names.
2348  return true;
2350  return true;
2351 
2352 #define CGSCC_PASS(NAME, CREATE_PASS) \
2353  if (Name == NAME) \
2354  return true;
2355 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2356  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2357  return true;
2358 #include "PassRegistry.def"
2359 
2360  return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2361 }
2362 
2363 template <typename CallbacksT>
2364 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2365  // Explicitly handle pass manager names.
2366  if (Name == "function")
2367  return true;
2368  if (Name == "loop" || Name == "loop-mssa")
2369  return true;
2370 
2371  // Explicitly handle custom-parsed pass names.
2373  return true;
2374 
2375 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2376  if (Name == NAME) \
2377  return true;
2378 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2379  if (checkParametrizedPassName(Name, NAME)) \
2380  return true;
2381 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2382  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2383  return true;
2384 #include "PassRegistry.def"
2385 
2386  return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2387 }
2388 
2389 template <typename CallbacksT>
2390 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2391  // Explicitly handle pass manager names.
2392  if (Name == "loop" || Name == "loop-mssa")
2393  return true;
2394 
2395  // Explicitly handle custom-parsed pass names.
2397  return true;
2398 
2399 #define LOOP_PASS(NAME, CREATE_PASS) \
2400  if (Name == NAME) \
2401  return true;
2402 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2403  if (checkParametrizedPassName(Name, NAME)) \
2404  return true;
2405 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2406  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2407  return true;
2408 #include "PassRegistry.def"
2409 
2410  return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2411 }
2412 
2414 PassBuilder::parsePipelineText(StringRef Text) {
2415  std::vector<PipelineElement> ResultPipeline;
2416 
2417  SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2418  &ResultPipeline};
2419  for (;;) {
2420  std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2421  size_t Pos = Text.find_first_of(",()");
2422  Pipeline.push_back({Text.substr(0, Pos), {}});
2423 
2424  // If we have a single terminating name, we're done.
2425  if (Pos == Text.npos)
2426  break;
2427 
2428  char Sep = Text[Pos];
2429  Text = Text.substr(Pos + 1);
2430  if (Sep == ',')
2431  // Just a name ending in a comma, continue.
2432  continue;
2433 
2434  if (Sep == '(') {
2435  // Push the inner pipeline onto the stack to continue processing.
2436  PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2437  continue;
2438  }
2439 
2440  assert(Sep == ')' && "Bogus separator!");
2441  // When handling the close parenthesis, we greedily consume them to avoid
2442  // empty strings in the pipeline.
2443  do {
2444  // If we try to pop the outer pipeline we have unbalanced parentheses.
2445  if (PipelineStack.size() == 1)
2446  return None;
2447 
2448  PipelineStack.pop_back();
2449  } while (Text.consume_front(")"));
2450 
2451  // Check if we've finished parsing.
2452  if (Text.empty())
2453  break;
2454 
2455  // Otherwise, the end of an inner pipeline always has to be followed by
2456  // a comma, and then we can continue.
2457  if (!Text.consume_front(","))
2458  return None;
2459  }
2460 
2461  if (PipelineStack.size() > 1)
2462  // Unbalanced paretheses.
2463  return None;
2464 
2465  assert(PipelineStack.back() == &ResultPipeline &&
2466  "Wrong pipeline at the bottom of the stack!");
2467  return {std::move(ResultPipeline)};
2468 }
2469 
2470 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2471  const PipelineElement &E) {
2472  auto &Name = E.Name;
2473  auto &InnerPipeline = E.InnerPipeline;
2474 
2475  // First handle complex passes like the pass managers which carry pipelines.
2476  if (!InnerPipeline.empty()) {
2477  if (Name == "module") {
2478  ModulePassManager NestedMPM(DebugLogging);
2479  if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2480  return Err;
2481  MPM.addPass(std::move(NestedMPM));
2482  return Error::success();
2483  }
2484  if (Name == "cgscc") {
2485  CGSCCPassManager CGPM(DebugLogging);
2486  if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
2487  return Err;
2489  return Error::success();
2490  }
2491  if (Name == "function") {
2492  FunctionPassManager FPM(DebugLogging);
2493  if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2494  return Err;
2496  return Error::success();
2497  }
2498  if (auto Count = parseRepeatPassName(Name)) {
2499  ModulePassManager NestedMPM(DebugLogging);
2500  if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2501  return Err;
2502  MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2503  return Error::success();
2504  }
2505 
2506  for (auto &C : ModulePipelineParsingCallbacks)
2507  if (C(Name, MPM, InnerPipeline))
2508  return Error::success();
2509 
2510  // Normal passes can't have pipelines.
2511  return make_error<StringError>(
2512  formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2514  ;
2515  }
2516 
2517  // Manually handle aliases for pre-configured pipeline fragments.
2519  SmallVector<StringRef, 3> Matches;
2520  if (!DefaultAliasRegex.match(Name, &Matches))
2521  return make_error<StringError>(
2522  formatv("unknown default pipeline alias '{0}'", Name).str(),
2524 
2525  assert(Matches.size() == 3 && "Must capture two matched strings!");
2526 
2527  OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2528  .Case("O0", OptimizationLevel::O0)
2529  .Case("O1", OptimizationLevel::O1)
2530  .Case("O2", OptimizationLevel::O2)
2531  .Case("O3", OptimizationLevel::O3)
2532  .Case("Os", OptimizationLevel::Os)
2533  .Case("Oz", OptimizationLevel::Oz);
2534  if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" &&
2535  Matches[1] != "lto") {
2536  MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" ||
2537  Matches[1] == "lto-pre-link"));
2538  return Error::success();
2539  }
2540 
2541  // This is consistent with old pass manager invoked via opt, but
2542  // inconsistent with clang. Clang doesn't enable loop vectorization
2543  // but does enable slp vectorization at Oz.
2544  PTO.LoopVectorization =
2545  L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2546  PTO.SLPVectorization =
2547  L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2548 
2549  if (Matches[1] == "default") {
2551  } else if (Matches[1] == "thinlto-pre-link") {
2553  } else if (Matches[1] == "thinlto") {
2554  MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
2555  } else if (Matches[1] == "lto-pre-link") {
2557  } else {
2558  assert(Matches[1] == "lto" && "Not one of the matched options!");
2559  MPM.addPass(buildLTODefaultPipeline(L, nullptr));
2560  }
2561  return Error::success();
2562  }
2563 
2564  // Finally expand the basic registered passes from the .inc file.
2565 #define MODULE_PASS(NAME, CREATE_PASS) \
2566  if (Name == NAME) { \
2567  MPM.addPass(CREATE_PASS); \
2568  return Error::success(); \
2569  }
2570 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2571  if (Name == "require<" NAME ">") { \
2572  MPM.addPass( \
2573  RequireAnalysisPass< \
2574  std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
2575  return Error::success(); \
2576  } \
2577  if (Name == "invalidate<" NAME ">") { \
2578  MPM.addPass(InvalidateAnalysisPass< \
2579  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2580  return Error::success(); \
2581  }
2582 #define CGSCC_PASS(NAME, CREATE_PASS) \
2583  if (Name == NAME) { \
2584  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
2585  return Error::success(); \
2586  }
2587 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2588  if (Name == NAME) { \
2589  MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
2590  return Error::success(); \
2591  }
2592 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2593  if (checkParametrizedPassName(Name, NAME)) { \
2594  auto Params = parsePassParameters(PARSER, Name, NAME); \
2595  if (!Params) \
2596  return Params.takeError(); \
2597  MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2598  return Error::success(); \
2599  }
2600 #define LOOP_PASS(NAME, CREATE_PASS) \
2601  if (Name == NAME) { \
2602  MPM.addPass( \
2603  createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2604  CREATE_PASS, false, false, DebugLogging))); \
2605  return Error::success(); \
2606  }
2607 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2608  if (checkParametrizedPassName(Name, NAME)) { \
2609  auto Params = parsePassParameters(PARSER, Name, NAME); \
2610  if (!Params) \
2611  return Params.takeError(); \
2612  MPM.addPass( \
2613  createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2614  CREATE_PASS(Params.get()), false, false, DebugLogging))); \
2615  return Error::success(); \
2616  }
2617 #include "PassRegistry.def"
2618 
2619  for (auto &C : ModulePipelineParsingCallbacks)
2620  if (C(Name, MPM, InnerPipeline))
2621  return Error::success();
2622  return make_error<StringError>(
2623  formatv("unknown module pass '{0}'", Name).str(),
2625 }
2626 
2627 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2628  const PipelineElement &E) {
2629  auto &Name = E.Name;
2630  auto &InnerPipeline = E.InnerPipeline;
2631 
2632  // First handle complex passes like the pass managers which carry pipelines.
2633  if (!InnerPipeline.empty()) {
2634  if (Name == "cgscc") {
2635  CGSCCPassManager NestedCGPM(DebugLogging);
2636  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2637  return Err;
2638  // Add the nested pass manager with the appropriate adaptor.
2639  CGPM.addPass(std::move(NestedCGPM));
2640  return Error::success();
2641  }
2642  if (Name == "function") {
2643  FunctionPassManager FPM(DebugLogging);
2644  if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2645  return Err;
2646  // Add the nested pass manager with the appropriate adaptor.
2648  return Error::success();
2649  }
2650  if (auto Count = parseRepeatPassName(Name)) {
2651  CGSCCPassManager NestedCGPM(DebugLogging);
2652  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2653  return Err;
2654  CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2655  return Error::success();
2656  }
2657  if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2658  CGSCCPassManager NestedCGPM(DebugLogging);
2659  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2660  return Err;
2661  CGPM.addPass(
2662  createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2663  return Error::success();
2664  }
2665 
2666  for (auto &C : CGSCCPipelineParsingCallbacks)
2667  if (C(Name, CGPM, InnerPipeline))
2668  return Error::success();
2669 
2670  // Normal passes can't have pipelines.
2671  return make_error<StringError>(
2672  formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2674  }
2675 
2676 // Now expand the basic registered passes from the .inc file.
2677 #define CGSCC_PASS(NAME, CREATE_PASS) \
2678  if (Name == NAME) { \
2679  CGPM.addPass(CREATE_PASS); \
2680  return Error::success(); \
2681  }
2682 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2683  if (Name == "require<" NAME ">") { \
2684  CGPM.addPass(RequireAnalysisPass< \
2685  std::remove_reference<decltype(CREATE_PASS)>::type, \
2686  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
2687  CGSCCUpdateResult &>()); \
2688  return Error::success(); \
2689  } \
2690  if (Name == "invalidate<" NAME ">") { \
2691  CGPM.addPass(InvalidateAnalysisPass< \
2692  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2693  return Error::success(); \
2694  }
2695 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2696  if (Name == NAME) { \
2697  CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
2698  return Error::success(); \
2699  }
2700 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2701  if (checkParametrizedPassName(Name, NAME)) { \
2702  auto Params = parsePassParameters(PARSER, Name, NAME); \
2703  if (!Params) \
2704  return Params.takeError(); \
2705  CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2706  return Error::success(); \
2707  }
2708 #define LOOP_PASS(NAME, CREATE_PASS) \
2709  if (Name == NAME) { \
2710  CGPM.addPass( \
2711  createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2712  CREATE_PASS, false, false, DebugLogging))); \
2713  return Error::success(); \
2714  }
2715 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2716  if (checkParametrizedPassName(Name, NAME)) { \
2717  auto Params = parsePassParameters(PARSER, Name, NAME); \
2718  if (!Params) \
2719  return Params.takeError(); \
2720  CGPM.addPass( \
2721  createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2722  CREATE_PASS(Params.get()), false, false, DebugLogging))); \
2723  return Error::success(); \
2724  }
2725 #include "PassRegistry.def"
2726 
2727  for (auto &C : CGSCCPipelineParsingCallbacks)
2728  if (C(Name, CGPM, InnerPipeline))
2729  return Error::success();
2730  return make_error<StringError>(
2731  formatv("unknown cgscc pass '{0}'", Name).str(),
2733 }
2734 
2735 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2736  const PipelineElement &E) {
2737  auto &Name = E.Name;
2738  auto &InnerPipeline = E.InnerPipeline;
2739 
2740  // First handle complex passes like the pass managers which carry pipelines.
2741  if (!InnerPipeline.empty()) {
2742  if (Name == "function") {
2743  FunctionPassManager NestedFPM(DebugLogging);
2744  if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2745  return Err;
2746  // Add the nested pass manager with the appropriate adaptor.
2747  FPM.addPass(std::move(NestedFPM));
2748  return Error::success();
2749  }
2750  if (Name == "loop" || Name == "loop-mssa") {
2751  LoopPassManager LPM(DebugLogging);
2752  if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
2753  return Err;
2754  // Add the nested pass manager with the appropriate adaptor.
2755  bool UseMemorySSA = (Name == "loop-mssa");
2756  bool UseBFI = llvm::any_of(
2757  InnerPipeline, [](auto Pipeline) { return Pipeline.Name == "licm"; });
2758  FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2759  UseBFI, DebugLogging));
2760  return Error::success();
2761  }
2762  if (auto Count = parseRepeatPassName(Name)) {
2763  FunctionPassManager NestedFPM(DebugLogging);
2764  if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2765  return Err;
2766  FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2767  return Error::success();
2768  }
2769 
2770  for (auto &C : FunctionPipelineParsingCallbacks)
2771  if (C(Name, FPM, InnerPipeline))
2772  return Error::success();
2773 
2774  // Normal passes can't have pipelines.
2775  return make_error<StringError>(
2776  formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2778  }
2779 
2780 // Now expand the basic registered passes from the .inc file.
2781 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2782  if (Name == NAME) { \
2783  FPM.addPass(CREATE_PASS); \
2784  return Error::success(); \
2785  }
2786 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2787  if (checkParametrizedPassName(Name, NAME)) { \
2788  auto Params = parsePassParameters(PARSER, Name, NAME); \
2789  if (!Params) \
2790  return Params.takeError(); \
2791  FPM.addPass(CREATE_PASS(Params.get())); \
2792  return Error::success(); \
2793  }
2794 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2795  if (Name == "require<" NAME ">") { \
2796  FPM.addPass( \
2797  RequireAnalysisPass< \
2798  std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2799  return Error::success(); \
2800  } \
2801  if (Name == "invalidate<" NAME ">") { \
2802  FPM.addPass(InvalidateAnalysisPass< \
2803  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2804  return Error::success(); \
2805  }
2806 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2807 // bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2808 // "guard-widening");
2809 // The risk is that it may become obsolete if we're not careful.
2810 #define LOOP_PASS(NAME, CREATE_PASS) \
2811  if (Name == NAME) { \
2812  FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \
2813  DebugLogging)); \
2814  return Error::success(); \
2815  }
2816 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2817  if (checkParametrizedPassName(Name, NAME)) { \
2818  auto Params = parsePassParameters(PARSER, Name, NAME); \
2819  if (!Params) \
2820  return Params.takeError(); \
2821  FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
2822  false, false, DebugLogging)); \
2823  return Error::success(); \
2824  }
2825 #include "PassRegistry.def"
2826 
2827  for (auto &C : FunctionPipelineParsingCallbacks)
2828  if (C(Name, FPM, InnerPipeline))
2829  return Error::success();
2830  return make_error<StringError>(
2831  formatv("unknown function pass '{0}'", Name).str(),
2833 }
2834 
2835 Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
2836  const PipelineElement &E) {
2837  StringRef Name = E.Name;
2838  auto &InnerPipeline = E.InnerPipeline;
2839 
2840  // First handle complex passes like the pass managers which carry pipelines.
2841  if (!InnerPipeline.empty()) {
2842  if (Name == "loop") {
2843  LoopPassManager NestedLPM(DebugLogging);
2844  if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2845  return Err;
2846  // Add the nested pass manager with the appropriate adaptor.
2847  LPM.addPass(std::move(NestedLPM));
2848  return Error::success();
2849  }
2850  if (auto Count = parseRepeatPassName(Name)) {
2851  LoopPassManager NestedLPM(DebugLogging);
2852  if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2853  return Err;
2854  LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2855  return Error::success();
2856  }
2857 
2858  for (auto &C : LoopPipelineParsingCallbacks)
2859  if (C(Name, LPM, InnerPipeline))
2860  return Error::success();
2861 
2862  // Normal passes can't have pipelines.
2863  return make_error<StringError>(
2864  formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2866  }
2867 
2868 // Now expand the basic registered passes from the .inc file.
2869 #define LOOP_PASS(NAME, CREATE_PASS) \
2870  if (Name == NAME) { \
2871  LPM.addPass(CREATE_PASS); \
2872  return Error::success(); \
2873  }
2874 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2875  if (checkParametrizedPassName(Name, NAME)) { \
2876  auto Params = parsePassParameters(PARSER, Name, NAME); \
2877  if (!Params) \
2878  return Params.takeError(); \
2879  LPM.addPass(CREATE_PASS(Params.get())); \
2880  return Error::success(); \
2881  }
2882 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2883  if (Name == "require<" NAME ">") { \
2884  LPM.addPass(RequireAnalysisPass< \
2885  std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2886  LoopAnalysisManager, LoopStandardAnalysisResults &, \
2887  LPMUpdater &>()); \
2888  return Error::success(); \
2889  } \
2890  if (Name == "invalidate<" NAME ">") { \
2891  LPM.addPass(InvalidateAnalysisPass< \
2892  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2893  return Error::success(); \
2894  }
2895 #include "PassRegistry.def"
2896 
2897  for (auto &C : LoopPipelineParsingCallbacks)
2898  if (C(Name, LPM, InnerPipeline))
2899  return Error::success();
2900  return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2902 }
2903 
2904 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2905 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2906  if (Name == NAME) { \
2907  AA.registerModuleAnalysis< \
2908  std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2909  return true; \
2910  }
2911 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2912  if (Name == NAME) { \
2913  AA.registerFunctionAnalysis< \
2914  std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2915  return true; \
2916  }
2917 #include "PassRegistry.def"
2918 
2919  for (auto &C : AAParsingCallbacks)
2920  if (C(Name, AA))
2921  return true;
2922  return false;
2923 }
2924 
2925 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2926  ArrayRef<PipelineElement> Pipeline) {
2927  for (const auto &Element : Pipeline) {
2928  if (auto Err = parseLoopPass(LPM, Element))
2929  return Err;
2930  }
2931  return Error::success();
2932 }
2933 
2934 Error PassBuilder::parseFunctionPassPipeline(
2936  for (const auto &Element : Pipeline) {
2937  if (auto Err = parseFunctionPass(FPM, Element))
2938  return Err;
2939  }
2940  return Error::success();
2941 }
2942 
2943 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2944  ArrayRef<PipelineElement> Pipeline) {
2945  for (const auto &Element : Pipeline) {
2946  if (auto Err = parseCGSCCPass(CGPM, Element))
2947  return Err;
2948  }
2949  return Error::success();
2950 }
2951 
2954  CGSCCAnalysisManager &CGAM,
2955  ModuleAnalysisManager &MAM) {
2956  MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2957  MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2958  CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2959  FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2960  FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2961  FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2962  LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2963 }
2964 
2965 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2966  ArrayRef<PipelineElement> Pipeline) {
2967  for (const auto &Element : Pipeline) {
2968  if (auto Err = parseModulePass(MPM, Element))
2969  return Err;
2970  }
2971  return Error::success();
2972 }
2973 
2974 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2975 // FIXME: Should this routine accept a TargetMachine or require the caller to
2976 // pre-populate the analysis managers with target-specific stuff?
2978  StringRef PipelineText) {
2979  auto Pipeline = parsePipelineText(PipelineText);
2980  if (!Pipeline || Pipeline->empty())
2981  return make_error<StringError>(
2982  formatv("invalid pipeline '{0}'", PipelineText).str(),
2984 
2985  // If the first name isn't at the module layer, wrap the pipeline up
2986  // automatically.
2987  StringRef FirstName = Pipeline->front().Name;
2988 
2989  if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2990  if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2991  Pipeline = {{"cgscc", std::move(*Pipeline)}};
2992  } else if (isFunctionPassName(FirstName,
2993  FunctionPipelineParsingCallbacks)) {
2994  Pipeline = {{"function", std::move(*Pipeline)}};
2995  } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2996  Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2997  } else {
2998  for (auto &C : TopLevelPipelineParsingCallbacks)
2999  if (C(MPM, *Pipeline, DebugLogging))
3000  return Error::success();
3001 
3002  // Unknown pass or pipeline name!
3003  auto &InnerPipeline = Pipeline->front().InnerPipeline;
3004  return make_error<StringError>(
3005  formatv("unknown {0} name '{1}'",
3006  (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
3007  .str(),
3009  }
3010  }
3011 
3012  if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
3013  return Err;
3014  return Error::success();
3015 }
3016 
3017 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
3019  StringRef PipelineText) {
3020  auto Pipeline = parsePipelineText(PipelineText);
3021  if (!Pipeline || Pipeline->empty())
3022  return make_error<StringError>(
3023  formatv("invalid pipeline '{0}'", PipelineText).str(),
3025 
3026  StringRef FirstName = Pipeline->front().Name;
3027  if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
3028  return make_error<StringError>(
3029  formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
3030  PipelineText)
3031  .str(),
3033 
3034  if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
3035  return Err;
3036  return Error::success();
3037 }
3038 
3039 // Primary pass pipeline description parsing routine for a \c
3040 // FunctionPassManager
3042  StringRef PipelineText) {
3043  auto Pipeline = parsePipelineText(PipelineText);
3044  if (!Pipeline || Pipeline->empty())
3045  return make_error<StringError>(
3046  formatv("invalid pipeline '{0}'", PipelineText).str(),
3048 
3049  StringRef FirstName = Pipeline->front().Name;
3050  if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
3051  return make_error<StringError>(
3052  formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
3053  PipelineText)
3054  .str(),
3056 
3057  if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
3058  return Err;
3059  return Error::success();
3060 }
3061 
3062 // Primary pass pipeline description parsing routine for a \c LoopPassManager
3064  StringRef PipelineText) {
3065  auto Pipeline = parsePipelineText(PipelineText);
3066  if (!Pipeline || Pipeline->empty())
3067  return make_error<StringError>(
3068  formatv("invalid pipeline '{0}'", PipelineText).str(),
3070 
3071  if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
3072  return Err;
3073 
3074  return Error::success();
3075 }
3076 
3078  // If the pipeline just consists of the word 'default' just replace the AA
3079  // manager with our default one.
3080  if (PipelineText == "default") {
3081  AA = buildDefaultAAPipeline();
3082  return Error::success();
3083  }
3084 
3085  while (!PipelineText.empty()) {
3086  StringRef Name;
3087  std::tie(Name, PipelineText) = PipelineText.split(',');
3088  if (!parseAAPassName(AA, Name))
3089  return make_error<StringError>(
3090  formatv("unknown alias analysis name '{0}'", Name).str(),
3092  }
3093 
3094  return Error::success();
3095 }
3096 
3098 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3099  if (PassName == NAME) \
3100  return true;
3101 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3102  if (PassName == NAME) \
3103  return true;
3104 #include "PassRegistry.def"
3105  return false;
3106 }
3107 
3109 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
3110  if (PassName == NAME) \
3111  return true;
3112 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
3113  if (PassName == NAME) \
3114  return true;
3115 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
3116  if (PassName == NAME) \
3117  return true;
3118 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
3119  if (PassName == NAME) \
3120  return true;
3121 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3122  if (PassName == NAME) \
3123  return true;
3124 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3125  if (PassName == NAME) \
3126  return true;
3127 #include "PassRegistry.def"
3128  return false;
3129 }
3130 
3132  OS << " " << PassName << "\n";
3133 }
3134 
3136  // TODO: print pass descriptions when they are available
3137 
3138  OS << "Module passes:\n";
3139 #define MODULE_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3140 #include "PassRegistry.def"
3141 
3142  OS << "Module analyses:\n";
3143 #define MODULE_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3144 #include "PassRegistry.def"
3145 
3146  OS << "Module alias analyses:\n";
3147 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3148 #include "PassRegistry.def"
3149 
3150  OS << "CGSCC passes:\n";
3151 #define CGSCC_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3152 #include "PassRegistry.def"
3153 
3154  OS << "CGSCC analyses:\n";
3155 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3156 #include "PassRegistry.def"
3157 
3158  OS << "Function passes:\n";
3159 #define FUNCTION_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3160 #include "PassRegistry.def"
3161 
3162  OS << "Function analyses:\n";
3163 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3164 #include "PassRegistry.def"
3165 
3166  OS << "Function alias analyses:\n";
3167 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3168 #include "PassRegistry.def"
3169 
3170  OS << "Loop passes:\n";
3171 #define LOOP_PASS(NAME, CREATE_PASS) printPassName(NAME, OS);
3172 #include "PassRegistry.def"
3173 
3174  OS << "Loop analyses:\n";
3175 #define LOOP_ANALYSIS(NAME, CREATE_PASS) printPassName(NAME, OS);
3176 #include "PassRegistry.def"
3177 }
3178 
3181  bool DebugLogging)> &C) {
3182  TopLevelPipelineParsingCallbacks.push_back(C);
3183 }
llvm::GVNOptions
A set of parameters to control various transforms performed by GVN pass.
Definition: GVN.h:73
llvm::StringSwitch::Case
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
llvm::GlobalsAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: GlobalsModRef.h:132
AddressSanitizer.h
RegionInfo.h
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
EnableGVNSink
cl::opt< bool > EnableGVNSink
llvm::PipelineTuningOptions::Coroutines
bool Coroutines
Tuning option to enable/disable coroutine intrinsic lowering.
Definition: PassBuilder.h:114
TypeBasedAliasAnalysis.h
llvm::PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & >
Definition: LoopPassManager.h:70
Instrumentation.h
SymbolRewriter.h
llvm::BasicAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: BasicAliasAnalysis.h:233
llvm::IPSCCPPass
Pass to perform interprocedural constant propagation.
Definition: SCCP.h:30
llvm::ScopedNoAliasAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: ScopedNoAliasAA.h:53
AssumptionCache.h
StackSafetyAnalysis.h
llvm::AAManager
A manager for alias analyses.
Definition: AliasAnalysis.h:1221
startsWithDefaultPipelineAliasPrefix
static bool startsWithDefaultPipelineAliasPrefix(StringRef Name)
Tests whether a pass name starts with a valid prefix for a default pipeline alias.
Definition: PassBuilder.cpp:2285
IROutliner.h
EnableSyntheticCounts
static cl::opt< bool > EnableSyntheticCounts("enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run synthetic function entry count generation " "pass"))
llvm::PassBuilder::OptimizationLevel::O1
static const OptimizationLevel O1
Optimize quickly without destroying debuggability.
Definition: PassBuilder.h:199
Lint.h
LowerWidenableCondition.h
DeadArgumentElimination.h
llvm
Definition: AllocatorList.h:23
LoopSimplify.h
DominanceFrontier.h
StripSymbols.h
Phase
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
Definition: AArch64FalkorHWPFFix.cpp:231
llvm::SampleProfileProbePass
Definition: SampleProfileProbe.h:130
llvm::StringRef::empty
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:156
Mem2Reg.h
MemorySanitizer.h
llvm::LoopSinkPass
A pass that does profile-guided sinking of instructions into loops.
Definition: LoopSink.h:33
llvm::SystemZISD::TM
@ TM
Definition: SystemZISelLowering.h:65
llvm::PassManager::isEmpty
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:574
OpenMPOpt.h
llvm::InferFunctionAttrsPass
A pass which infers function attributes from the names and signatures of function declarations in a m...
Definition: InferFunctionAttrs.h:25
llvm::SampleProfileLoaderPass
The sample profiler data loader pass.
Definition: SampleProfile.h:25
CalledValuePropagation.h
Annotation2Metadata.h
llvm::LoopInterchangePass
Definition: LoopInterchange.h:17
EnableIROutliner
cl::opt< bool > EnableIROutliner
llvm::AlignmentFromAssumptionsPass
Definition: AlignmentFromAssumptions.h:29
EnableLoopFlatten
cl::opt< bool > EnableLoopFlatten
PartialInlining.h
Inliner.h
BreakCriticalEdges.h
llvm::Annotation2MetadataPass
Pass to convert @llvm.global.annotations to !annotation metadata.
Definition: Annotation2Metadata.h:24
ThreadSanitizer.h
llvm::ThinOrFullLTOPhase::ThinLTOPostLink
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
llvm::PassInfoMixin
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:374
GlobalSplit.h
llvm::ModuleAnalysisManagerFunctionProxy
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Definition: PassManager.h:1206
llvm::GVNHoistPass
A simple and fast domtree-based GVN pass to hoist common expressions from sibling branches.
Definition: GVN.h:366
llvm::InstrProfOptions::UseBFIInPromotion
bool UseBFIInPromotion
Definition: Instrumentation.h:132
CorrelatedValuePropagation.h
llvm::LoopIdiomRecognizePass
Performs Loop Idiom Recognize Pass.
Definition: LoopIdiomRecognize.h:40
llvm::Function
Definition: Function.h:61
InstructionNamer.h
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:529
NaryReassociate.h
llvm::CrossDSOCFIPass
Definition: CrossDSOCFI.h:20
llvm::StringSwitch::Default
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
llvm::ReassociatePass
Reassociate commutative expressions.
Definition: Reassociate.h:71
InferAddressSpaces.h
LCSSA.h
ConstantMerge.h
llvm::SetLicmMssaNoAccForPromotionCap
cl::opt< unsigned > SetLicmMssaNoAccForPromotionCap
AlignmentFromAssumptions.h
llvm::StringRef::npos
static constexpr size_t npos
Definition: StringRef.h:59
LowerAtomic.h
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
FunctionImport.h
llvm::APInt::getSExtValue
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1643
IndVarSimplify.h
ErrorHandling.h
LoopAccessAnalysis.h
InstCount.h
SCCP.h
llvm::PassBuilder::buildModuleOptimizationPipeline
ModulePassManager buildModuleOptimizationPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Construct the core LLVM module optimization pipeline.
Definition: PassBuilder.cpp:1182
UnifyFunctionExitNodes.h
llvm::Error::success
static ErrorSuccess success()
Create a success value.
Definition: Error.h:332
TailRecursionElimination.h
DivRemPairs.h
llvm::LoopRotatePass
A simple loop rotation transformation.
Definition: LoopRotation.h:23
llvm::BDCEPass
Definition: BDCE.h:25
DeadStoreElimination.h
llvm::PipelineTuningOptions
Tunable parameters for passes in the default pipelines.
Definition: PassBuilder.h:86
llvm::StringRef::consume_front
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition: StringRef.h:683
OptimizationRemarkEmitter.h
CoroCleanup.h
isLoopPassName
static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2390
GlobalsModRef.h
EnableHotColdSplit
cl::opt< bool > EnableHotColdSplit
VectorCombine.h
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
MergeFunctions.h
llvm::PseudoProbeUpdatePass
Definition: SampleProfileProbe.h:138
llvm::ModuleAnalysisManagerCGSCCProxy
OuterAnalysisManagerProxy< ModuleAnalysisManager, LazyCallGraph::SCC, LazyCallGraph & > ModuleAnalysisManagerCGSCCProxy
A proxy from a ModuleAnalysisManager to an SCC.
Definition: CGSCCPassManager.h:215
IVUsersPrinter.h
LoopFlatten.h
ScalarEvolution.h
LoopAccessAnalysisPrinter.h
ElimAvailExtern.h
llvm::PipelineTuningOptions::ForgetAllSCEVInLoopUnroll
bool ForgetAllSCEVInLoopUnroll
Tuning option to forget all SCEV loops in LoopUnroll.
Definition: PassBuilder.h:108
llvm::LoopUnrollOptions
A set of parameters used to control various transforms performed by the LoopUnroll pass.
Definition: LoopUnrollPass.h:61
InjectTLIMappings.h
ConstraintElimination.h
Float2Int.h
llvm::LoopStandardAnalysisResults
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
Definition: LoopAnalysisManager.h:52
CGProfile.h
isModulePassName
static bool isModulePassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2310
MetaRenamer.h
llvm::LoopDistributePass
Definition: LoopDistribute.h:25
JumpThreading.h
HotColdSplitting.h
llvm::createCGSCCToFunctionPassAdaptor
CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:494
llvm::createModuleToPostOrderCGSCCPassAdaptor
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:376
MaxDevirtIterations
cl::opt< unsigned > MaxDevirtIterations
llvm::PGOInstrumentationUse
The profile annotation (profile-instr-use) pass for IR based PGO.
Definition: PGOInstrumentation.h:58
llvm::Optional
Definition: APInt.h:33
llvm::InlineParams
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:160
LoopUnrollAndJamPass.h
PartiallyInlineLibCalls.h
DivergenceAnalysis.h
llvm::LoopLoadEliminationPass
Pass to forward loads in a loop around the backedge to subsequent iterations.
Definition: LoopLoadElimination.h:27
llvm::StackLifetime::LivenessType::Must
@ Must
FixIrreducible.h
llvm::ThinOrFullLTOPhase::ThinLTOPreLink
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
llvm::TailCallElimPass
Definition: TailRecursionElimination.h:60
llvm::CoroCleanupPass
Definition: CoroCleanup.h:23
LazyValueInfo.h
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
llvm::EnableMSSALoopDependency
cl::opt< bool > EnableMSSALoopDependency
Enables memory ssa as a dependency for loop passes.
LoopInstSimplify.h
MemoryDependenceAnalysis.h
ModuleSummaryAnalysis.h
StripDeadPrototypes.h
llvm::PassBuilder::OptimizationLevel::Oz
static const OptimizationLevel Oz
A very specialized mode that will optimize for code size at any and all costs.
Definition: PassBuilder.h:250
llvm::SLPVectorizerPass
Definition: SLPVectorizer.h:58
llvm::ReversePostOrderFunctionAttrsPass
A pass to do RPO deduction and propagation of function attributes.
Definition: FunctionAttrs.h:69
llvm::PartialInlinerPass
Pass to remove unused function declarations.
Definition: PartialInlining.h:24
llvm::PGOOptions::SampleUse
@ SampleUse
Definition: PassBuilder.h:36
llvm::DSEPass
This class implements a trivial dead store elimination.
Definition: DeadStoreElimination.h:28
isLTOPreLink
static bool isLTOPreLink(ThinOrFullLTOPhase Phase)
Definition: PassBuilder.cpp:520
llvm::PassBuilder::addPGOInstrPassesForO0
void addPGOInstrPassesForO0(ModulePassManager &MPM, bool RunProfileGen, bool IsCS, std::string ProfileFile, std::string ProfileRemappingFile)
Add PGOInstrumenation passes for O0 only.
Definition: PassBuilder.cpp:931
BasicAliasAnalysis.h
llvm::PassBuilder::OptimizationLevel::O2
static const OptimizationLevel O2
Optimize for fast execution as much as possible without triggering significant incremental compile ti...
Definition: PassBuilder.h:216
llvm::PassBuilder::PassBuilder
PassBuilder(bool DebugLogging=false, TargetMachine *TM=nullptr, PipelineTuningOptions PTO=PipelineTuningOptions(), Optional< PGOOptions > PGOOpt=None, PassInstrumentationCallbacks *PIC=nullptr)
Definition: PassBuilder.cpp:443
DataFlowSanitizer.h
llvm::Regex::match
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition: Regex.cpp:86
CoroElide.h
llvm::LoopUnrollOptions::setPartial
LoopUnrollOptions & setPartial(bool Partial)
Enables or disables partial unrolling.
Definition: LoopUnrollPass.h:87
Internalize.h
llvm::PipelineTuningOptions::LicmMssaNoAccForPromotionCap
unsigned LicmMssaNoAccForPromotionCap
Tuning option to disable promotion to scalars in LICM with MemorySSA, if the number of access is too ...
Definition: PassBuilder.h:122
MergedLoadStoreMotion.h
llvm::LoopVectorizeOptions::setInterleaveOnlyWhenForced
LoopVectorizeOptions & setInterleaveOnlyWhenForced(bool Value)
Definition: LoopVectorize.h:108
MustExecute.h
llvm::AggressiveInstCombinePass
Definition: AggressiveInstCombine.h:24
llvm::InstrProfiling
Instrumentation based profiling lowering pass.
Definition: InstrProfiling.h:35
llvm::VectorCombinePass
Optimize scalar/vector interactions in IR using target cost models.
Definition: VectorCombine.h:23
llvm::ConstraintEliminationPass
Definition: ConstraintElimination.h:16
llvm::ModuleInlinerWrapperPass::addRequiredModuleAnalysis
void addRequiredModuleAnalysis()
Allow adding module-level analyses benefiting the contained CGSCC passes.
Definition: Inliner.h:135
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::HotColdSplittingPass
Pass to outline cold regions.
Definition: HotColdSplitting.h:61
llvm::PassBuilder::OptimizationLevel::O0
static const OptimizationLevel O0
Disable as many optimizations as possible.
Definition: PassBuilder.h:183
llvm::TypeBasedAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: TypeBasedAliasAnalysis.h:59
Scalarizer.h
llvm::PassBuilder::parsePassPipeline
Error parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText)
Parse a textual pass pipeline description into a ModulePassManager.
Definition: PassBuilder.cpp:2977
llvm::StringRef::substr
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:612
llvm::ThinOrFullLTOPhase
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:73
CommandLine.h
LoopRotation.h
llvm::formatv
auto formatv(const char *Fmt, Ts &&... Vals) -> formatv_object< decltype(std::make_tuple(detail::build_format_adapter(std::forward< Ts >(Vals))...))>
Definition: FormatVariadic.h:250
llvm::MODULE
@ MODULE
Definition: Attributor.h:3845
EnableConstraintElimination
cl::opt< bool > EnableConstraintElimination
Reg2Mem.h
llvm::ModuleInlinerWrapperPass
Module pass, wrapping the inliner pass.
Definition: Inliner.h:118
llvm::CoroElidePass
Definition: CoroElide.h:25
RunPartialInlining
cl::opt< bool > RunPartialInlining
llvm::ControlHeightReductionPass
Definition: ControlHeightReduction.h:22
llvm::CorrelatedValuePropagationPass
Definition: CorrelatedValuePropagation.h:18
EnableO3NonTrivialUnswitching
static cl::opt< bool > EnableO3NonTrivialUnswitching("enable-npm-O3-nontrivial-unswitch", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable non-trivial loop unswitching for -O3"))
StackLifetime.h
TargetMachine.h
llvm::AttributorPass
}
Definition: Attributor.h:2425
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:421
RunNewGVN
cl::opt< bool > RunNewGVN
llvm::LoopUnrollOptions::setPeeling
LoopUnrollOptions & setPeeling(bool Peeling)
Enables or disables loop peeling.
Definition: LoopUnrollPass.h:99
AlwaysInliner.h
InstrProfiling.h
PostDominators.h
LoopIdiomRecognize.h
HWAddressSanitizer.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::LICMPass
Performs Loop Invariant Code Motion Pass.
Definition: LICM.h:46
HelloWorld.h
llvm::StringRef::split
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:730
EnableOrderFileInstrumentation
cl::opt< bool > EnableOrderFileInstrumentation
ArgumentPromotion.h
DisablePreInliner
cl::opt< bool > DisablePreInliner
EnableMemProfiler
static cl::opt< bool > EnableMemProfiler("enable-mem-prof", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable memory profiler"))
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::PostOrderFunctionAttrsPass
Computes function attributes in post-order over the call graph.
Definition: FunctionAttrs.h:50
BlockExtractor.h
llvm::CGSCCAnalysisManagerModuleProxy
InnerAnalysisManagerProxy< CGSCCAnalysisManager, Module > CGSCCAnalysisManagerModuleProxy
A proxy from a CGSCCAnalysisManager to a Module.
Definition: CGSCCPassManager.h:168
LowerInvoke.h
llvm::ModuleInlinerWrapperPass::getPM
CGSCCPassManager & getPM()
Allow adding more CGSCC passes, besides inlining.
Definition: Inliner.h:132
isFunctionPassName
static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2364
llvm::PassBuilder::buildFunctionSimplificationPipeline
FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM function canonicalization and simplification pipeline.
Definition: PassBuilder.cpp:663
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
EnableLoopInterchange
cl::opt< bool > EnableLoopInterchange
llvm::AMDGPU::PALMD::Key
Key
PAL metadata keys.
Definition: AMDGPUMetadata.h:481
llvm::PGOOptions::IRUse
@ IRUse
Definition: PassBuilder.h:36
llvm::CoroEarlyPass
Definition: CoroEarly.h:26
IP
Definition: NVPTXLowerArgs.cpp:166
TargetLibraryInfo.h
RewriteStatepointsForGC.h
AssumeBundleBuilder.h
llvm::GVNSinkPass
Uses an "inverted" value numbering to decide the similarity of expressions and sinks similar expressi...
Definition: GVN.h:373
llvm::PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & >::addPass
std::enable_if_t< is_detected< HasRunOnLoopT, PassT >::value > addPass(PassT Pass)
Definition: LoopPassManager.h:111
LowerSwitch.h
InlineAdvisor.h
StructurizeCFG.h
llvm::PGOIndirectCallPromotion
The indirect function call promotion pass.
Definition: PGOInstrumentation.h:73
EnablePGOInlineDeferral
static cl::opt< bool > EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true), cl::Hidden, cl::desc("Enable inline deferral during PGO"))
Flag to enable inline deferral during PGO.
IRPrintingPasses.h
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
MemCpyOptimizer.h
llvm::CanonicalizeAliasesPass
Simple pass that canonicalizes aliases.
Definition: CanonicalizeAliases.h:22
GVN.h
llvm::LoopDeletionPass
Definition: LoopDeletion.h:24
llvm::PromotePass
Definition: Mem2Reg.h:23
llvm::LoopFlattenPass
Definition: LoopFlatten.h:23
llvm::getInlineParams
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
Definition: InlineCost.cpp:2754
llvm::PGOOptions::CSIRInstr
@ CSIRInstr
Definition: PassBuilder.h:37
ConstantHoisting.h
llvm::PassBuilder::buildInlinerPipeline
ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the module pipeline that performs inlining as well as the inlining-driven cleanups.
Definition: PassBuilder.cpp:962
llvm::InstrProfOptions::DoCounterPromotion
bool DoCounterPromotion
Definition: Instrumentation.h:126
LazyCallGraph.h
CFLAndersAliasAnalysis.h
llvm::GlobalOptPass
Optimize globals that never have their address taken.
Definition: GlobalOpt.h:25
llvm::SyntheticCountsPropagation
Definition: SyntheticCountsPropagation.h:17
llvm::PGOMemOPSizeOpt
The profile size based optimization pass for memory intrinsics.
Definition: PGOInstrumentation.h:86
FormatVariadic.h
llvm::PassBuilder::buildThinLTOPreLinkDefaultPipeline
ModulePassManager buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, ThinLTO-targeting default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1466
PrintPasses.h
ScopedNoAliasAA.h
llvm::StringRef::getAsInteger
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:511
llvm::None
const NoneType None
Definition: None.h:23
CanonicalizeFreezeInLoops.h
llvm::MemProfilerPass
Public interface to the memory profiler pass for instrumenting code to profile memory accesses.
Definition: MemProfiler.h:29
llvm::LoopSimplifyCFGPass
Performs basic CFG simplifications to assist other loop passes.
Definition: LoopSimplifyCFG.h:26
llvm::PGOInstrumentationGenCreateVar
The instrumentation (profile-instr-gen) pass for IR based PGO.
Definition: PGOInstrumentation.h:35
llvm::MergedLoadStoreMotionPass
Definition: MergedLoadStoreMotion.h:41
ControlHeightReduction.h
InstSimplifyPass.h
llvm::LowerTypeTestsPass
Definition: LowerTypeTests.h:200
llvm::PassBuilder::isAnalysisPassName
bool isAnalysisPassName(StringRef PassName)
Returns true if the pass name is the name of a (non-alias) analysis pass.
Definition: PassBuilder.cpp:3108
llvm::PassBuilder::isAAPassName
bool isAAPassName(StringRef PassName)
Returns true if the pass name is the name of an alias analysis pass.
Definition: PassBuilder.cpp:3097
FlattenedProfileUsed
cl::opt< bool > FlattenedProfileUsed
LoopInfo.h
llvm::AnnotationRemarksPass
Definition: AnnotationRemarks.h:23
llvm::SimpleLoopUnswitchPass
This pass transforms loops that contain branches or switches on loop- invariant conditions to have mu...
Definition: SimpleLoopUnswitch.h:62
llvm::cl::ZeroOrMore
@ ZeroOrMore
Definition: CommandLine.h:117
LoopFuse.h
SROA.h
SampleProfile.h
llvm::GlobalSplitPass
Pass to perform split of global variables.
Definition: GlobalSplit.h:26
MakeGuardsExplicit.h
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
llvm::StackLifetime::LivenessType::May
@ May
ForceFunctionAttrs.h
llvm::cl::opt
Definition: CommandLine.h:1419
Attributor.h
llvm::ModuleMemProfilerPass
Public interface to the memory profiler module pass for instrumenting code to profile memory allocati...
Definition: MemProfiler.h:38
llvm::createModuleToFunctionPassAdaptor
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: PassManager.h:1252
LoadStoreVectorizer.h
SpeculativeExecution.h
checkParametrizedPassName
static bool checkParametrizedPassName(StringRef Name, StringRef PassName)
Definition: PassBuilder.cpp:2019
InlineSizeEstimatorAnalysis.h
llvm::PassBuilder::registerFunctionAnalyses
void registerFunctionAnalyses(FunctionAnalysisManager &FAM)
Registers all available function analysis passes.
Definition: PassBuilder.cpp:494
llvm::cl::values
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:696
BranchProbabilityInfo.h
llvm::GVN
The core GVN pass object.
Definition: GVN.h:116
EarlyCSE.h
CrossDSOCFI.h
CFGPrinter.h
GCOVProfiler.h
ProfileSummaryInfo.h
DefaultAliasRegex
static const Regex DefaultAliasRegex("^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$")
llvm::WholeProgramDevirtPass
Definition: WholeProgramDevirt.h:223
llvm::LPMUpdater
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
Definition: LoopPassManager.h:243
CoroSplit.h
BoundsChecking.h
llvm::AssumeSimplifyPass
This pass attempts to minimize the number of assume without loosing any information.
Definition: AssumeBundleBuilder.h:54
StripGCRelocates.h
llvm::PassBuilder::buildLTOPreLinkDefaultPipeline
ModulePassManager buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, LTO-targeting default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1583
llvm::PassBuilder::registerModuleAnalyses
void registerModuleAnalyses(ModuleAnalysisManager &MAM)
Registers all available module analysis passes.
Definition: PassBuilder.cpp:476
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::InstrProfOptions::InstrProfileOutput
std::string InstrProfileOutput
Definition: Instrumentation.h:135
llvm::AttributorCGSCCPass
Definition: Attributor.h:2428
PGOInstrumentation.h
CGSCCPassManager.h
llvm::AnalysisKey
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:72
llvm::StringRef::front
LLVM_NODISCARD char front() const
front - Get the first character in the string.
Definition: StringRef.h:164
llvm::PassBuilder::buildLTODefaultPipeline
ModulePassManager buildLTODefaultPipeline(OptimizationLevel Level, ModuleSummaryIndex *ExportSummary)
Build an LTO default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1592
llvm::ForceFunctionAttrsPass
Pass which forces specific function attributes into the IR, primarily as a debugging tool.
Definition: ForceFunctionAttrs.h:22
AggressiveInstCombine.h
UseBFI
UseBFI
Definition: MachineLICM.cpp:83
LowerExpectIntrinsic.h
llvm::PassBuilder::buildThinLTODefaultPipeline
ModulePassManager buildThinLTODefaultPipeline(OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary)
Build an ThinLTO default optimization pipeline to a pass manager.
Definition: PassBuilder.cpp:1527
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:440
llvm::DeadArgumentEliminationPass
Eliminate dead arguments (and return values) from functions.
Definition: DeadArgumentElimination.h:38
ADCE.h
llvm::PassBuilder::buildDefaultAAPipeline
AAManager buildDefaultAAPipeline()
Build the default AAManager with the default alias analysis pipeline registered.
Definition: PassBuilder.cpp:1972
InferFunctionAttrs.h
SimpleLoopUnswitch.h
llvm::PipelineTuningOptions::PipelineTuningOptions
PipelineTuningOptions()
Constructor sets pipeline tuning defaults based on cl::opts.
Definition: PassBuilder.cpp:281
llvm::InstrProfOptions
Options for the frontend instrumentation based profiling pass.
Definition: Instrumentation.h:121
Enable
@ Enable
Definition: DwarfDebug.cpp:88
llvm::SpeculativeExecutionPass
Definition: SpeculativeExecution.h:69
llvm::StackLifetime::LivenessType
LivenessType
Definition: StackLifetime.h:84
llvm::TargetMachine::registerDefaultAliasAnalyses
virtual void registerDefaultAliasAnalyses(AAManager &)
Allow the target to register alias analyses with the AAManager for use with the new pass manager.
Definition: TargetMachine.h:338
SeparateConstOffsetFromGEP.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::PassBuilder::printPassNames
void printPassNames(raw_ostream &OS)
Print pass names.
Definition: PassBuilder.cpp:3135
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::EliminateAvailableExternallyPass
A pass that transforms external global definitions into declarations.
Definition: ElimAvailExtern.h:24
Regex.h
llvm::ADCEPass
A DCE pass that assumes instructions are dead until proven otherwise.
Definition: ADCE.h:31
MemDerefPrinter.h
llvm::PipelineTuningOptions::SLPVectorization
bool SLPVectorization
Tuning option to enable/disable slp loop vectorization, set based on opt level.
Definition: PassBuilder.h:101
Reassociate.h
llvm::FunctionAnalysisManagerLoopProxy
OuterAnalysisManagerProxy< FunctionAnalysisManager, Loop, LoopStandardAnalysisResults & > FunctionAnalysisManagerLoopProxy
A proxy from a FunctionAnalysisManager to a Loop.
Definition: LoopAnalysisManager.h:156
llvm::createDevirtSCCRepeatedPass
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: CGSCCPassManager.h:539
llvm::JumpThreadingPass
This pass performs 'jump threading', which looks at blocks that have multiple predecessors and multip...
Definition: JumpThreading.h:79
SafepointIRVerifier.h
GuardWidening.h
llvm::LoopFullUnrollPass
Loop unroll pass that only does full loop unrolling and peeling.
Definition: LoopUnrollPass.h:26
parseDevirtPassName
static Optional< int > parseDevirtPassName(StringRef Name)
Definition: PassBuilder.cpp:2010
llvm::LowerExpectIntrinsicPass
Definition: LowerExpectIntrinsic.h:23
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
LoopNestAnalysis.h
llvm::printBeforePasses
std::vector< std::string > printBeforePasses()
Definition: PrintPasses.cpp:73
ObjCARCAliasAnalysis.h
llvm::InliningAdvisorMode::Development
@ Development
llvm::PassBuilder::OptimizationLevel::Os
static const OptimizationLevel Os
Similar to O2 but tries to optimize for small code size instead of fast execution without triggering ...
Definition: PassBuilder.h:242
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
SampleProfileProbe.h
LoopPassManager.h
llvm::Float2IntPass
Definition: Float2Int.h:26
llvm::LibCallsShrinkWrapPass
Definition: LibCallsShrinkWrap.h:18
InductiveRangeCheckElimination.h
ModuleDebugInfoPrinter.h
llvm::AnalysisInfoMixin
A CRTP mix-in that provides informational APIs needed for analysis passes.
Definition: PassManager.h:391
NameAnonGlobals.h
llvm::CalledValuePropagationPass
Definition: CalledValuePropagation.h:26
llvm::WarnMissedTransformationsPass
Definition: WarnMissedTransforms.h:24
llvm::IndVarSimplifyPass
Definition: IndVarSimplify.h:25
LowerMatrixIntrinsics.h
llvm::GlobalDCEPass
Pass to remove unused function declarations.
Definition: GlobalDCE.h:29
llvm::printAfterPasses
std::vector< std::string > printAfterPasses()
Definition: PrintPasses.cpp:77
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:70
LoopInterchange.h
llvm::PipelineTuningOptions::CallGraphProfile
bool CallGraphProfile
Tuning option to enable/disable call graph profile.
Definition: PassBuilder.h:126
llvm::ProfileSummaryAnalysis
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Definition: ProfileSummaryInfo.h:206
llvm::ThinOrFullLTOPhase::FullLTOPostLink
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
llvm::LowerMatrixIntrinsicsPass
Definition: LowerMatrixIntrinsics.h:19
llvm::NewGVNPass
Definition: NewGVN.h:23
LoopExtractor.h
SyntheticCountsPropagation.h
DDG.h
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::AddDiscriminatorsPass
Definition: AddDiscriminators.h:24
CanonicalizeAliases.h
llvm::CGSCC
@ CGSCC
Definition: Attributor.h:3846
LibCallsShrinkWrap.h
llvm::AAManager::registerModuleAnalysis
void registerModuleAnalysis()
Register a specific AA result.
Definition: AliasAnalysis.h:1231
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1512
ExtraVectorizerPasses
cl::opt< bool > ExtraVectorizerPasses
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
EnableKnowledgeRetention
cl::opt< bool > EnableKnowledgeRetention
llvm::InstCombinePass
Definition: InstCombine.h:25
SanitizerCoverage.h
llvm::LoopAnalysisManagerFunctionProxy
InnerAnalysisManagerProxy< LoopAnalysisManager, Function > LoopAnalysisManagerFunctionProxy
A proxy from a LoopAnalysisManager to a Function.
Definition: LoopAnalysisManager.h:79
GlobalDCE.h
clEnumValN
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:671
SpeculateAroundPHIs.h
llvm::LoopVectorizeOptions::setVectorizeOnlyWhenForced
LoopVectorizeOptions & setVectorizeOnlyWhenForced(bool Value)
Definition: LoopVectorize.h:113
llvm::PipelineTuningOptions::LoopVectorization
bool LoopVectorization
Tuning option to enable/disable loop vectorization, set based on opt level.
Definition: PassBuilder.h:97
NewGVN.h
llvm::PICLevel::Level
Level
Definition: CodeGen.h:33
llvm::PassBuilder::registerParseTopLevelPipelineCallback
void registerParseTopLevelPipelineCallback(const std::function< bool(ModulePassManager &, ArrayRef< PipelineElement >, bool DebugLogging)> &C)
Register a callback for a top-level pipeline entry.
Definition: PassBuilder.cpp:3179
llvm::LoopUnrollOptions::setFullUnrollMaxCount
LoopUnrollOptions & setFullUnrollMaxCount(unsigned O)
Definition: LoopUnrollPass.h:124
LowerTypeTests.h
BlockFrequencyInfo.h
CallSiteSplitting.h
llvm::LoopVectorizeOptions
Definition: LoopVectorize.h:83
llvm::InstrOrderFilePass
The instrumentation pass for recording function order.
Definition: InstrOrderFile.h:21
LoopSimplifyCFG.h
llvm::CGProfilePass
Definition: CGProfile.h:19
AnnotationRemarks.h
llvm::PassBuilder::parseAAPipeline
Error parseAAPipeline(AAManager &AA, StringRef PipelineText)
Parse a textual alias analysis pipeline into the provided AA manager.
Definition: PassBuilder.cpp:3077
LoopVectorize.h
SCCP.h
DemandedBits.h
llvm::PassManager< Function >
llvm::LoopUnrollAndJamPass
A simple loop rotation transformation.
Definition: LoopUnrollAndJamPass.h:18
MergeICmps.h
LoopVersioningLICM.h
callbacksAcceptPassName
static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks)
Tests whether registered callbacks will accept a given pass name.
Definition: PassBuilder.cpp:2299
FunctionPropertiesAnalysis.h
llvm::IROutlinerPass
Pass to outline similar regions.
Definition: IROutliner.h:351
PhiValues.h
llvm::PassBuilder::registerLoopAnalyses
void registerLoopAnalyses(LoopAnalysisManager &LAM)
Registers all available loop analysis passes.
Definition: PassBuilder.cpp:503
name
static const char * name
Definition: SVEIntrinsicOpts.cpp:93
StraightLineStrengthReduce.h
llvm::ForgetSCEVInLoopUnroll
cl::opt< bool > ForgetSCEVInLoopUnroll
SLPVectorizer.h
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
LoopLoadElimination.h
llvm::CGSCCAnalysisManagerFunctionProxy
OuterAnalysisManagerProxy< CGSCCAnalysisManager, Function > CGSCCAnalysisManagerFunctionProxy
A proxy from a CGSCCAnalysisManager to a Function.
Definition: CGSCCPassManager.h:427
Verifier.h
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
LoopStrengthReduce.h
llvm::PassBuilder::OptimizationLevel
LLVM-provided high-level optimization levels.
Definition: PassBuilder.h:164
llvm::inconvertibleErrorCode
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
llvm::PassBuilder::buildPerModuleDefaultPipeline
ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Build a per-module default optimization pipeline.
Definition: PassBuilder.cpp:1425
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
LoopUnrollPass.h
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
llvm::SCCPPass
This pass performs function-level constant propagation and merging.
Definition: SCCP.h:36
EnableMatrix
cl::opt< bool > EnableMatrix
llvm::PipelineTuningOptions::MergeFunctions
bool MergeFunctions
Tuning option to enable/disable function merging.
Definition: PassBuilder.h:130
IRSimilarityIdentifier.h
UnifyLoopExits.h
llvm::PassBuilder::registerCGSCCAnalyses
void registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM)
Registers all available CGSCC analysis passes.
Definition: PassBuilder.cpp:485
PerformMandatoryInliningsFirst
static cl::opt< bool > PerformMandatoryInliningsFirst("mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Perform mandatory inlinings module-wide, before performing " "inlining."))
llvm::PassBuilder::OptimizationLevel::O3
static const OptimizationLevel O3
Optimize for fast execution as much as possible.
Definition: PassBuilder.h:231
llvm::NameAnonGlobalPass
Simple pass that provides a name to every anonymous globals.
Definition: NameAnonGlobals.h:23
PassManager.h
LowerConstantIntrinsics.h
llvm::ThinOrFullLTOPhase::None
@ None
No LTO/ThinLTO behavior needed.
addAnnotationRemarksPass
static void addAnnotationRemarksPass(ModulePassManager &MPM)
Definition: PassBuilder.cpp:513
ScalarEvolutionAliasAnalysis.h
llvm::PGOOptions::CSIRUse
@ CSIRUse
Definition: PassBuilder.h:37
StringSwitch.h
llvm::CoroSplitPass
Definition: CoroSplit.h:24
ScalarizeMaskedMemIntrin.h
llvm::PassBuilder::buildO0DefaultPipeline
ModulePassManager buildO0DefaultPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Build an O0 pipeline with the minimal semantically required passes.
Definition: PassBuilder.cpp:1877
llvm::AnalysisManager::registerPass
bool registerPass(PassBuilderT &&PassBuilder)
Register an analysis pass with the manager.
Definition: PassManager.h:847
Sink.h
LoopVersioning.h
WholeProgramDevirt.h
LowerGuardIntrinsic.h
llvm::MemCpyOptPass
Definition: MemCpyOptimizer.h:42
llvm::AAManager::registerFunctionAnalysis
void registerFunctionAnalysis()
Register a specific AA result.
Definition: AliasAnalysis.h:1226
llvm::ArgumentPromotionPass
Argument promotion pass.
Definition: ArgumentPromotion.h:24
getInlineParamsFromOptLevel
static InlineParams getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level)
Definition: PassBuilder.cpp:957
llvm::DivRemPairsPass
Hoist/decompose integer division and remainder instructions to enable CFG improvements and better cod...
Definition: DivRemPairs.h:23
LoopDistribute.h
llvm::LoopUnrollOptions::setRuntime
LoopUnrollOptions & setRuntime(bool Runtime)
Enables or disables unrolling of loops with runtime trip count.
Definition: LoopUnrollPass.h:93
llvm::AlwaysInlinerPass
Inlines functions marked as "always_inline".
Definition: AlwaysInliner.h:29
CallGraph.h
WarnMissedTransforms.h
LoopReroll.h
DDGPrinter.h
llvm::ThinOrFullLTOPhase::FullLTOPreLink
@ FullLTOPreLink
Full LTO prelink phase.
llvm::SimplifyCFGOptions
Definition: SimplifyCFGOptions.h:23
MemorySSA.h
AliasAnalysisEvaluator.h
EnableUnrollAndJam
cl::opt< bool > EnableUnrollAndJam
GlobalOpt.h
llvm::InjectTLIMappings
Definition: InjectTLIMappings.h:20
LoopSink.h
PreInlineThreshold
cl::opt< int > PreInlineThreshold
IVUsers.h
AttributorRun
cl::opt< AttributorRunOption > AttributorRun
llvm::LowerConstantIntrinsicsPass
Definition: LowerConstantIntrinsics.h:23
ObjCARC.h
llvm::OpenMPOptPass
OpenMP optimizations pass.
Definition: OpenMPOpt.h:65
llvm::ModuleSummaryIndex
Class to hold module path string table and global value map, and encapsulate methods for operating on...
Definition: ModuleSummaryIndex.h:1026
llvm::PassInstrumentationCallbacks
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
Definition: PassInstrumentation.h:65
Dominators.h
llvm::StringRef::consume_back
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
Definition: StringRef.h:693
llvm::InliningAdvisorMode::Default
@ Default
StripNonLineTableDebugInfo.h
llvm::PassBuilder::crossRegisterProxies
void crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, ModuleAnalysisManager &MAM)
Cross register the analysis managers through their proxies.
Definition: PassBuilder.cpp:2952
llvm::LoopUnrollOptions::setOptLevel
LoopUnrollOptions & setOptLevel(int O)
Definition: LoopUnrollPass.h:112
llvm::PGOInstrumentationGen
The instrumentation (profile-instr-gen) pass for IR based PGO.
Definition: PGOInstrumentation.h:47
llvm::CallSiteSplittingPass
Definition: CallSiteSplitting.h:17
LoopDeletion.h
llvm::LoopInstSimplifyPass
Performs Loop Inst Simplify Pass.
Definition: LoopInstSimplify.h:25
MemProfiler.h
llvm::createFunctionToLoopPassAdaptor
std::enable_if_t< is_detected< HasRunOnLoopT, LoopPassT >::value, FunctionToLoopPassAdaptor > createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA=false, bool UseBlockFrequencyInfo=false, bool DebugLogging=false)
A function to deduce a loop pass type and wrap it in the templated adaptor.
Definition: LoopPassManager.h:449
TargetTransformInfo.h
llvm::MemorySanitizerOptions
Definition: MemorySanitizer.h:21
Delinearization.h
LICM.h
LoopCacheAnalysis.h
CoroEarly.h
llvm::LoopUnrollOptions::setProfileBasedPeeling
LoopUnrollOptions & setProfileBasedPeeling(int O)
Definition: LoopUnrollPass.h:118
llvm::PGOOptions::IRInstr
@ IRInstr
Definition: PassBuilder.h:36
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:43
DCE.h
AliasSetTracker.h
llvm::InliningAdvisorMode::Release
@ Release
llvm::StringRef::find_first_of
LLVM_NODISCARD size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:413
BDCE.h
EnableCHR
cl::opt< bool > EnableCHR
InstrOrderFile.h
parseRepeatPassName
static Optional< int > parseRepeatPassName(StringRef Name)
Definition: PassBuilder.cpp:2001
InstCombine.h
DependenceAnalysis.h
llvm::InstSimplifyPass
Run instruction simplification across each instruction in the function.
Definition: InstSimplifyPass.h:34
llvm::RequireAnalysisPass
A utility pass template to force an analysis result to be available.
Definition: PassManager.h:1273
llvm::PassBuilder::buildModuleSimplificationPipeline
ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM module canonicalization and simplification pipeline.
Definition: PassBuilder.cpp:1026
llvm::PipelineTuningOptions::LoopUnrolling
bool LoopUnrolling
Tuning option to enable/disable loop unrolling. Its default value is true.
Definition: PassBuilder.h:104
llvm::cl::desc
Definition: CommandLine.h:411
llvm::Regex
Definition: Regex.h:28
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:112
llvm::PipelineTuningOptions::LicmMssaOptCap
unsigned LicmMssaOptCap
Tuning option to cap the number of calls to retrive clobbering accesses in MemorySSA,...
Definition: PassBuilder.h:118
LoopPredication.h
UseInlineAdvisor
static cl::opt< InliningAdvisorMode > UseInlineAdvisor("enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden, cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"), cl::values(clEnumValN(InliningAdvisorMode::Default, "default", "Heuristics-based inliner version."), clEnumValN(InliningAdvisorMode::Development, "development", "Use development mode (runtime-loadable model)."), clEnumValN(InliningAdvisorMode::Release, "release", "Use release mode (AOT-compiled model).")))
llvm::PipelineTuningOptions::LoopInterleaving
bool LoopInterleaving
Tuning option to set loop interleaving on/off, set based on opt level.
Definition: PassBuilder.h:93
EntryExitInstrumenter.h
SimplifyCFG.h
llvm::SetLicmMssaOptCap
cl::opt< unsigned > SetLicmMssaOptCap
llvm::MergeFunctionsPass
Merge identical functions.
Definition: MergeFunctions.h:25
PoisonChecking.h
llvm::PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & >::isEmpty
bool isEmpty() const
Definition: LoopPassManager.h:156
llvm::LoopUnrollPass
Loop unroll pass that will support both full and partial unrolling.
Definition: LoopUnrollPass.h:133
llvm::FunctionAnalysisManagerModuleProxy
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
Definition: PassManager.h:1052
FunctionAttrs.h
Debug.h
llvm::SimplifyCFGPass
A pass to simplify and canonicalize the CFG of a function.
Definition: SimplifyCFG.h:29
isCGSCCPassName
static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks)
Definition: PassBuilder.cpp:2339
llvm::LoopVectorizePass
The LoopVectorize Pass.
Definition: LoopVectorize.h:129
PassBuilder.h
PassName
static const char PassName[]
Definition: X86LowerAMXIntrinsics.cpp:606
llvm::PassManager::addPass
std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT Pass)
Definition: PassManager.h:553
CFLSteensAliasAnalysis.h
llvm::LoopUnrollOptions::setUpperBound
LoopUnrollOptions & setUpperBound(bool UpperBound)
Enables or disables the use of trip count upper bound in loop unrolling.
Definition: LoopUnrollPass.h:106
LoopDataPrefetch.h
llvm::SROA
An optimization pass providing Scalar Replacement of Aggregates.
Definition: SROA.h:65
llvm::createRepeatedPass
RepeatedPass< PassT > createRepeatedPass(int Count, PassT P)
Definition: PassManager.h:1363
printPassName
static void printPassName(StringRef PassName, raw_ostream &OS)
Definition: PassBuilder.cpp:3131
llvm::EarlyCSEPass
A simple and fast domtree-based CSE pass.
Definition: EarlyCSE.h:30
llvm::SpeculateAroundPHIsPass
This pass handles simple speculating of instructions around PHIs when doing so is profitable for a pa...
Definition: SpeculateAroundPHIs.h:103
llvm::ConstantMergePass
A pass that merges duplicate global constants into a single constant.
Definition: ConstantMerge.h:29
EnableGVNHoist
cl::opt< bool > EnableGVNHoist
AddDiscriminators.h