LLVM  12.0.0git
PassBuilder.cpp
Go to the documentation of this file.
1 //===- Parsing, selection, and construction of pass pipelines -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
13 /// pipelines.
14 ///
15 //===----------------------------------------------------------------------===//
16 
18 #include "llvm/ADT/StringSwitch.h"
30 #include "llvm/Analysis/DDG.h"
39 #include "llvm/Analysis/IVUsers.h"
45 #include "llvm/Analysis/Lint.h"
48 #include "llvm/Analysis/LoopInfo.h"
70 #include "llvm/IR/Dominators.h"
72 #include "llvm/IR/PassManager.h"
73 #include "llvm/IR/PrintPasses.h"
75 #include "llvm/IR/Verifier.h"
77 #include "llvm/Support/Debug.h"
80 #include "llvm/Support/Regex.h"
138 #include "llvm/Transforms/ObjCARC.h"
239 
240 using namespace llvm;
241 
243 
245  "enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden,
246  cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"),
248  "Heuristics-based inliner version."),
250  "Use development mode (runtime-loadable model)."),
252  "Use release mode (AOT-compiled model).")));
253 
255  "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
256  cl::desc("Run synthetic function entry count generation "
257  "pass"));
258 
259 static const Regex DefaultAliasRegex(
260  "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
261 
262 /// Flag to enable inline deferral during PGO.
263 static cl::opt<bool>
264  EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true),
265  cl::Hidden,
266  cl::desc("Enable inline deferral during PGO"));
267 
268 static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
270  cl::desc("Enable memory profiler"));
271 
273  "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
274  cl::desc("Perform mandatory inlinings module-wide, before performing "
275  "inlining."));
276 
278  LoopInterleaving = true;
279  LoopVectorization = true;
280  SLPVectorization = false;
281  LoopUnrolling = true;
283  Coroutines = false;
286  CallGraphProfile = true;
287  MergeFunctions = false;
288  UniqueLinkageNames = false;
289 }
290 
297 extern cl::opt<bool> EnableCHR;
300 extern cl::opt<bool> RunNewGVN;
302 
304 
307 
309 
312 
314  /*SpeedLevel*/ 0,
315  /*SizeLevel*/ 0};
317  /*SpeedLevel*/ 1,
318  /*SizeLevel*/ 0};
320  /*SpeedLevel*/ 2,
321  /*SizeLevel*/ 0};
323  /*SpeedLevel*/ 3,
324  /*SizeLevel*/ 0};
326  /*SpeedLevel*/ 2,
327  /*SizeLevel*/ 1};
329  /*SpeedLevel*/ 2,
330  /*SizeLevel*/ 2};
331 
332 namespace {
333 
334 // The following passes/analyses have custom names, otherwise their name will
335 // include `(anonymous namespace)`. These are special since they are only for
336 // testing purposes and don't live in a header file.
337 
338 /// No-op module pass which does nothing.
339 struct NoOpModulePass : PassInfoMixin<NoOpModulePass> {
341  return PreservedAnalyses::all();
342  }
343 
344  static StringRef name() { return "NoOpModulePass"; }
345 };
346 
347 /// No-op module analysis.
348 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
350  static AnalysisKey Key;
351 
352 public:
353  struct Result {};
354  Result run(Module &, ModuleAnalysisManager &) { return Result(); }
355  static StringRef name() { return "NoOpModuleAnalysis"; }
356 };
357 
358 /// No-op CGSCC pass which does nothing.
359 struct NoOpCGSCCPass : PassInfoMixin<NoOpCGSCCPass> {
362  return PreservedAnalyses::all();
363  }
364  static StringRef name() { return "NoOpCGSCCPass"; }
365 };
366 
367 /// No-op CGSCC analysis.
368 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
370  static AnalysisKey Key;
371 
372 public:
373  struct Result {};
375  return Result();
376  }
377  static StringRef name() { return "NoOpCGSCCAnalysis"; }
378 };
379 
380 /// No-op function pass which does nothing.
381 struct NoOpFunctionPass : PassInfoMixin<NoOpFunctionPass> {
383  return PreservedAnalyses::all();
384  }
385  static StringRef name() { return "NoOpFunctionPass"; }
386 };
387 
388 /// No-op function analysis.
389 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
391  static AnalysisKey Key;
392 
393 public:
394  struct Result {};
395  Result run(Function &, FunctionAnalysisManager &) { return Result(); }
396  static StringRef name() { return "NoOpFunctionAnalysis"; }
397 };
398 
399 /// No-op loop pass which does nothing.
400 struct NoOpLoopPass : PassInfoMixin<NoOpLoopPass> {
403  return PreservedAnalyses::all();
404  }
405  static StringRef name() { return "NoOpLoopPass"; }
406 };
407 
408 /// No-op loop analysis.
409 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
411  static AnalysisKey Key;
412 
413 public:
414  struct Result {};
416  return Result();
417  }
418  static StringRef name() { return "NoOpLoopAnalysis"; }
419 };
420 
425 
426 /// Whether or not we should populate a PassInstrumentationCallbacks's class to
427 /// pass name map.
428 ///
429 /// This is for optimization purposes so we don't populate it if we never use
430 /// it. This should be updated if new pass instrumentation wants to use the map.
431 /// We currently only use this for --print-before/after.
432 bool shouldPopulateClassToPassNames() {
433  return !printBeforePasses().empty() || !printAfterPasses().empty();
434 }
435 
436 } // namespace
437 
441  : DebugLogging(DebugLogging), TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
442  if (TM)
443  TM->registerPassBuilderCallbacks(*this, DebugLogging);
444  if (PIC && shouldPopulateClassToPassNames()) {
445 #define MODULE_PASS(NAME, CREATE_PASS) \
446  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
447 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
448  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
449 #define FUNCTION_PASS(NAME, CREATE_PASS) \
450  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
451 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
452  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
453 #define LOOP_PASS(NAME, CREATE_PASS) \
454  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
455 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
456  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
457 #define CGSCC_PASS(NAME, CREATE_PASS) \
458  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
459 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
460  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
461 #include "PassRegistry.def"
462  }
463 }
464 
465 void PassBuilder::invokePeepholeEPCallbacks(
467  for (auto &C : PeepholeEPCallbacks)
468  C(FPM, Level);
469 }
470 
472 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
473  MAM.registerPass([&] { return CREATE_PASS; });
474 #include "PassRegistry.def"
475 
476  for (auto &C : ModuleAnalysisRegistrationCallbacks)
477  C(MAM);
478 }
479 
481 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
482  CGAM.registerPass([&] { return CREATE_PASS; });
483 #include "PassRegistry.def"
484 
485  for (auto &C : CGSCCAnalysisRegistrationCallbacks)
486  C(CGAM);
487 }
488 
490 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
491  FAM.registerPass([&] { return CREATE_PASS; });
492 #include "PassRegistry.def"
493 
494  for (auto &C : FunctionAnalysisRegistrationCallbacks)
495  C(FAM);
496 }
497 
499 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
500  LAM.registerPass([&] { return CREATE_PASS; });
501 #include "PassRegistry.def"
502 
503  for (auto &C : LoopAnalysisRegistrationCallbacks)
504  C(LAM);
505 }
506 
507 // Helper to add AnnotationRemarksPass.
512 }
513 
514 // TODO: Investigate the cost/benefit of tail call elimination on debugging.
516 PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
518 
519  FunctionPassManager FPM(DebugLogging);
520 
521  // Form SSA out of local memory accesses after breaking apart aggregates into
522  // scalars.
523  FPM.addPass(SROA());
524 
525  // Catch trivial redundancies
526  FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
527 
528  // Hoisting of scalars and load expressions.
529  FPM.addPass(SimplifyCFGPass());
530  FPM.addPass(InstCombinePass());
531 
533 
534  invokePeepholeEPCallbacks(FPM, Level);
535 
536  FPM.addPass(SimplifyCFGPass());
537 
538  // Form canonically associated expression trees, and simplify the trees using
539  // basic mathematical properties. For example, this will form (nearly)
540  // minimal multiplication trees.
541  FPM.addPass(ReassociatePass());
542 
543  // Add the primary loop simplification pipeline.
544  // FIXME: Currently this is split into two loop pass pipelines because we run
545  // some function passes in between them. These can and should be removed
546  // and/or replaced by scheduling the loop pass equivalents in the correct
547  // positions. But those equivalent passes aren't powerful enough yet.
548  // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
549  // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
550  // fully replace `SimplifyCFGPass`, and the closest to the other we have is
551  // `LoopInstSimplify`.
552  LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
553 
554  // Simplify the loop body. We do this initially to clean up after other loop
555  // passes run, either when iterating on a loop or on inner loops with
556  // implications on the outer loop.
557  LPM1.addPass(LoopInstSimplifyPass());
558  LPM1.addPass(LoopSimplifyCFGPass());
559 
560  LPM1.addPass(LoopRotatePass(/* Disable header duplication */ true));
561  // TODO: Investigate promotion cap for O1.
562  LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
563  LPM1.addPass(SimpleLoopUnswitchPass());
564 
565  LPM2.addPass(LoopIdiomRecognizePass());
566  LPM2.addPass(IndVarSimplifyPass());
567 
568  for (auto &C : LateLoopOptimizationsEPCallbacks)
569  C(LPM2, Level);
570 
571  LPM2.addPass(LoopDeletionPass());
572  // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
573  // because it changes IR to makes profile annotation in back compile
574  // inaccurate. The normal unroller doesn't pay attention to forced full unroll
575  // attributes so we need to make sure and allow the full unroll pass to pay
576  // attention to it.
577  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
578  PGOOpt->Action != PGOOptions::SampleUse)
579  LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
580  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
582 
583  for (auto &C : LoopOptimizerEndEPCallbacks)
584  C(LPM2, Level);
585 
586  // We provide the opt remark emitter pass for LICM to use. We only need to do
587  // this once as it is immutable.
588  FPM.addPass(
591  std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
592  DebugLogging));
593  FPM.addPass(SimplifyCFGPass());
594  FPM.addPass(InstCombinePass());
595  if (EnableLoopFlatten)
596  FPM.addPass(LoopFlattenPass());
597  // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
598  // *All* loop passes must preserve it, in order to be able to use it.
600  std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
601  DebugLogging));
602 
603  // Delete small array after loop unroll.
604  FPM.addPass(SROA());
605 
606  // Specially optimize memory movement as it doesn't look like dataflow in SSA.
607  FPM.addPass(MemCpyOptPass());
608 
609  // Sparse conditional constant propagation.
610  // FIXME: It isn't clear why we do this *after* loop passes rather than
611  // before...
612  FPM.addPass(SCCPPass());
613 
614  // Delete dead bit computations (instcombine runs after to fold away the dead
615  // computations, and then ADCE will run later to exploit any new DCE
616  // opportunities that creates).
617  FPM.addPass(BDCEPass());
618 
619  // Run instcombine after redundancy and dead bit elimination to exploit
620  // opportunities opened up by them.
621  FPM.addPass(InstCombinePass());
622  invokePeepholeEPCallbacks(FPM, Level);
623 
624  if (PTO.Coroutines)
625  FPM.addPass(CoroElidePass());
626 
627  for (auto &C : ScalarOptimizerLateEPCallbacks)
628  C(FPM, Level);
629 
630  // Finally, do an expensive DCE pass to catch all the dead code exposed by
631  // the simplifications and basic cleanup after all the simplifications.
632  // TODO: Investigate if this is too expensive.
633  FPM.addPass(ADCEPass());
634  FPM.addPass(SimplifyCFGPass());
635  FPM.addPass(InstCombinePass());
636  invokePeepholeEPCallbacks(FPM, Level);
637 
638  return FPM;
639 }
640 
644  assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
645 
646  // The O1 pipeline has a separate pipeline creation function to simplify
647  // construction readability.
648  if (Level.getSpeedupLevel() == 1)
649  return buildO1FunctionSimplificationPipeline(Level, Phase);
650 
651  FunctionPassManager FPM(DebugLogging);
652 
653  // Form SSA out of local memory accesses after breaking apart aggregates into
654  // scalars.
655  FPM.addPass(SROA());
656 
657  // Catch trivial redundancies
658  FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
661 
662  // Hoisting of scalars and load expressions.
663  if (EnableGVNHoist)
664  FPM.addPass(GVNHoistPass());
665 
666  // Global value numbering based sinking.
667  if (EnableGVNSink) {
668  FPM.addPass(GVNSinkPass());
669  FPM.addPass(SimplifyCFGPass());
670  }
671 
674 
675  // Speculative execution if the target has divergent branches; otherwise nop.
676  FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
677 
678  // Optimize based on known information about branches, and cleanup afterward.
679  FPM.addPass(JumpThreadingPass());
681 
682  FPM.addPass(SimplifyCFGPass());
685  FPM.addPass(InstCombinePass());
686 
687  if (!Level.isOptimizingForSize())
689 
690  invokePeepholeEPCallbacks(FPM, Level);
691 
692  // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
693  // using the size value profile. Don't perform this when optimizing for size.
694  if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
695  !Level.isOptimizingForSize())
696  FPM.addPass(PGOMemOPSizeOpt());
697 
698  FPM.addPass(TailCallElimPass());
699  FPM.addPass(SimplifyCFGPass());
700 
701  // Form canonically associated expression trees, and simplify the trees using
702  // basic mathematical properties. For example, this will form (nearly)
703  // minimal multiplication trees.
704  FPM.addPass(ReassociatePass());
705 
706  // Add the primary loop simplification pipeline.
707  // FIXME: Currently this is split into two loop pass pipelines because we run
708  // some function passes in between them. These can and should be removed
709  // and/or replaced by scheduling the loop pass equivalents in the correct
710  // positions. But those equivalent passes aren't powerful enough yet.
711  // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
712  // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
713  // fully replace `SimplifyCFGPass`, and the closest to the other we have is
714  // `LoopInstSimplify`.
715  LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
716 
717  // Simplify the loop body. We do this initially to clean up after other loop
718  // passes run, either when iterating on a loop or on inner loops with
719  // implications on the outer loop.
720  LPM1.addPass(LoopInstSimplifyPass());
721  LPM1.addPass(LoopSimplifyCFGPass());
722 
723  // Disable header duplication in loop rotation at -Oz.
724  LPM1.addPass(LoopRotatePass(Level != OptimizationLevel::Oz));
725  // TODO: Investigate promotion cap for O1.
726  LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
727  LPM1.addPass(
728  SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
730  LPM2.addPass(IndVarSimplifyPass());
731 
732  for (auto &C : LateLoopOptimizationsEPCallbacks)
733  C(LPM2, Level);
734 
735  LPM2.addPass(LoopDeletionPass());
736  // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
737  // because it changes IR to makes profile annotation in back compile
738  // inaccurate. The normal unroller doesn't pay attention to forced full unroll
739  // attributes so we need to make sure and allow the full unroll pass to pay
740  // attention to it.
741  if (Phase != ThinOrFullLTOPhase::ThinLTOPreLink || !PGOOpt ||
742  PGOOpt->Action != PGOOptions::SampleUse)
743  LPM2.addPass(LoopFullUnrollPass(Level.getSpeedupLevel(),
744  /* OnlyWhenForced= */ !PTO.LoopUnrolling,
746 
747  for (auto &C : LoopOptimizerEndEPCallbacks)
748  C(LPM2, Level);
749 
750  // We provide the opt remark emitter pass for LICM to use. We only need to do
751  // this once as it is immutable.
752  FPM.addPass(
755  std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
756  DebugLogging));
757  FPM.addPass(SimplifyCFGPass());
758  FPM.addPass(InstCombinePass());
759  if (EnableLoopFlatten)
760  FPM.addPass(LoopFlattenPass());
761  // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
762  // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
763  // *All* loop passes must preserve it, in order to be able to use it.
765  std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
766  DebugLogging));
767 
768  // Delete small array after loop unroll.
769  FPM.addPass(SROA());
770 
771  // Eliminate redundancies.
773  if (RunNewGVN)
774  FPM.addPass(NewGVNPass());
775  else
776  FPM.addPass(GVN());
777 
778  // Specially optimize memory movement as it doesn't look like dataflow in SSA.
779  FPM.addPass(MemCpyOptPass());
780 
781  // Sparse conditional constant propagation.
782  // FIXME: It isn't clear why we do this *after* loop passes rather than
783  // before...
784  FPM.addPass(SCCPPass());
785 
786  // Delete dead bit computations (instcombine runs after to fold away the dead
787  // computations, and then ADCE will run later to exploit any new DCE
788  // opportunities that creates).
789  FPM.addPass(BDCEPass());
790 
791  // Run instcombine after redundancy and dead bit elimination to exploit
792  // opportunities opened up by them.
793  FPM.addPass(InstCombinePass());
794  invokePeepholeEPCallbacks(FPM, Level);
795 
796  // Re-consider control flow based optimizations after redundancy elimination,
797  // redo DCE, etc.
798  FPM.addPass(JumpThreadingPass());
800 
801  // Finally, do an expensive DCE pass to catch all the dead code exposed by
802  // the simplifications and basic cleanup after all the simplifications.
803  // TODO: Investigate if this is too expensive.
804  FPM.addPass(ADCEPass());
805 
806  FPM.addPass(DSEPass());
809  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
810 
811  if (PTO.Coroutines)
812  FPM.addPass(CoroElidePass());
813 
814  for (auto &C : ScalarOptimizerLateEPCallbacks)
815  C(FPM, Level);
816 
817  FPM.addPass(SimplifyCFGPass());
818  FPM.addPass(InstCombinePass());
819  invokePeepholeEPCallbacks(FPM, Level);
820 
821  if (EnableCHR && Level == OptimizationLevel::O3 && PGOOpt &&
822  (PGOOpt->Action == PGOOptions::IRUse ||
823  PGOOpt->Action == PGOOptions::SampleUse))
825 
826  return FPM;
827 }
828 
829 void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
832 }
833 
834 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
836  bool RunProfileGen, bool IsCS,
837  std::string ProfileFile,
838  std::string ProfileRemappingFile) {
839  assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
840  if (!IsCS && !DisablePreInliner) {
841  InlineParams IP;
842 
844 
845  // FIXME: The hint threshold has the same value used by the regular inliner
846  // when not optimzing for size. This should probably be lowered after
847  // performance testing.
848  // FIXME: this comment is cargo culted from the old pass manager, revisit).
849  IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
850  ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
851  CGSCCPassManager &CGPipeline = MIWP.getPM();
852 
854  FPM.addPass(SROA());
855  FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
856  FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
857  FPM.addPass(InstCombinePass()); // Combine silly sequences.
858  invokePeepholeEPCallbacks(FPM, Level);
859 
861 
862  MPM.addPass(std::move(MIWP));
863 
864  // Delete anything that is now dead to make sure that we don't instrument
865  // dead code. Instrumentation can end up keeping dead code around and
866  // dramatically increase code size.
867  MPM.addPass(GlobalDCEPass());
868  }
869 
870  if (!RunProfileGen) {
871  assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
872  MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
873  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
874  // RequireAnalysisPass for PSI before subsequent non-module passes.
876  return;
877  }
878 
879  // Perform PGO instrumentation.
880  MPM.addPass(PGOInstrumentationGen(IsCS));
881 
883  // Disable header duplication in loop rotation at -Oz.
886  /*UseBlockFrequencyInfo=*/false, DebugLogging));
888 
889  // Add the profile lowering pass.
890  InstrProfOptions Options;
891  if (!ProfileFile.empty())
892  Options.InstrProfileOutput = ProfileFile;
893  // Do counter promotion at Level greater than O0.
894  Options.DoCounterPromotion = true;
895  Options.UseBFIInPromotion = IsCS;
896  MPM.addPass(InstrProfiling(Options, IsCS));
897 }
898 
900  bool RunProfileGen, bool IsCS,
901  std::string ProfileFile,
902  std::string ProfileRemappingFile) {
903  if (!RunProfileGen) {
904  assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
905  MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
906  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
907  // RequireAnalysisPass for PSI before subsequent non-module passes.
909  return;
910  }
911 
912  // Perform PGO instrumentation.
913  MPM.addPass(PGOInstrumentationGen(IsCS));
914  // Add the profile lowering pass.
915  InstrProfOptions Options;
916  if (!ProfileFile.empty())
917  Options.InstrProfileOutput = ProfileFile;
918  // Do not do counter promotion at O0.
919  Options.DoCounterPromotion = false;
920  Options.UseBFIInPromotion = IsCS;
921  MPM.addPass(InstrProfiling(Options, IsCS));
922 }
923 
924 static InlineParams
926  return getInlineParams(Level.getSpeedupLevel(), Level.getSizeLevel());
927 }
928 
930  OptimizationLevel Level, ThinOrFullLTOPhase Phase, bool MandatoryOnly) {
932  if (Phase == ThinOrFullLTOPhase::ThinLTOPreLink && PGOOpt &&
933  PGOOpt->Action == PGOOptions::SampleUse)
934  IP.HotCallSiteThreshold = 0;
935 
936  if (PGOOpt)
938 
940  IP, DebugLogging,
943 
944  if (MandatoryOnly)
945  return MIWP;
946 
947  // Require the GlobalsAA analysis for the module so we can query it within
948  // the CGSCC pipeline.
950 
951  // Require the ProfileSummaryAnalysis for the module so we can query it within
952  // the inliner pass.
954 
955  // Now begin the main postorder CGSCC pipeline.
956  // FIXME: The current CGSCC pipeline has its origins in the legacy pass
957  // manager and trying to emulate its precise behavior. Much of this doesn't
958  // make a lot of sense and we should revisit the core CGSCC structure.
959  CGSCCPassManager &MainCGPipeline = MIWP.getPM();
960 
961  // Note: historically, the PruneEH pass was run first to deduce nounwind and
962  // generally clean up exception handling overhead. It isn't clear this is
963  // valuable as the inliner doesn't currently care whether it is inlining an
964  // invoke or a call.
965 
967  MainCGPipeline.addPass(AttributorCGSCCPass());
968 
969  if (PTO.Coroutines)
970  MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
971 
972  // Now deduce any function attributes based in the current code.
973  MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
974 
975  // When at O3 add argument promotion to the pass pipeline.
976  // FIXME: It isn't at all clear why this should be limited to O3.
978  MainCGPipeline.addPass(ArgumentPromotionPass());
979 
980  // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
981  // there are no OpenMP runtime calls present in the module.
983  MainCGPipeline.addPass(OpenMPOptPass());
984 
985  for (auto &C : CGSCCOptimizerLateEPCallbacks)
986  C(MainCGPipeline, Level);
987 
988  // Lastly, add the core function simplification pipeline nested inside the
989  // CGSCC walk.
990  MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
992 
993  return MIWP;
994 }
995 
999  ModulePassManager MPM(DebugLogging);
1000 
1001  // Add UniqueInternalLinkageNames Pass which renames internal linkage
1002  // symbols with unique names.
1003  if (PTO.UniqueLinkageNames)
1005 
1006  // Place pseudo probe instrumentation as the first pass of the pipeline to
1007  // minimize the impact of optimization changes.
1008  if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1011 
1012  bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1013 
1014  // In ThinLTO mode, when flattened profile is used, all the available
1015  // profile information will be annotated in PreLink phase so there is
1016  // no need to load the profile again in PostLink.
1017  bool LoadSampleProfile =
1018  HasSampleProfile &&
1020 
1021  // During the ThinLTO backend phase we perform early indirect call promotion
1022  // here, before globalopt. Otherwise imported available_externally functions
1023  // look unreferenced and are removed. If we are going to load the sample
1024  // profile then defer until later.
1025  // TODO: See if we can move later and consolidate with the location where
1026  // we perform ICP when we are loading a sample profile.
1027  // TODO: We pass HasSampleProfile (whether there was a sample profile file
1028  // passed to the compile) to the SamplePGO flag of ICP. This is used to
1029  // determine whether the new direct calls are annotated with prof metadata.
1030  // Ideally this should be determined from whether the IR is annotated with
1031  // sample profile, and not whether the a sample profile was provided on the
1032  // command line. E.g. for flattened profiles where we will not be reloading
1033  // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1034  // provide the sample profile file.
1035  if (Phase == ThinOrFullLTOPhase::ThinLTOPostLink && !LoadSampleProfile)
1036  MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
1037 
1038  // Do basic inference of function attributes from known properties of system
1039  // libraries and other oracles.
1041 
1042  // Create an early function pass manager to cleanup the output of the
1043  // frontend.
1044  FunctionPassManager EarlyFPM(DebugLogging);
1045  EarlyFPM.addPass(SimplifyCFGPass());
1046  EarlyFPM.addPass(SROA());
1047  EarlyFPM.addPass(EarlyCSEPass());
1048  EarlyFPM.addPass(LowerExpectIntrinsicPass());
1049  if (PTO.Coroutines)
1050  EarlyFPM.addPass(CoroEarlyPass());
1052  EarlyFPM.addPass(CallSiteSplittingPass());
1053 
1054  // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
1055  // to convert bitcast to direct calls so that they can be inlined during the
1056  // profile annotation prepration step.
1057  // More details about SamplePGO design can be found in:
1058  // https://research.google.com/pubs/pub45290.html
1059  // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
1060  if (LoadSampleProfile)
1061  EarlyFPM.addPass(InstCombinePass());
1063 
1064  if (LoadSampleProfile) {
1065  // Annotate sample profile right after early FPM to ensure freshness of
1066  // the debug info.
1067  MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1068  PGOOpt->ProfileRemappingFile, Phase));
1069  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1070  // RequireAnalysisPass for PSI before subsequent non-module passes.
1072  // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
1073  // for the profile annotation to be accurate in the ThinLTO backend.
1075  // We perform early indirect call promotion here, before globalopt.
1076  // This is important for the ThinLTO backend phase because otherwise
1077  // imported available_externally functions look unreferenced and are
1078  // removed.
1080  Phase == ThinOrFullLTOPhase::ThinLTOPostLink, true /* SamplePGO */));
1081  }
1082 
1084  MPM.addPass(AttributorPass());
1085 
1086  // Lower type metadata and the type.test intrinsic in the ThinLTO
1087  // post link pipeline after ICP. This is to enable usage of the type
1088  // tests in ICP sequences.
1090  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1091 
1092  for (auto &C : PipelineEarlySimplificationEPCallbacks)
1093  C(MPM, Level);
1094 
1095  // Interprocedural constant propagation now that basic cleanup has occurred
1096  // and prior to optimizing globals.
1097  // FIXME: This position in the pipeline hasn't been carefully considered in
1098  // years, it should be re-analyzed.
1099  MPM.addPass(IPSCCPPass());
1100 
1101  // Attach metadata to indirect call sites indicating the set of functions
1102  // they may target at run-time. This should follow IPSCCP.
1104 
1105  // Optimize globals to try and fold them into constants.
1106  MPM.addPass(GlobalOptPass());
1107 
1108  // Promote any localized globals to SSA registers.
1109  // FIXME: Should this instead by a run of SROA?
1110  // FIXME: We should probably run instcombine and simplify-cfg afterward to
1111  // delete control flows that are dead once globals have been folded to
1112  // constants.
1114 
1115  // Remove any dead arguments exposed by cleanups and constant folding
1116  // globals.
1118 
1119  // Create a small function pass pipeline to cleanup after all the global
1120  // optimizations.
1121  FunctionPassManager GlobalCleanupPM(DebugLogging);
1122  GlobalCleanupPM.addPass(InstCombinePass());
1123  invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1124 
1125  GlobalCleanupPM.addPass(SimplifyCFGPass());
1126  MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
1127 
1128  // Add all the requested passes for instrumentation PGO, if requested.
1129  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1130  (PGOOpt->Action == PGOOptions::IRInstr ||
1131  PGOOpt->Action == PGOOptions::IRUse)) {
1132  addPGOInstrPasses(MPM, Level,
1133  /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
1134  /* IsCS */ false, PGOOpt->ProfileFile,
1135  PGOOpt->ProfileRemappingFile);
1136  MPM.addPass(PGOIndirectCallPromotion(false, false));
1137  }
1138  if (PGOOpt && Phase != ThinOrFullLTOPhase::ThinLTOPostLink &&
1139  PGOOpt->CSAction == PGOOptions::CSIRInstr)
1140  MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
1141 
1142  // Synthesize function entry counts for non-PGO compilation.
1143  if (EnableSyntheticCounts && !PGOOpt)
1145 
1147  MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/true));
1148  MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/false));
1149 
1153  }
1154 
1155  return MPM;
1156 }
1157 
1160  bool LTOPreLink) {
1161  ModulePassManager MPM(DebugLogging);
1162 
1163  // Optimize globals now that the module is fully simplified.
1164  MPM.addPass(GlobalOptPass());
1165  MPM.addPass(GlobalDCEPass());
1166 
1167  // Run partial inlining pass to partially inline functions that have
1168  // large bodies.
1169  if (RunPartialInlining)
1170  MPM.addPass(PartialInlinerPass());
1171 
1172  // Remove avail extern fns and globals definitions since we aren't compiling
1173  // an object file for later LTO. For LTO we want to preserve these so they
1174  // are eligible for inlining at link-time. Note if they are unreferenced they
1175  // will be removed by GlobalDCE later, so this only impacts referenced
1176  // available externally globals. Eventually they will be suppressed during
1177  // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1178  // may make globals referenced by available external functions dead and saves
1179  // running remaining passes on the eliminated functions. These should be
1180  // preserved during prelinking for link-time inlining decisions.
1181  if (!LTOPreLink)
1183 
1185  MPM.addPass(InstrOrderFilePass());
1186 
1187  // Do RPO function attribute inference across the module to forward-propagate
1188  // attributes where applicable.
1189  // FIXME: Is this really an optimization rather than a canonicalization?
1191 
1192  // Do a post inline PGO instrumentation and use pass. This is a context
1193  // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1194  // cross-module inline has not been done yet. The context sensitive
1195  // instrumentation is after all the inlines are done.
1196  if (!LTOPreLink && PGOOpt) {
1197  if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1198  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1199  /* IsCS */ true, PGOOpt->CSProfileGenFile,
1200  PGOOpt->ProfileRemappingFile);
1201  else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1202  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1203  /* IsCS */ true, PGOOpt->ProfileFile,
1204  PGOOpt->ProfileRemappingFile);
1205  }
1206 
1207  // Re-require GloblasAA here prior to function passes. This is particularly
1208  // useful as the above will have inlined, DCE'ed, and function-attr
1209  // propagated everything. We should at this point have a reasonably minimal
1210  // and richly annotated call graph. By computing aliasing and mod/ref
1211  // information for all local globals here, the late loop passes and notably
1212  // the vectorizer will be able to use them to help recognize vectorizable
1213  // memory operations.
1215 
1216  FunctionPassManager OptimizePM(DebugLogging);
1217  OptimizePM.addPass(Float2IntPass());
1218  OptimizePM.addPass(LowerConstantIntrinsicsPass());
1219 
1220  if (EnableMatrix) {
1221  OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1222  OptimizePM.addPass(EarlyCSEPass());
1223  }
1224 
1225  // FIXME: We need to run some loop optimizations to re-rotate loops after
1226  // simplify-cfg and others undo their rotation.
1227 
1228  // Optimize the loop execution. These passes operate on entire loop nests
1229  // rather than on each loop in an inside-out manner, and so they are actually
1230  // function passes.
1231 
1232  for (auto &C : VectorizerStartEPCallbacks)
1233  C(OptimizePM, Level);
1234 
1235  // First rotate loops that may have been un-rotated by prior passes.
1236  // Disable header duplication at -Oz.
1239  /*UseBlockFrequencyInfo=*/false, DebugLogging));
1240 
1241  // Distribute loops to allow partial vectorization. I.e. isolate dependences
1242  // into separate loop that would otherwise inhibit vectorization. This is
1243  // currently only performed for loops marked with the metadata
1244  // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1245  OptimizePM.addPass(LoopDistributePass());
1246 
1247  // Populates the VFABI attribute with the scalar-to-vector mappings
1248  // from the TargetLibraryInfo.
1249  OptimizePM.addPass(InjectTLIMappings());
1250 
1251  // Now run the core loop vectorizer.
1252  OptimizePM.addPass(LoopVectorizePass(
1254 
1255  // Eliminate loads by forwarding stores from the previous iteration to loads
1256  // of the current iteration.
1257  OptimizePM.addPass(LoopLoadEliminationPass());
1258 
1259  // Cleanup after the loop optimization passes.
1260  OptimizePM.addPass(InstCombinePass());
1261 
1262  // Now that we've formed fast to execute loop structures, we do further
1263  // optimizations. These are run afterward as they might block doing complex
1264  // analyses and transforms such as what are needed for loop vectorization.
1265 
1266  // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1267  // GVN, loop transforms, and others have already run, so it's now better to
1268  // convert to more optimized IR using more aggressive simplify CFG options.
1269  // The extra sinking transform can create larger basic blocks, so do this
1270  // before SLP vectorization.
1271  // FIXME: study whether hoisting and/or sinking of common instructions should
1272  // be delayed until after SLP vectorizer.
1274  .forwardSwitchCondToPhi(true)
1275  .convertSwitchToLookupTable(true)
1276  .needCanonicalLoops(false)
1277  .hoistCommonInsts(true)
1278  .sinkCommonInsts(true)));
1279 
1280  // Optimize parallel scalar instruction chains into SIMD instructions.
1281  if (PTO.SLPVectorization)
1282  OptimizePM.addPass(SLPVectorizerPass());
1283 
1284  // Enhance/cleanup vector code.
1285  OptimizePM.addPass(VectorCombinePass());
1286  OptimizePM.addPass(InstCombinePass());
1287 
1288  // Unroll small loops to hide loop backedge latency and saturate any parallel
1289  // execution resources of an out-of-order processor. We also then need to
1290  // clean up redundancies and loop invariant code.
1291  // FIXME: It would be really good to use a loop-integrated instruction
1292  // combiner for cleanup here so that the unrolling and LICM can be pipelined
1293  // across the loop nests.
1294  // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1295  if (EnableUnrollAndJam && PTO.LoopUnrolling) {
1296  OptimizePM.addPass(LoopUnrollAndJamPass(Level.getSpeedupLevel()));
1297  }
1299  Level.getSpeedupLevel(), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1301  OptimizePM.addPass(WarnMissedTransformationsPass());
1302  OptimizePM.addPass(InstCombinePass());
1306  EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
1307 
1308  // Now that we've vectorized and unrolled loops, we may have more refined
1309  // alignment information, try to re-derive it here.
1310  OptimizePM.addPass(AlignmentFromAssumptionsPass());
1311 
1312  // Split out cold code. Splitting is done late to avoid hiding context from
1313  // other optimizations and inadvertently regressing performance. The tradeoff
1314  // is that this has a higher code size cost than splitting early.
1315  if (EnableHotColdSplit && !LTOPreLink)
1317 
1318  // Search the code for similar regions of code. If enough similar regions can
1319  // be found where extracting the regions into their own function will decrease
1320  // the size of the program, we extract the regions, a deduplicate the
1321  // structurally similar regions.
1322  if (EnableIROutliner)
1323  MPM.addPass(IROutlinerPass());
1324 
1325  // Merge functions if requested.
1326  if (PTO.MergeFunctions)
1327  MPM.addPass(MergeFunctionsPass());
1328 
1329  // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1330  // canonicalization pass that enables other optimizations. As a result,
1331  // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1332  // result too early.
1333  OptimizePM.addPass(LoopSinkPass());
1334 
1335  // And finally clean up LCSSA form before generating code.
1336  OptimizePM.addPass(InstSimplifyPass());
1337 
1338  // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1339  // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1340  // flattening of blocks.
1341  OptimizePM.addPass(DivRemPairsPass());
1342 
1343  // LoopSink (and other loop passes since the last simplifyCFG) might have
1344  // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1345  OptimizePM.addPass(SimplifyCFGPass());
1346 
1347  // Optimize PHIs by speculating around them when profitable. Note that this
1348  // pass needs to be run after any PRE or similar pass as it is essentially
1349  // inserting redundancies into the program. This even includes SimplifyCFG.
1350  OptimizePM.addPass(SpeculateAroundPHIsPass());
1351 
1352  if (PTO.Coroutines)
1353  OptimizePM.addPass(CoroCleanupPass());
1354 
1355  // Add the core optimizing pipeline.
1357 
1358  for (auto &C : OptimizerLastEPCallbacks)
1359  C(MPM, Level);
1360 
1361  if (PTO.CallGraphProfile)
1362  MPM.addPass(CGProfilePass());
1363 
1364  // Now we need to do some global optimization transforms.
1365  // FIXME: It would seem like these should come first in the optimization
1366  // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1367  // ordering here.
1368  MPM.addPass(GlobalDCEPass());
1369  MPM.addPass(ConstantMergePass());
1370 
1371  return MPM;
1372 }
1373 
1376  bool LTOPreLink) {
1378  "Must request optimizations for the default pipeline!");
1379 
1380  ModulePassManager MPM(DebugLogging);
1381 
1382  // Convert @llvm.global.annotations to !annotation metadata.
1384 
1385  // Force any function attributes we want the rest of the pipeline to observe.
1387 
1388  // Apply module pipeline start EP callback.
1389  for (auto &C : PipelineStartEPCallbacks)
1390  C(MPM, Level);
1391 
1392  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1394 
1395  // Add the core simplification pipeline.
1399 
1400  // Now add the optimization pipeline.
1401  MPM.addPass(buildModuleOptimizationPipeline(Level, LTOPreLink));
1402 
1403  // Emit annotation remarks.
1405 
1406  if (LTOPreLink)
1407  addRequiredLTOPreLinkPasses(MPM);
1408 
1409  return MPM;
1410 }
1411 
1415  "Must request optimizations for the default pipeline!");
1416 
1417  ModulePassManager MPM(DebugLogging);
1418 
1419  // Convert @llvm.global.annotations to !annotation metadata.
1421 
1422  // Force any function attributes we want the rest of the pipeline to observe.
1424 
1425  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1427 
1428  // Apply module pipeline start EP callback.
1429  for (auto &C : PipelineStartEPCallbacks)
1430  C(MPM, Level);
1431 
1432  // If we are planning to perform ThinLTO later, we don't bloat the code with
1433  // unrolling/vectorization/... now. Just simplify the module as much as we
1434  // can.
1437 
1438  // Run partial inlining pass to partially inline functions that have
1439  // large bodies.
1440  // FIXME: It isn't clear whether this is really the right place to run this
1441  // in ThinLTO. Because there is another canonicalization and simplification
1442  // phase that will run after the thin link, running this here ends up with
1443  // less information than will be available later and it may grow functions in
1444  // ways that aren't beneficial.
1445  if (RunPartialInlining)
1446  MPM.addPass(PartialInlinerPass());
1447 
1448  // Reduce the size of the IR as much as possible.
1449  MPM.addPass(GlobalOptPass());
1450 
1451  // Module simplification splits coroutines, but does not fully clean up
1452  // coroutine intrinsics. To ensure ThinLTO optimization passes don't trip up
1453  // on these, we schedule the cleanup here.
1454  if (PTO.Coroutines)
1456 
1457  // Emit annotation remarks.
1459 
1460  addRequiredLTOPreLinkPasses(MPM);
1461 
1462  return MPM;
1463 }
1464 
1466  OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1467  ModulePassManager MPM(DebugLogging);
1468 
1469  // Convert @llvm.global.annotations to !annotation metadata.
1471 
1472  if (ImportSummary) {
1473  // These passes import type identifier resolutions for whole-program
1474  // devirtualization and CFI. They must run early because other passes may
1475  // disturb the specific instruction patterns that these passes look for,
1476  // creating dependencies on resolutions that may not appear in the summary.
1477  //
1478  // For example, GVN may transform the pattern assume(type.test) appearing in
1479  // two basic blocks into assume(phi(type.test, type.test)), which would
1480  // transform a dependency on a WPD resolution into a dependency on a type
1481  // identifier resolution for CFI.
1482  //
1483  // Also, WPD has access to more precise information than ICP and can
1484  // devirtualize more effectively, so it should operate on the IR first.
1485  //
1486  // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1487  // metadata and intrinsics.
1488  MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1489  MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1490  }
1491 
1493  return MPM;
1494 
1495  // Force any function attributes we want the rest of the pipeline to observe.
1497 
1498  // Add the core simplification pipeline.
1501 
1502  // Now add the optimization pipeline.
1504 
1505  // Emit annotation remarks.
1507 
1508  return MPM;
1509 }
1510 
1514  "Must request optimizations for the default pipeline!");
1515  // FIXME: We should use a customized pre-link pipeline!
1517  /* LTOPreLink */ true);
1518 }
1519 
1522  ModuleSummaryIndex *ExportSummary) {
1523  ModulePassManager MPM(DebugLogging);
1524 
1525  // Convert @llvm.global.annotations to !annotation metadata.
1527 
1528  if (Level == OptimizationLevel::O0) {
1529  // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1530  // metadata and intrinsics.
1531  MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1532  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1533  // Run a second time to clean up any type tests left behind by WPD for use
1534  // in ICP.
1535  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1536 
1537  // Emit annotation remarks.
1539 
1540  return MPM;
1541  }
1542 
1543  if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1544  // Load sample profile before running the LTO optimization pipeline.
1545  MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1546  PGOOpt->ProfileRemappingFile,
1548  // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1549  // RequireAnalysisPass for PSI before subsequent non-module passes.
1551  }
1552 
1553  // Remove unused virtual tables to improve the quality of code generated by
1554  // whole-program devirtualization and bitset lowering.
1555  MPM.addPass(GlobalDCEPass());
1556 
1557  // Force any function attributes we want the rest of the pipeline to observe.
1559 
1560  // Do basic inference of function attributes from known properties of system
1561  // libraries and other oracles.
1563 
1564  if (Level.getSpeedupLevel() > 1) {
1565  FunctionPassManager EarlyFPM(DebugLogging);
1566  EarlyFPM.addPass(CallSiteSplittingPass());
1568 
1569  // Indirect call promotion. This should promote all the targets that are
1570  // left by the earlier promotion pass that promotes intra-module targets.
1571  // This two-step promotion is to save the compile time. For LTO, it should
1572  // produce the same result as if we only do promotion here.
1574  true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1575  // Propagate constants at call sites into the functions they call. This
1576  // opens opportunities for globalopt (and inlining) by substituting function
1577  // pointers passed as arguments to direct uses of functions.
1578  MPM.addPass(IPSCCPPass());
1579 
1580  // Attach metadata to indirect call sites indicating the set of functions
1581  // they may target at run-time. This should follow IPSCCP.
1583  }
1584 
1585  // Now deduce any function attributes based in the current code.
1588 
1589  // Do RPO function attribute inference across the module to forward-propagate
1590  // attributes where applicable.
1591  // FIXME: Is this really an optimization rather than a canonicalization?
1593 
1594  // Use in-range annotations on GEP indices to split globals where beneficial.
1595  MPM.addPass(GlobalSplitPass());
1596 
1597  // Run whole program optimization of virtual call when the list of callees
1598  // is fixed.
1599  MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1600 
1601  // Stop here at -O1.
1602  if (Level == OptimizationLevel::O1) {
1603  // The LowerTypeTestsPass needs to run to lower type metadata and the
1604  // type.test intrinsics. The pass does nothing if CFI is disabled.
1605  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1606  // Run a second time to clean up any type tests left behind by WPD for use
1607  // in ICP (which is performed earlier than this in the regular LTO
1608  // pipeline).
1609  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1610 
1611  // Emit annotation remarks.
1613 
1614  return MPM;
1615  }
1616 
1617  // Optimize globals to try and fold them into constants.
1618  MPM.addPass(GlobalOptPass());
1619 
1620  // Promote any localized globals to SSA registers.
1622 
1623  // Linking modules together can lead to duplicate global constant, only
1624  // keep one copy of each constant.
1625  MPM.addPass(ConstantMergePass());
1626 
1627  // Remove unused arguments from functions.
1629 
1630  // Reduce the code after globalopt and ipsccp. Both can open up significant
1631  // simplification opportunities, and both can propagate functions through
1632  // function pointers. When this happens, we often have to resolve varargs
1633  // calls, etc, so let instcombine do this.
1634  FunctionPassManager PeepholeFPM(DebugLogging);
1636  PeepholeFPM.addPass(AggressiveInstCombinePass());
1637  PeepholeFPM.addPass(InstCombinePass());
1638  invokePeepholeEPCallbacks(PeepholeFPM, Level);
1639 
1641 
1643  DebugLogging,
1645  // Note: historically, the PruneEH pass was run first to deduce nounwind and
1646  // generally clean up exception handling overhead. It isn't clear this is
1647  // valuable as the inliner doesn't currently care whether it is inlining an
1648  // invoke or a call.
1649  // Run the inliner now.
1651  DebugLogging));
1652 
1653  // Optimize globals again after we ran the inliner.
1654  MPM.addPass(GlobalOptPass());
1655 
1656  // Garbage collect dead functions.
1657  // FIXME: Add ArgumentPromotion pass after once it's ported.
1658  MPM.addPass(GlobalDCEPass());
1659 
1660  FunctionPassManager FPM(DebugLogging);
1661  // The IPO Passes may leave cruft around. Clean up after them.
1662  FPM.addPass(InstCombinePass());
1663  invokePeepholeEPCallbacks(FPM, Level);
1664 
1665  FPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1666 
1667  // Do a post inline PGO instrumentation and use pass. This is a context
1668  // sensitive PGO pass.
1669  if (PGOOpt) {
1670  if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1671  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true,
1672  /* IsCS */ true, PGOOpt->CSProfileGenFile,
1673  PGOOpt->ProfileRemappingFile);
1674  else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1675  addPGOInstrPasses(MPM, Level, /* RunProfileGen */ false,
1676  /* IsCS */ true, PGOOpt->ProfileFile,
1677  PGOOpt->ProfileRemappingFile);
1678  }
1679 
1680  // Break up allocas
1681  FPM.addPass(SROA());
1682 
1683  // LTO provides additional opportunities for tailcall elimination due to
1684  // link-time inlining, and visibility of nocapture attribute.
1685  FPM.addPass(TailCallElimPass());
1686 
1687  // Run a few AA driver optimizations here and now to cleanup the code.
1689 
1692  // FIXME: here we run IP alias analysis in the legacy PM.
1693 
1694  FunctionPassManager MainFPM;
1695 
1696  // FIXME: once we fix LoopPass Manager, add LICM here.
1697  // FIXME: once we provide support for enabling MLSM, add it here.
1698  if (RunNewGVN)
1699  MainFPM.addPass(NewGVNPass());
1700  else
1701  MainFPM.addPass(GVN());
1702 
1703  // Remove dead memcpy()'s.
1704  MainFPM.addPass(MemCpyOptPass());
1705 
1706  // Nuke dead stores.
1707  MainFPM.addPass(DSEPass());
1708 
1709  // FIXME: at this point, we run a bunch of loop passes:
1710  // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1711  // loopVectorize. Enable them once the remaining issue with LPM
1712  // are sorted out.
1713 
1714  MainFPM.addPass(InstCombinePass());
1715  MainFPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true)));
1716  MainFPM.addPass(SCCPPass());
1717  MainFPM.addPass(InstCombinePass());
1718  MainFPM.addPass(BDCEPass());
1719 
1720  // FIXME: We may want to run SLPVectorizer here.
1721  // After vectorization, assume intrinsics may tell us more
1722  // about pointer alignments.
1723 #if 0
1724  MainFPM.add(AlignmentFromAssumptionsPass());
1725 #endif
1726 
1727  // FIXME: Conditionally run LoadCombine here, after it's ported
1728  // (in case we still have this pass, given its questionable usefulness).
1729 
1730  MainFPM.addPass(InstCombinePass());
1731  invokePeepholeEPCallbacks(MainFPM, Level);
1732  MainFPM.addPass(JumpThreadingPass(/*InsertFreezeWhenUnfoldingSelect*/ true));
1734 
1735  // Create a function that performs CFI checks for cross-DSO calls with
1736  // targets in the current module.
1737  MPM.addPass(CrossDSOCFIPass());
1738 
1739  // Lower type metadata and the type.test intrinsic. This pass supports
1740  // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1741  // to be run at link time if CFI is enabled. This pass does nothing if
1742  // CFI is disabled.
1743  MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1744  // Run a second time to clean up any type tests left behind by WPD for use
1745  // in ICP (which is performed earlier than this in the regular LTO pipeline).
1746  MPM.addPass(LowerTypeTestsPass(nullptr, nullptr, true));
1747 
1748  // Enable splitting late in the FullLTO post-link pipeline. This is done in
1749  // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1750  if (EnableHotColdSplit)
1752 
1753  // Add late LTO optimization passes.
1754  // Delete basic blocks, which optimization passes may have killed.
1756  SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))));
1757 
1758  // Drop bodies of available eternally objects to improve GlobalDCE.
1760 
1761  // Now that we have optimized the program, discard unreachable functions.
1762  MPM.addPass(GlobalDCEPass());
1763 
1764  if (PTO.MergeFunctions)
1765  MPM.addPass(MergeFunctionsPass());
1766 
1767  // Emit annotation remarks.
1769 
1770  return MPM;
1771 }
1772 
1774  bool LTOPreLink) {
1776  "buildO0DefaultPipeline should only be used with O0");
1777 
1778  ModulePassManager MPM(DebugLogging);
1779 
1780  // Add UniqueInternalLinkageNames Pass which renames internal linkage
1781  // symbols with unique names.
1782  if (PTO.UniqueLinkageNames)
1784 
1785  if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
1786  PGOOpt->Action == PGOOptions::IRUse))
1788  MPM,
1789  /* RunProfileGen */ (PGOOpt->Action == PGOOptions::IRInstr),
1790  /* IsCS */ false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1791 
1792  for (auto &C : PipelineStartEPCallbacks)
1793  C(MPM, Level);
1794  for (auto &C : PipelineEarlySimplificationEPCallbacks)
1795  C(MPM, Level);
1796 
1797  // Build a minimal pipeline based on the semantics required by LLVM,
1798  // which is just that always inlining occurs. Further, disable generating
1799  // lifetime intrinsics to avoid enabling further optimizations during
1800  // code generation.
1801  // However, we need to insert lifetime intrinsics to avoid invalid access
1802  // caused by multithreaded coroutines.
1804  /*InsertLifetimeIntrinsics=*/PTO.Coroutines));
1805 
1806  if (PTO.MergeFunctions)
1807  MPM.addPass(MergeFunctionsPass());
1808 
1809  if (EnableMatrix)
1810  MPM.addPass(
1812 
1813  if (!CGSCCOptimizerLateEPCallbacks.empty()) {
1814  CGSCCPassManager CGPM(DebugLogging);
1815  for (auto &C : CGSCCOptimizerLateEPCallbacks)
1816  C(CGPM, Level);
1817  if (!CGPM.isEmpty())
1819  }
1820  if (!LateLoopOptimizationsEPCallbacks.empty()) {
1821  LoopPassManager LPM(DebugLogging);
1822  for (auto &C : LateLoopOptimizationsEPCallbacks)
1823  C(LPM, Level);
1824  if (!LPM.isEmpty()) {
1827  }
1828  }
1829  if (!LoopOptimizerEndEPCallbacks.empty()) {
1830  LoopPassManager LPM(DebugLogging);
1831  for (auto &C : LoopOptimizerEndEPCallbacks)
1832  C(LPM, Level);
1833  if (!LPM.isEmpty()) {
1836  }
1837  }
1838  if (!ScalarOptimizerLateEPCallbacks.empty()) {
1839  FunctionPassManager FPM(DebugLogging);
1840  for (auto &C : ScalarOptimizerLateEPCallbacks)
1841  C(FPM, Level);
1842  if (!FPM.isEmpty())
1844  }
1845  if (!VectorizerStartEPCallbacks.empty()) {
1846  FunctionPassManager FPM(DebugLogging);
1847  for (auto &C : VectorizerStartEPCallbacks)
1848  C(FPM, Level);
1849  if (!FPM.isEmpty())
1851  }
1852 
1853  if (PTO.Coroutines) {
1855 
1856  CGSCCPassManager CGPM(DebugLogging);
1857  CGPM.addPass(CoroSplitPass());
1860 
1862  }
1863 
1864  for (auto &C : OptimizerLastEPCallbacks)
1865  C(MPM, Level);
1866 
1867  if (LTOPreLink)
1868  addRequiredLTOPreLinkPasses(MPM);
1869 
1870  return MPM;
1871 }
1872 
1874  AAManager AA;
1875 
1876  // The order in which these are registered determines their priority when
1877  // being queried.
1878 
1879  // First we register the basic alias analysis that provides the majority of
1880  // per-function local AA logic. This is a stateless, on-demand local set of
1881  // AA techniques.
1883 
1884  // Next we query fast, specialized alias analyses that wrap IR-embedded
1885  // information about aliasing.
1888 
1889  // Add support for querying global aliasing information when available.
1890  // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1891  // analysis, all that the `AAManager` can do is query for any *cached*
1892  // results from `GlobalsAA` through a readonly proxy.
1894 
1895  // Add target-specific alias analyses.
1896  if (TM)
1898 
1899  return AA;
1900 }
1901 
1903  if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1904  return None;
1905  int Count;
1906  if (Name.getAsInteger(0, Count) || Count <= 0)
1907  return None;
1908  return Count;
1909 }
1910 
1912  if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1913  return None;
1914  int Count;
1915  if (Name.getAsInteger(0, Count) || Count < 0)
1916  return None;
1917  return Count;
1918 }
1919 
1921  if (!Name.consume_front(PassName))
1922  return false;
1923  // normal pass name w/o parameters == default parameters
1924  if (Name.empty())
1925  return true;
1926  return Name.startswith("<") && Name.endswith(">");
1927 }
1928 
1929 namespace {
1930 
1931 /// This performs customized parsing of pass name with parameters.
1932 ///
1933 /// We do not need parametrization of passes in textual pipeline very often,
1934 /// yet on a rare occasion ability to specify parameters right there can be
1935 /// useful.
1936 ///
1937 /// \p Name - parameterized specification of a pass from a textual pipeline
1938 /// is a string in a form of :
1939 /// PassName '<' parameter-list '>'
1940 ///
1941 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1942 /// It takes a string-ref of parameters and returns either StringError or a
1943 /// parameter list in a form of a custom parameters type, all wrapped into
1944 /// Expected<> template class.
1945 ///
1946 template <typename ParametersParseCallableT>
1947 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1948  StringRef PassName) -> decltype(Parser(StringRef{})) {
1949  using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1950 
1951  StringRef Params = Name;
1952  if (!Params.consume_front(PassName)) {
1953  assert(false &&
1954  "unable to strip pass name from parametrized pass specification");
1955  }
1956  if (Params.empty())
1957  return ParametersT{};
1958  if (!Params.consume_front("<") || !Params.consume_back(">")) {
1959  assert(false && "invalid format for parametrized pass name");
1960  }
1961 
1962  Expected<ParametersT> Result = Parser(Params);
1963  assert((Result || Result.template errorIsA<StringError>()) &&
1964  "Pass parameter parser can only return StringErrors.");
1965  return Result;
1966 }
1967 
1968 /// Parser of parameters for LoopUnroll pass.
1969 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1970  LoopUnrollOptions UnrollOpts;
1971  while (!Params.empty()) {
1972  StringRef ParamName;
1973  std::tie(ParamName, Params) = Params.split(';');
1974  int OptLevel = StringSwitch<int>(ParamName)
1975  .Case("O0", 0)
1976  .Case("O1", 1)
1977  .Case("O2", 2)
1978  .Case("O3", 3)
1979  .Default(-1);
1980  if (OptLevel >= 0) {
1981  UnrollOpts.setOptLevel(OptLevel);
1982  continue;
1983  }
1984  if (ParamName.consume_front("full-unroll-max=")) {
1985  int Count;
1986  if (ParamName.getAsInteger(0, Count))
1987  return make_error<StringError>(
1988  formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1990  UnrollOpts.setFullUnrollMaxCount(Count);
1991  continue;
1992  }
1993 
1994  bool Enable = !ParamName.consume_front("no-");
1995  if (ParamName == "partial") {
1996  UnrollOpts.setPartial(Enable);
1997  } else if (ParamName == "peeling") {
1998  UnrollOpts.setPeeling(Enable);
1999  } else if (ParamName == "profile-peeling") {
2000  UnrollOpts.setProfileBasedPeeling(Enable);
2001  } else if (ParamName == "runtime") {
2002  UnrollOpts.setRuntime(Enable);
2003  } else if (ParamName == "upperbound") {
2004  UnrollOpts.setUpperBound(Enable);
2005  } else {
2006  return make_error<StringError>(
2007  formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
2009  }
2010  }
2011  return UnrollOpts;
2012 }
2013 
2014 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
2016  while (!Params.empty()) {
2017  StringRef ParamName;
2018  std::tie(ParamName, Params) = Params.split(';');
2019 
2020  if (ParamName == "recover") {
2021  Result.Recover = true;
2022  } else if (ParamName == "kernel") {
2023  Result.Kernel = true;
2024  } else if (ParamName.consume_front("track-origins=")) {
2025  if (ParamName.getAsInteger(0, Result.TrackOrigins))
2026  return make_error<StringError>(
2027  formatv("invalid argument to MemorySanitizer pass track-origins "
2028  "parameter: '{0}' ",
2029  ParamName)
2030  .str(),
2032  } else {
2033  return make_error<StringError>(
2034  formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
2035  .str(),
2037  }
2038  }
2039  return Result;
2040 }
2041 
2042 /// Parser of parameters for SimplifyCFG pass.
2043 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
2045  while (!Params.empty()) {
2046  StringRef ParamName;
2047  std::tie(ParamName, Params) = Params.split(';');
2048 
2049  bool Enable = !ParamName.consume_front("no-");
2050  if (ParamName == "forward-switch-cond") {
2051  Result.forwardSwitchCondToPhi(Enable);
2052  } else if (ParamName == "switch-to-lookup") {
2053  Result.convertSwitchToLookupTable(Enable);
2054  } else if (ParamName == "keep-loops") {
2055  Result.needCanonicalLoops(Enable);
2056  } else if (ParamName == "hoist-common-insts") {
2057  Result.hoistCommonInsts(Enable);
2058  } else if (ParamName == "sink-common-insts") {
2059  Result.sinkCommonInsts(Enable);
2060  } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
2061  APInt BonusInstThreshold;
2062  if (ParamName.getAsInteger(0, BonusInstThreshold))
2063  return make_error<StringError>(
2064  formatv("invalid argument to SimplifyCFG pass bonus-threshold "
2065  "parameter: '{0}' ",
2066  ParamName).str(),
2068  Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
2069  } else {
2070  return make_error<StringError>(
2071  formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
2073  }
2074  }
2075  return Result;
2076 }
2077 
2078 /// Parser of parameters for LoopVectorize pass.
2079 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
2080  LoopVectorizeOptions Opts;
2081  while (!Params.empty()) {
2082  StringRef ParamName;
2083  std::tie(ParamName, Params) = Params.split(';');
2084 
2085  bool Enable = !ParamName.consume_front("no-");
2086  if (ParamName == "interleave-forced-only") {
2087  Opts.setInterleaveOnlyWhenForced(Enable);
2088  } else if (ParamName == "vectorize-forced-only") {
2089  Opts.setVectorizeOnlyWhenForced(Enable);
2090  } else {
2091  return make_error<StringError>(
2092  formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
2094  }
2095  }
2096  return Opts;
2097 }
2098 
2099 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
2100  bool Result = false;
2101  while (!Params.empty()) {
2102  StringRef ParamName;
2103  std::tie(ParamName, Params) = Params.split(';');
2104 
2105  bool Enable = !ParamName.consume_front("no-");
2106  if (ParamName == "nontrivial") {
2107  Result = Enable;
2108  } else {
2109  return make_error<StringError>(
2110  formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
2111  .str(),
2113  }
2114  }
2115  return Result;
2116 }
2117 
2118 Expected<bool> parseMergedLoadStoreMotionOptions(StringRef Params) {
2119  bool Result = false;
2120  while (!Params.empty()) {
2121  StringRef ParamName;
2122  std::tie(ParamName, Params) = Params.split(';');
2123 
2124  bool Enable = !ParamName.consume_front("no-");
2125  if (ParamName == "split-footer-bb") {
2126  Result = Enable;
2127  } else {
2128  return make_error<StringError>(
2129  formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
2130  ParamName)
2131  .str(),
2133  }
2134  }
2135  return Result;
2136 }
2137 
2138 Expected<GVNOptions> parseGVNOptions(StringRef Params) {
2140  while (!Params.empty()) {
2141  StringRef ParamName;
2142  std::tie(ParamName, Params) = Params.split(';');
2143 
2144  bool Enable = !ParamName.consume_front("no-");
2145  if (ParamName == "pre") {
2146  Result.setPRE(Enable);
2147  } else if (ParamName == "load-pre") {
2148  Result.setLoadPRE(Enable);
2149  } else if (ParamName == "split-backedge-load-pre") {
2150  Result.setLoadPRESplitBackedge(Enable);
2151  } else if (ParamName == "memdep") {
2152  Result.setMemDep(Enable);
2153  } else {
2154  return make_error<StringError>(
2155  formatv("invalid GVN pass parameter '{0}' ", ParamName).str(),
2157  }
2158  }
2159  return Result;
2160 }
2161 
2163 parseStackLifetimeOptions(StringRef Params) {
2165  while (!Params.empty()) {
2166  StringRef ParamName;
2167  std::tie(ParamName, Params) = Params.split(';');
2168 
2169  if (ParamName == "may") {
2171  } else if (ParamName == "must") {
2173  } else {
2174  return make_error<StringError>(
2175  formatv("invalid StackLifetime parameter '{0}' ", ParamName).str(),
2177  }
2178  }
2179  return Result;
2180 }
2181 
2182 } // namespace
2183 
2184 /// Tests whether a pass name starts with a valid prefix for a default pipeline
2185 /// alias.
2187  return Name.startswith("default") || Name.startswith("thinlto") ||
2188  Name.startswith("lto");
2189 }
2190 
2191 /// Tests whether registered callbacks will accept a given pass name.
2192 ///
2193 /// When parsing a pipeline text, the type of the outermost pipeline may be
2194 /// omitted, in which case the type is automatically determined from the first
2195 /// pass name in the text. This may be a name that is handled through one of the
2196 /// callbacks. We check this through the oridinary parsing callbacks by setting
2197 /// up a dummy PassManager in order to not force the client to also handle this
2198 /// type of query.
2199 template <typename PassManagerT, typename CallbacksT>
2200 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
2201  if (!Callbacks.empty()) {
2202  PassManagerT DummyPM;
2203  for (auto &CB : Callbacks)
2204  if (CB(Name, DummyPM, {}))
2205  return true;
2206  }
2207  return false;
2208 }
2209 
2210 template <typename CallbacksT>
2211 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
2212  // Manually handle aliases for pre-configured pipeline fragments.
2214  return DefaultAliasRegex.match(Name);
2215 
2216  // Explicitly handle pass manager names.
2217  if (Name == "module")
2218  return true;
2219  if (Name == "cgscc")
2220  return true;
2221  if (Name == "function")
2222  return true;
2223 
2224  // Explicitly handle custom-parsed pass names.
2226  return true;
2227 
2228 #define MODULE_PASS(NAME, CREATE_PASS) \
2229  if (Name == NAME) \
2230  return true;
2231 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2232  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2233  return true;
2234 #include "PassRegistry.def"
2235 
2236  return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
2237 }
2238 
2239 template <typename CallbacksT>
2240 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
2241  // Explicitly handle pass manager names.
2242  if (Name == "cgscc")
2243  return true;
2244  if (Name == "function")
2245  return true;
2246 
2247  // Explicitly handle custom-parsed pass names.
2249  return true;
2251  return true;
2252 
2253 #define CGSCC_PASS(NAME, CREATE_PASS) \
2254  if (Name == NAME) \
2255  return true;
2256 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2257  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2258  return true;
2259 #include "PassRegistry.def"
2260 
2261  return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
2262 }
2263 
2264 template <typename CallbacksT>
2265 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
2266  // Explicitly handle pass manager names.
2267  if (Name == "function")
2268  return true;
2269  if (Name == "loop" || Name == "loop-mssa")
2270  return true;
2271 
2272  // Explicitly handle custom-parsed pass names.
2274  return true;
2275 
2276 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2277  if (Name == NAME) \
2278  return true;
2279 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2280  if (checkParametrizedPassName(Name, NAME)) \
2281  return true;
2282 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2283  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2284  return true;
2285 #include "PassRegistry.def"
2286 
2287  return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
2288 }
2289 
2290 template <typename CallbacksT>
2291 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
2292  // Explicitly handle pass manager names.
2293  if (Name == "loop" || Name == "loop-mssa")
2294  return true;
2295 
2296  // Explicitly handle custom-parsed pass names.
2298  return true;
2299 
2300 #define LOOP_PASS(NAME, CREATE_PASS) \
2301  if (Name == NAME) \
2302  return true;
2303 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2304  if (checkParametrizedPassName(Name, NAME)) \
2305  return true;
2306 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2307  if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
2308  return true;
2309 #include "PassRegistry.def"
2310 
2311  return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
2312 }
2313 
2315 PassBuilder::parsePipelineText(StringRef Text) {
2316  std::vector<PipelineElement> ResultPipeline;
2317 
2318  SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
2319  &ResultPipeline};
2320  for (;;) {
2321  std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
2322  size_t Pos = Text.find_first_of(",()");
2323  Pipeline.push_back({Text.substr(0, Pos), {}});
2324 
2325  // If we have a single terminating name, we're done.
2326  if (Pos == Text.npos)
2327  break;
2328 
2329  char Sep = Text[Pos];
2330  Text = Text.substr(Pos + 1);
2331  if (Sep == ',')
2332  // Just a name ending in a comma, continue.
2333  continue;
2334 
2335  if (Sep == '(') {
2336  // Push the inner pipeline onto the stack to continue processing.
2337  PipelineStack.push_back(&Pipeline.back().InnerPipeline);
2338  continue;
2339  }
2340 
2341  assert(Sep == ')' && "Bogus separator!");
2342  // When handling the close parenthesis, we greedily consume them to avoid
2343  // empty strings in the pipeline.
2344  do {
2345  // If we try to pop the outer pipeline we have unbalanced parentheses.
2346  if (PipelineStack.size() == 1)
2347  return None;
2348 
2349  PipelineStack.pop_back();
2350  } while (Text.consume_front(")"));
2351 
2352  // Check if we've finished parsing.
2353  if (Text.empty())
2354  break;
2355 
2356  // Otherwise, the end of an inner pipeline always has to be followed by
2357  // a comma, and then we can continue.
2358  if (!Text.consume_front(","))
2359  return None;
2360  }
2361 
2362  if (PipelineStack.size() > 1)
2363  // Unbalanced paretheses.
2364  return None;
2365 
2366  assert(PipelineStack.back() == &ResultPipeline &&
2367  "Wrong pipeline at the bottom of the stack!");
2368  return {std::move(ResultPipeline)};
2369 }
2370 
2371 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
2372  const PipelineElement &E) {
2373  auto &Name = E.Name;
2374  auto &InnerPipeline = E.InnerPipeline;
2375 
2376  // First handle complex passes like the pass managers which carry pipelines.
2377  if (!InnerPipeline.empty()) {
2378  if (Name == "module") {
2379  ModulePassManager NestedMPM(DebugLogging);
2380  if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2381  return Err;
2382  MPM.addPass(std::move(NestedMPM));
2383  return Error::success();
2384  }
2385  if (Name == "cgscc") {
2386  CGSCCPassManager CGPM(DebugLogging);
2387  if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline))
2388  return Err;
2390  return Error::success();
2391  }
2392  if (Name == "function") {
2393  FunctionPassManager FPM(DebugLogging);
2394  if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2395  return Err;
2397  return Error::success();
2398  }
2399  if (auto Count = parseRepeatPassName(Name)) {
2400  ModulePassManager NestedMPM(DebugLogging);
2401  if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline))
2402  return Err;
2403  MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
2404  return Error::success();
2405  }
2406 
2407  for (auto &C : ModulePipelineParsingCallbacks)
2408  if (C(Name, MPM, InnerPipeline))
2409  return Error::success();
2410 
2411  // Normal passes can't have pipelines.
2412  return make_error<StringError>(
2413  formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
2415  ;
2416  }
2417 
2418  // Manually handle aliases for pre-configured pipeline fragments.
2420  SmallVector<StringRef, 3> Matches;
2421  if (!DefaultAliasRegex.match(Name, &Matches))
2422  return make_error<StringError>(
2423  formatv("unknown default pipeline alias '{0}'", Name).str(),
2425 
2426  assert(Matches.size() == 3 && "Must capture two matched strings!");
2427 
2428  OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
2429  .Case("O0", OptimizationLevel::O0)
2430  .Case("O1", OptimizationLevel::O1)
2431  .Case("O2", OptimizationLevel::O2)
2432  .Case("O3", OptimizationLevel::O3)
2433  .Case("Os", OptimizationLevel::Os)
2434  .Case("Oz", OptimizationLevel::Oz);
2435  if (L == OptimizationLevel::O0 && Matches[1] != "thinlto" &&
2436  Matches[1] != "lto") {
2437  MPM.addPass(buildO0DefaultPipeline(L, Matches[1] == "thinlto-pre-link" ||
2438  Matches[1] == "lto-pre-link"));
2439  return Error::success();
2440  }
2441 
2442  // This is consistent with old pass manager invoked via opt, but
2443  // inconsistent with clang. Clang doesn't enable loop vectorization
2444  // but does enable slp vectorization at Oz.
2445  PTO.LoopVectorization =
2446  L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2447  PTO.SLPVectorization =
2448  L.getSpeedupLevel() > 1 && L != OptimizationLevel::Oz;
2449 
2450  if (Matches[1] == "default") {
2452  } else if (Matches[1] == "thinlto-pre-link") {
2454  } else if (Matches[1] == "thinlto") {
2455  MPM.addPass(buildThinLTODefaultPipeline(L, nullptr));
2456  } else if (Matches[1] == "lto-pre-link") {
2458  } else {
2459  assert(Matches[1] == "lto" && "Not one of the matched options!");
2460  MPM.addPass(buildLTODefaultPipeline(L, nullptr));
2461  }
2462  return Error::success();
2463  }
2464 
2465  // Finally expand the basic registered passes from the .inc file.
2466 #define MODULE_PASS(NAME, CREATE_PASS) \
2467  if (Name == NAME) { \
2468  MPM.addPass(CREATE_PASS); \
2469  return Error::success(); \
2470  }
2471 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
2472  if (Name == "require<" NAME ">") { \
2473  MPM.addPass( \
2474  RequireAnalysisPass< \
2475  std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
2476  return Error::success(); \
2477  } \
2478  if (Name == "invalidate<" NAME ">") { \
2479  MPM.addPass(InvalidateAnalysisPass< \
2480  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2481  return Error::success(); \
2482  }
2483 #define CGSCC_PASS(NAME, CREATE_PASS) \
2484  if (Name == NAME) { \
2485  MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(CREATE_PASS)); \
2486  return Error::success(); \
2487  }
2488 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2489  if (Name == NAME) { \
2490  MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS)); \
2491  return Error::success(); \
2492  }
2493 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2494  if (checkParametrizedPassName(Name, NAME)) { \
2495  auto Params = parsePassParameters(PARSER, Name, NAME); \
2496  if (!Params) \
2497  return Params.takeError(); \
2498  MPM.addPass(createModuleToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2499  return Error::success(); \
2500  }
2501 #define LOOP_PASS(NAME, CREATE_PASS) \
2502  if (Name == NAME) { \
2503  MPM.addPass( \
2504  createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2505  CREATE_PASS, false, false, DebugLogging))); \
2506  return Error::success(); \
2507  }
2508 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2509  if (checkParametrizedPassName(Name, NAME)) { \
2510  auto Params = parsePassParameters(PARSER, Name, NAME); \
2511  if (!Params) \
2512  return Params.takeError(); \
2513  MPM.addPass( \
2514  createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2515  CREATE_PASS(Params.get()), false, false, DebugLogging))); \
2516  return Error::success(); \
2517  }
2518 #include "PassRegistry.def"
2519 
2520  for (auto &C : ModulePipelineParsingCallbacks)
2521  if (C(Name, MPM, InnerPipeline))
2522  return Error::success();
2523  return make_error<StringError>(
2524  formatv("unknown module pass '{0}'", Name).str(),
2526 }
2527 
2528 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
2529  const PipelineElement &E) {
2530  auto &Name = E.Name;
2531  auto &InnerPipeline = E.InnerPipeline;
2532 
2533  // First handle complex passes like the pass managers which carry pipelines.
2534  if (!InnerPipeline.empty()) {
2535  if (Name == "cgscc") {
2536  CGSCCPassManager NestedCGPM(DebugLogging);
2537  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2538  return Err;
2539  // Add the nested pass manager with the appropriate adaptor.
2540  CGPM.addPass(std::move(NestedCGPM));
2541  return Error::success();
2542  }
2543  if (Name == "function") {
2544  FunctionPassManager FPM(DebugLogging);
2545  if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline))
2546  return Err;
2547  // Add the nested pass manager with the appropriate adaptor.
2549  return Error::success();
2550  }
2551  if (auto Count = parseRepeatPassName(Name)) {
2552  CGSCCPassManager NestedCGPM(DebugLogging);
2553  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2554  return Err;
2555  CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
2556  return Error::success();
2557  }
2558  if (auto MaxRepetitions = parseDevirtPassName(Name)) {
2559  CGSCCPassManager NestedCGPM(DebugLogging);
2560  if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline))
2561  return Err;
2562  CGPM.addPass(
2563  createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
2564  return Error::success();
2565  }
2566 
2567  for (auto &C : CGSCCPipelineParsingCallbacks)
2568  if (C(Name, CGPM, InnerPipeline))
2569  return Error::success();
2570 
2571  // Normal passes can't have pipelines.
2572  return make_error<StringError>(
2573  formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
2575  }
2576 
2577 // Now expand the basic registered passes from the .inc file.
2578 #define CGSCC_PASS(NAME, CREATE_PASS) \
2579  if (Name == NAME) { \
2580  CGPM.addPass(CREATE_PASS); \
2581  return Error::success(); \
2582  }
2583 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
2584  if (Name == "require<" NAME ">") { \
2585  CGPM.addPass(RequireAnalysisPass< \
2586  std::remove_reference<decltype(CREATE_PASS)>::type, \
2587  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
2588  CGSCCUpdateResult &>()); \
2589  return Error::success(); \
2590  } \
2591  if (Name == "invalidate<" NAME ">") { \
2592  CGPM.addPass(InvalidateAnalysisPass< \
2593  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2594  return Error::success(); \
2595  }
2596 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2597  if (Name == NAME) { \
2598  CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS)); \
2599  return Error::success(); \
2600  }
2601 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2602  if (checkParametrizedPassName(Name, NAME)) { \
2603  auto Params = parsePassParameters(PARSER, Name, NAME); \
2604  if (!Params) \
2605  return Params.takeError(); \
2606  CGPM.addPass(createCGSCCToFunctionPassAdaptor(CREATE_PASS(Params.get()))); \
2607  return Error::success(); \
2608  }
2609 #define LOOP_PASS(NAME, CREATE_PASS) \
2610  if (Name == NAME) { \
2611  CGPM.addPass( \
2612  createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2613  CREATE_PASS, false, false, DebugLogging))); \
2614  return Error::success(); \
2615  }
2616 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2617  if (checkParametrizedPassName(Name, NAME)) { \
2618  auto Params = parsePassParameters(PARSER, Name, NAME); \
2619  if (!Params) \
2620  return Params.takeError(); \
2621  CGPM.addPass( \
2622  createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
2623  CREATE_PASS(Params.get()), false, false, DebugLogging))); \
2624  return Error::success(); \
2625  }
2626 #include "PassRegistry.def"
2627 
2628  for (auto &C : CGSCCPipelineParsingCallbacks)
2629  if (C(Name, CGPM, InnerPipeline))
2630  return Error::success();
2631  return make_error<StringError>(
2632  formatv("unknown cgscc pass '{0}'", Name).str(),
2634 }
2635 
2636 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
2637  const PipelineElement &E) {
2638  auto &Name = E.Name;
2639  auto &InnerPipeline = E.InnerPipeline;
2640 
2641  // First handle complex passes like the pass managers which carry pipelines.
2642  if (!InnerPipeline.empty()) {
2643  if (Name == "function") {
2644  FunctionPassManager NestedFPM(DebugLogging);
2645  if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2646  return Err;
2647  // Add the nested pass manager with the appropriate adaptor.
2648  FPM.addPass(std::move(NestedFPM));
2649  return Error::success();
2650  }
2651  if (Name == "loop" || Name == "loop-mssa") {
2652  LoopPassManager LPM(DebugLogging);
2653  if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline))
2654  return Err;
2655  // Add the nested pass manager with the appropriate adaptor.
2656  bool UseMemorySSA = (Name == "loop-mssa");
2657  bool UseBFI = llvm::any_of(
2658  InnerPipeline, [](auto Pipeline) { return Pipeline.Name == "licm"; });
2659  FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
2660  UseBFI, DebugLogging));
2661  return Error::success();
2662  }
2663  if (auto Count = parseRepeatPassName(Name)) {
2664  FunctionPassManager NestedFPM(DebugLogging);
2665  if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline))
2666  return Err;
2667  FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
2668  return Error::success();
2669  }
2670 
2671  for (auto &C : FunctionPipelineParsingCallbacks)
2672  if (C(Name, FPM, InnerPipeline))
2673  return Error::success();
2674 
2675  // Normal passes can't have pipelines.
2676  return make_error<StringError>(
2677  formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
2679  }
2680 
2681 // Now expand the basic registered passes from the .inc file.
2682 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2683  if (Name == NAME) { \
2684  FPM.addPass(CREATE_PASS); \
2685  return Error::success(); \
2686  }
2687 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2688  if (checkParametrizedPassName(Name, NAME)) { \
2689  auto Params = parsePassParameters(PARSER, Name, NAME); \
2690  if (!Params) \
2691  return Params.takeError(); \
2692  FPM.addPass(CREATE_PASS(Params.get())); \
2693  return Error::success(); \
2694  }
2695 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2696  if (Name == "require<" NAME ">") { \
2697  FPM.addPass( \
2698  RequireAnalysisPass< \
2699  std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2700  return Error::success(); \
2701  } \
2702  if (Name == "invalidate<" NAME ">") { \
2703  FPM.addPass(InvalidateAnalysisPass< \
2704  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2705  return Error::success(); \
2706  }
2707 // FIXME: UseMemorySSA is set to false. Maybe we could do things like:
2708 // bool UseMemorySSA = !("canon-freeze" || "loop-predication" ||
2709 // "guard-widening");
2710 // The risk is that it may become obsolete if we're not careful.
2711 #define LOOP_PASS(NAME, CREATE_PASS) \
2712  if (Name == NAME) { \
2713  FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \
2714  DebugLogging)); \
2715  return Error::success(); \
2716  }
2717 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2718  if (checkParametrizedPassName(Name, NAME)) { \
2719  auto Params = parsePassParameters(PARSER, Name, NAME); \
2720  if (!Params) \
2721  return Params.takeError(); \
2722  FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
2723  false, false, DebugLogging)); \
2724  return Error::success(); \
2725  }
2726 #include "PassRegistry.def"
2727 
2728  for (auto &C : FunctionPipelineParsingCallbacks)
2729  if (C(Name, FPM, InnerPipeline))
2730  return Error::success();
2731  return make_error<StringError>(
2732  formatv("unknown function pass '{0}'", Name).str(),
2734 }
2735 
2736 Error PassBuilder::parseLoopPass(LoopPassManager &LPM,
2737  const PipelineElement &E) {
2738  StringRef Name = E.Name;
2739  auto &InnerPipeline = E.InnerPipeline;
2740 
2741  // First handle complex passes like the pass managers which carry pipelines.
2742  if (!InnerPipeline.empty()) {
2743  if (Name == "loop") {
2744  LoopPassManager NestedLPM(DebugLogging);
2745  if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2746  return Err;
2747  // Add the nested pass manager with the appropriate adaptor.
2748  LPM.addPass(std::move(NestedLPM));
2749  return Error::success();
2750  }
2751  if (auto Count = parseRepeatPassName(Name)) {
2752  LoopPassManager NestedLPM(DebugLogging);
2753  if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline))
2754  return Err;
2755  LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2756  return Error::success();
2757  }
2758 
2759  for (auto &C : LoopPipelineParsingCallbacks)
2760  if (C(Name, LPM, InnerPipeline))
2761  return Error::success();
2762 
2763  // Normal passes can't have pipelines.
2764  return make_error<StringError>(
2765  formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2767  }
2768 
2769 // Now expand the basic registered passes from the .inc file.
2770 #define LOOP_PASS(NAME, CREATE_PASS) \
2771  if (Name == NAME) { \
2772  LPM.addPass(CREATE_PASS); \
2773  return Error::success(); \
2774  }
2775 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2776  if (checkParametrizedPassName(Name, NAME)) { \
2777  auto Params = parsePassParameters(PARSER, Name, NAME); \
2778  if (!Params) \
2779  return Params.takeError(); \
2780  LPM.addPass(CREATE_PASS(Params.get())); \
2781  return Error::success(); \
2782  }
2783 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2784  if (Name == "require<" NAME ">") { \
2785  LPM.addPass(RequireAnalysisPass< \
2786  std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2787  LoopAnalysisManager, LoopStandardAnalysisResults &, \
2788  LPMUpdater &>()); \
2789  return Error::success(); \
2790  } \
2791  if (Name == "invalidate<" NAME ">") { \
2792  LPM.addPass(InvalidateAnalysisPass< \
2793  std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2794  return Error::success(); \
2795  }
2796 #include "PassRegistry.def"
2797 
2798  for (auto &C : LoopPipelineParsingCallbacks)
2799  if (C(Name, LPM, InnerPipeline))
2800  return Error::success();
2801  return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2803 }
2804 
2805 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2806 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2807  if (Name == NAME) { \
2808  AA.registerModuleAnalysis< \
2809  std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2810  return true; \
2811  }
2812 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2813  if (Name == NAME) { \
2814  AA.registerFunctionAnalysis< \
2815  std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2816  return true; \
2817  }
2818 #include "PassRegistry.def"
2819 
2820  for (auto &C : AAParsingCallbacks)
2821  if (C(Name, AA))
2822  return true;
2823  return false;
2824 }
2825 
2826 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2827  ArrayRef<PipelineElement> Pipeline) {
2828  for (const auto &Element : Pipeline) {
2829  if (auto Err = parseLoopPass(LPM, Element))
2830  return Err;
2831  }
2832  return Error::success();
2833 }
2834 
2835 Error PassBuilder::parseFunctionPassPipeline(
2837  for (const auto &Element : Pipeline) {
2838  if (auto Err = parseFunctionPass(FPM, Element))
2839  return Err;
2840  }
2841  return Error::success();
2842 }
2843 
2844 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2845  ArrayRef<PipelineElement> Pipeline) {
2846  for (const auto &Element : Pipeline) {
2847  if (auto Err = parseCGSCCPass(CGPM, Element))
2848  return Err;
2849  }
2850  return Error::success();
2851 }
2852 
2855  CGSCCAnalysisManager &CGAM,
2856  ModuleAnalysisManager &MAM) {
2857  MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2858  MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2859  CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2860  FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2861  FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2862  FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2863  LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2864 }
2865 
2866 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2867  ArrayRef<PipelineElement> Pipeline) {
2868  for (const auto &Element : Pipeline) {
2869  if (auto Err = parseModulePass(MPM, Element))
2870  return Err;
2871  }
2872  return Error::success();
2873 }
2874 
2875 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2876 // FIXME: Should this routine accept a TargetMachine or require the caller to
2877 // pre-populate the analysis managers with target-specific stuff?
2879  StringRef PipelineText) {
2880  auto Pipeline = parsePipelineText(PipelineText);
2881  if (!Pipeline || Pipeline->empty())
2882  return make_error<StringError>(
2883  formatv("invalid pipeline '{0}'", PipelineText).str(),
2885 
2886  // If the first name isn't at the module layer, wrap the pipeline up
2887  // automatically.
2888  StringRef FirstName = Pipeline->front().Name;
2889 
2890  if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2891  if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2892  Pipeline = {{"cgscc", std::move(*Pipeline)}};
2893  } else if (isFunctionPassName(FirstName,
2894  FunctionPipelineParsingCallbacks)) {
2895  Pipeline = {{"function", std::move(*Pipeline)}};
2896  } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2897  Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2898  } else {
2899  for (auto &C : TopLevelPipelineParsingCallbacks)
2900  if (C(MPM, *Pipeline, DebugLogging))
2901  return Error::success();
2902 
2903  // Unknown pass or pipeline name!
2904  auto &InnerPipeline = Pipeline->front().InnerPipeline;
2905  return make_error<StringError>(
2906  formatv("unknown {0} name '{1}'",
2907  (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2908  .str(),
2910  }
2911  }
2912 
2913  if (auto Err = parseModulePassPipeline(MPM, *Pipeline))
2914  return Err;
2915  return Error::success();
2916 }
2917 
2918 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2920  StringRef PipelineText) {
2921  auto Pipeline = parsePipelineText(PipelineText);
2922  if (!Pipeline || Pipeline->empty())
2923  return make_error<StringError>(
2924  formatv("invalid pipeline '{0}'", PipelineText).str(),
2926 
2927  StringRef FirstName = Pipeline->front().Name;
2928  if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2929  return make_error<StringError>(
2930  formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2931  PipelineText)
2932  .str(),
2934 
2935  if (auto Err = parseCGSCCPassPipeline(CGPM, *Pipeline))
2936  return Err;
2937  return Error::success();
2938 }
2939 
2940 // Primary pass pipeline description parsing routine for a \c
2941 // FunctionPassManager
2943  StringRef PipelineText) {
2944  auto Pipeline = parsePipelineText(PipelineText);
2945  if (!Pipeline || Pipeline->empty())
2946  return make_error<StringError>(
2947  formatv("invalid pipeline '{0}'", PipelineText).str(),
2949 
2950  StringRef FirstName = Pipeline->front().Name;
2951  if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2952  return make_error<StringError>(
2953  formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2954  PipelineText)
2955  .str(),
2957 
2958  if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline))
2959  return Err;
2960  return Error::success();
2961 }
2962 
2963 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2965  StringRef PipelineText) {
2966  auto Pipeline = parsePipelineText(PipelineText);
2967  if (!Pipeline || Pipeline->empty())
2968  return make_error<StringError>(
2969  formatv("invalid pipeline '{0}'", PipelineText).str(),
2971 
2972  if (auto Err = parseLoopPassPipeline(CGPM, *Pipeline))
2973  return Err;
2974 
2975  return Error::success();
2976 }
2977 
2979  // If the pipeline just consists of the word 'default' just replace the AA
2980  // manager with our default one.
2981  if (PipelineText == "default") {
2982  AA = buildDefaultAAPipeline();
2983  return Error::success();
2984  }
2985 
2986  while (!PipelineText.empty()) {
2987  StringRef Name;
2988  std::tie(Name, PipelineText) = PipelineText.split(',');
2989  if (!parseAAPassName(AA, Name))
2990  return make_error<StringError>(
2991  formatv("unknown alias analysis name '{0}'", Name).str(),
2993  }
2994 
2995  return Error::success();
2996 }
2997 
2999 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3000  if (PassName == NAME) \
3001  return true;
3002 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3003  if (PassName == NAME) \
3004  return true;
3005 #include "PassRegistry.def"
3006  return false;
3007 }
3008 
3010 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
3011  if (PassName == NAME) \
3012  return true;
3013 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
3014  if (PassName == NAME) \
3015  return true;
3016 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
3017  if (PassName == NAME) \
3018  return true;
3019 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
3020  if (PassName == NAME) \
3021  return true;
3022 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3023  if (PassName == NAME) \
3024  return true;
3025 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
3026  if (PassName == NAME) \
3027  return true;
3028 #include "PassRegistry.def"
3029  return false;
3030 }
3031 
3034  bool DebugLogging)> &C) {
3035  TopLevelPipelineParsingCallbacks.push_back(C);
3036 }
Super simple passes to force specific function attrs from the commandline into the IR for debugging p...
Defines passes for running instruction simplification across chunks of IR.
This file provides the interface for the pass responsible for both simplifying and canonicalizing the...
Analysis pass providing a never-invalidated alias analysis result.
const NoneType None
Definition: None.h:23
bool isEmpty() const
Returns if the pass manager contains any passes.
Definition: PassManager.h:574
A set of parameters used to control various transforms performed by the LoopUnroll pass.
uint64_t CallInst * C
Computes function attributes in post-order over the call graph.
Definition: FunctionAttrs.h:50
A simple loop rotation transformation.
Definition: LoopRotation.h:23
ModulePassManager buildLTODefaultPipeline(OptimizationLevel Level, ModuleSummaryIndex *ExportSummary)
Build an LTO default optimization pipeline to a pass manager.
A simple and fast domtree-based CSE pass.
Definition: EarlyCSE.h:30
cl::opt< bool > EnableLoopFlatten
The profile annotation (profile-instr-use) pass for IR based PGO.
cl::opt< bool > EnableIROutliner
Thresholds to tune inline cost analysis.
Definition: InlineCost.h:160
bool LoopVectorization
Tuning option to enable/disable loop vectorization, set based on opt level.
Definition: PassBuilder.h:96
Interfaces for registering analysis passes, producing common pass manager configurations,...
DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT Pass, int MaxIterations)
A function to deduce a function pass type and wrap it in the templated adaptor.
Public interface to the memory profiler pass for instrumenting code to profile memory accesses.
Definition: MemProfiler.h:29
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition: Pass.h:73
AAManager buildDefaultAAPipeline()
Build the default AAManager with the default alias analysis pipeline registered.
Eliminate dead arguments (and return values) from functions.
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
This is the interface for LLVM's inclusion-based alias analysis implemented with CFL graph reachabili...
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static Optional< int > parseRepeatPassName(StringRef Name)
This header provides classes for managing a pipeline of passes over loops in LLVM IR.
cl::opt< bool > ForgetSCEVInLoopUnroll
This is the interface for a simple mod/ref and alias analysis over globals.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
bool Coroutines
Tuning option to enable/disable coroutine intrinsic lowering.
Definition: PassBuilder.h:113
Error parseAAPipeline(AAManager &AA, StringRef PipelineText)
Parse a textual alias analysis pipeline into the provided AA manager.
This is the interface to build a ModuleSummaryIndex for a module.
OpenMP optimizations pass.
Definition: OpenMPOpt.h:65
UseBFI
Definition: MachineLICM.cpp:88
Uses an "inverted" value numbering to decide the similarity of expressions and sinks similar expressi...
Definition: GVN.h:366
InnerAnalysisManagerProxy< LoopAnalysisManager, Function > LoopAnalysisManagerFunctionProxy
A proxy from a LoopAnalysisManager to a Function.
Performs basic CFG simplifications to assist other loop passes.
static const OptimizationLevel Os
Similar to O2 but tries to optimize for small code size instead of fast execution without triggering ...
Definition: PassBuilder.h:244
This file provides the interface for LLVM's Scalar Replacement of Aggregates pass.
ThinLTO postlink (backend compile) phase.
This file implements a simple N^2 alias analysis accuracy evaluator.
InnerAnalysisManagerProxy< CGSCCAnalysisManager, Module > CGSCCAnalysisManagerModuleProxy
A proxy from a CGSCCAnalysisManager to a Module.
Implements a lazy call graph analysis and related passes for the new pass manager.
ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Build a per-module default optimization pipeline.
RepeatedPass< PassT > createRepeatedPass(int Count, PassT P)
Definition: PassManager.h:1363
This file implements the Loop Fusion pass.
void registerModuleAnalyses(ModuleAnalysisManager &MAM)
Registers all available module analysis passes.
bool consume_front(StringRef Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
Definition: StringRef.h:683
This is the interface for a metadata-based scoped no-alias analysis.
void registerModuleAnalysis()
Register a specific AA result.
LoopUnrollOptions & setFullUnrollMaxCount(unsigned O)
The instrumentation pass for recording function order.
LoopUnrollOptions & setUpperBound(bool UpperBound)
Enables or disables the use of trip count upper bound in loop unrolling.
auto formatv(const char *Fmt, Ts &&... Vals) -> formatv_object< decltype(std::make_tuple(detail::build_format_adapter(std::forward< Ts >(Vals))...))>
Pass to remove unused function declarations.
This file defines the interface for the loop nest analysis.
Public interface to the memory profiler module pass for instrumenting code to profile memory allocati...
Definition: MemProfiler.h:38
ModulePassManager buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, LTO-targeting default optimization pipeline to a pass manager.
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
OuterAnalysisManagerProxy< FunctionAnalysisManager, Loop, LoopStandardAnalysisResults & > FunctionAnalysisManagerLoopProxy
A proxy from a FunctionAnalysisManager to a Loop.
The adaptor from a function pass to a loop pass computes these analyses and makes them available to t...
F(f)
Loop unroll pass that only does full loop unrolling.
Loop unroll pass that will support both full and partial unrolling.
static const char * name
Pass to perform split of global variables.
Definition: GlobalSplit.h:26
This pass transforms loops that contain branches or switches on loop- invariant conditions to have mu...
A DCE pass that assumes instructions are dead until proven otherwise.
Definition: ADCE.h:31
static cl::opt< bool > EnableSyntheticCounts("enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Run synthetic function entry count generation " "pass"))
FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM function canonicalization and simplification pipeline.
A utility pass template to force an analysis result to be available.
Definition: PassManager.h:1273
static cl::opt< bool > EnableMemProfiler("enable-mem-prof", cl::init(false), cl::Hidden, cl::ZeroOrMore, cl::desc("Enable memory profiler"))
ThinLTO prelink (summary) phase.
Tunable parameters for passes in the default pipelines.
Definition: PassBuilder.h:85
Reassociate commutative expressions.
Definition: Reassociate.h:71
Simple pass that canonicalizes aliases.
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
Pass to convert @llvm.global.annotations to !annotation metadata.
cl::opt< bool > EnableHotColdSplit
Optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
Definition: InlineCost.h:190
bool isAnalysisPassName(StringRef PassName)
Returns true if the pass name is the name of a (non-alias) analysis pass.
Run instruction simplification across each instruction in the function.
Pass to forward loads in a loop around the backedge to subsequent iterations.
static cl::opt< bool > EnablePGOInlineDeferral("enable-npm-pgo-inline-deferral", cl::init(true), cl::Hidden, cl::desc("Enable inline deferral during PGO"))
Flag to enable inline deferral during PGO.
A pass that transforms external global definitions into declarations.
PassBuilder(bool DebugLogging=false, TargetMachine *TM=nullptr, PipelineTuningOptions PTO=PipelineTuningOptions(), Optional< PGOOptions > PGOOpt=None, PassInstrumentationCallbacks *PIC=nullptr)
This pass handles simple speculating of instructions around PHIs when doing so is profitable for a pa...
A simple and fast domtree-based GVN pass to hoist common expressions from sibling branches.
Definition: GVN.h:359
This is the interface for a SCEV-based alias analysis.
cl::opt< bool > RunPartialInlining
ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM module canonicalization and simplification pipeline.
A pass to do RPO deduction and propagation of function attributes.
Definition: FunctionAttrs.h:69
Optional< int > HintThreshold
Threshold to use for callees with inline hint.
Definition: InlineCost.h:165
Instrumentation based profiling lowering pass.
static const OptimizationLevel O3
Optimize for fast execution as much as possible.
Definition: PassBuilder.h:233
std::enable_if_t< is_detected< HasRunOnLoopT, PassT >::value > addPass(PassT Pass)
std::vector< std::string > printBeforePasses()
Definition: PrintPasses.cpp:73
bool registerPass(PassBuilderT &&PassBuilder)
Register an analysis pass with the manager.
Definition: PassManager.h:847
ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase, bool MandatoryOnly)
Construct the module pipeline that performs inlining as well as the inlining-driven cleanups.
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
std::enable_if_t<!std::is_same< PassT, PassManager >::value > addPass(PassT Pass)
Definition: PassManager.h:553
bool MergeFunctions
Tuning option to enable/disable function merging.
Definition: PassBuilder.h:129
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
Definition: PassManager.h:1052
Hoist/decompose integer division and remainder instructions to enable CFG improvements and better cod...
Definition: DivRemPairs.h:23
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
Contains a collection of routines for determining if a given instruction is guaranteed to execute if ...
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:612
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:156
OuterAnalysisManagerProxy< CGSCCAnalysisManager, Function > CGSCCAnalysisManagerFunctionProxy
A proxy from a CGSCCAnalysisManager to a Function.
static constexpr size_t npos
Definition: StringRef.h:59
void registerLoopAnalyses(LoopAnalysisManager &LAM)
Registers all available loop analysis passes.
Optimize scalar/vector interactions in IR using target cost models.
Definition: VectorCombine.h:23
The instrumentation (profile-instr-gen) pass for IR based PGO.
Key
PAL metadata keys.
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
The sample profiler data loader pass.
Definition: SampleProfile.h:25
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
bool match(StringRef String, SmallVectorImpl< StringRef > *Matches=nullptr, std::string *Error=nullptr) const
matches - Match the regex against a given String.
Definition: Regex.cpp:86
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1643
static bool isModulePassName(StringRef Name, CallbacksT &Callbacks)
This file provides the interface for a simple, fast CSE pass.
This file provides the interface for the GCOV style profiler pass.
Performs Loop Invariant Code Motion Pass.
Definition: LICM.h:46
The core GVN pass object.
Definition: GVN.h:115
cl::opt< unsigned > MaxDevirtIterations
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition: PassManager.h:374
void crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, ModuleAnalysisManager &MAM)
Cross register the analysis managers through their proxies.
Merge identical functions.
Pass to remove unused function declarations.
Definition: GlobalDCE.h:29
A lazily constructed view of the call graph of a module.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
A pass that does profile-guided sinking of instructions into loops.
Definition: LoopSink.h:33
virtual void registerDefaultAliasAnalyses(AAManager &)
Allow the target to register alias analyses with the AAManager for use with the new pass manager.
void registerFunctionAnalysis()
Register a specific AA result.
cl::opt< bool > EnableOrderFileInstrumentation
This class implements a trivial dead store elimination.
cl::opt< bool > DisablePreInliner
Class to hold module path string table and global value map, and encapsulate methods for operating on...
Argument promotion pass.
OuterAnalysisManagerProxy< ModuleAnalysisManager, LazyCallGraph::SCC, LazyCallGraph & > ModuleAnalysisManagerCGSCCProxy
A proxy from a ModuleAnalysisManager to an SCC.
LoopUnrollOptions & setPartial(bool Partial)
Enables or disables partial unrolling.
Pass to perform interprocedural constant propagation.
Definition: SCCP.h:30
A pass which infers function attributes from the names and signatures of function declarations in a m...
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:511
cl::opt< bool > FlattenedProfileUsed
LoopVectorizeOptions & setInterleaveOnlyWhenForced(bool Value)
bool UniqueLinkageNames
Uniquefy function linkage name. Its default value is false.
Definition: PassBuilder.h:132
ModulePassManager buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, ThinLTO-targeting default optimization pipeline to a pass manager.
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
This is the interface for LLVM's unification-based alias analysis implemented with CFL graph reachabi...
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:427
static bool checkParametrizedPassName(StringRef Name, StringRef PassName)
cl::opt< bool > EnableConstraintElimination
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
This is the interface for a metadata-based TBAA.
This file provides the interface for the pseudo probe implementation for AutoFDO.
cl::opt< bool > RunNewGVN
bool isAAPassName(StringRef PassName)
Returns true if the pass name is the name of an alias analysis pass.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:683
This file provides the primary interface to the instcombine pass.
A manager for alias analyses.
This pass attempts to minimize the number of assume without loosing any information.
A CRTP mix-in that provides informational APIs needed for analysis passes.
Definition: PassManager.h:391
A set of parameters to control various transforms performed by GVN pass.
Definition: GVN.h:72
static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks)
bool ForgetAllSCEVInLoopUnroll
Tuning option to forget all SCEV loops in LoopUnroll.
Definition: PassBuilder.h:107
void addPGOInstrPassesForO0(ModulePassManager &MPM, bool RunProfileGen, bool IsCS, std::string ProfileFile, std::string ProfileRemappingFile)
Add PGOInstrumenation passes for O0 only.
cl::opt< unsigned > SetLicmMssaOptCap
void registerFunctionAnalyses(FunctionAnalysisManager &FAM)
Registers all available function analysis passes.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1505
Analysis pass providing a never-invalidated alias analysis result.
This file provides the interface for LLVM's Global Value Numbering pass.
std::enable_if_t< is_detected< HasRunOnLoopT, LoopPassT >::value, FunctionToLoopPassAdaptor > createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA=false, bool UseBlockFrequencyInfo=false, bool DebugLogging=false)
A function to deduce a loop pass type and wrap it in the templated adaptor.
void addRequiredModuleAnalysis()
Allow adding module-level analyses benefiting the contained CGSCC passes.
Definition: Inliner.h:135
A simple loop rotation transformation.
LoopUnrollOptions & setProfileBasedPeeling(int O)
static const OptimizationLevel Oz
A very specialized mode that will optimize for code size at any and all costs.
Definition: PassBuilder.h:252
static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks)
Tests whether registered callbacks will accept a given pass name.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
std::string InstrProfileOutput
This file declares a simple ARC-aware AliasAnalysis using special knowledge of Objective C to enhance...
This file provides the interface for LLVM's Loop Data Prefetching Pass.
Provides passes to inlining "always_inline" functions.
This file provides the primary interface to the aggressive instcombine pass.
LoopUnrollOptions & setOptLevel(int O)
static const char PassName[]
This class provides an interface for updating the loop pass manager based on mutations to the loop ne...
Simple pass that provides a name to every anonymous globals.
static const OptimizationLevel O0
Disable as many optimizations as possible.
Definition: PassBuilder.h:185
This header defines the LoopLoadEliminationPass object.
static ErrorSuccess success()
Create a success value.
Definition: Error.h:332
bool LoopInterleaving
Tuning option to set loop interleaving on/off, set based on opt level.
Definition: PassBuilder.h:92
CGSCCPassManager & getPM()
Allow adding more CGSCC passes, besides inlining.
Definition: Inliner.h:132
LoopVectorizeOptions & setVectorizeOnlyWhenForced(bool Value)
InlineParams getInlineParams()
Generate the parameters to tune the inline cost analysis based only on the commandline options.
static Optional< int > parseDevirtPassName(StringRef Name)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1042
Full LTO postlink (backend compile) phase.
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:730
This pass performs 'jump threading', which looks at blocks that have multiple predecessors and multip...
Definition: JumpThreading.h:79
A pass to simplify and canonicalize the CFG of a function.
Definition: SimplifyCFG.h:29
static const Regex DefaultAliasRegex("^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$")
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
static InlineParams getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level)
PipelineTuningOptions()
Constructor sets pipeline tuning defaults based on cl::opts.
cl::opt< bool > EnableUnrollAndJam
void registerParseTopLevelPipelineCallback(const std::function< bool(ModulePassManager &, ArrayRef< PipelineElement >, bool DebugLogging)> &C)
Register a callback for a top-level pipeline entry.
Module pass, wrapping the inliner pass.
Definition: Inliner.h:119
The instrumentation (profile-instr-gen) pass for IR based PGO.
Error parsePassPipeline(ModulePassManager &MPM, StringRef PipelineText)
Parse a textual pass pipeline description into a ModulePassManager.
Pass to outline similar regions.
Definition: IROutliner.h:350
This pass converts vector operations into scalar operations, in order to expose optimization opportun...
Class for arbitrary precision integers.
Definition: APInt.h:70
Inlines functions marked as "always_inline".
Definition: AlwaysInliner.h:29
cl::opt< bool > EnableKnowledgeRetention
unsigned LicmMssaOptCap
Tuning option to cap the number of calls to retrive clobbering accesses in MemorySSA,...
Definition: PassBuilder.h:117
A pass that merges duplicate global constants into a single constant.
Definition: ConstantMerge.h:29
Analysis pass providing a never-invalidated alias analysis result.
bool LoopUnrolling
Tuning option to enable/disable loop unrolling. Its default value is true.
Definition: PassBuilder.h:103
Options for the frontend instrumentation based profiling pass.
Simple pass that provides a name to every anonymous globals.
This pass performs merges of loads and stores on both sides of a.
The indirect function call promotion pass.
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:658
Interfaces for passes which infer implicit function attributes from the name and signature of functio...
Analysis pass providing a never-invalidated alias analysis result.
std::vector< std::string > printAfterPasses()
Definition: PrintPasses.cpp:77
No LTO/ThinLTO behavior needed.
LoopUnrollOptions & setRuntime(bool Runtime)
Enables or disables unrolling of loops with runtime trip count.
cl::opt< unsigned > SetLicmMssaNoAccForPromotionCap
static cl::opt< InliningAdvisorMode > UseInlineAdvisor("enable-ml-inliner", cl::init(InliningAdvisorMode::Default), cl::Hidden, cl::desc("Enable ML policy for inliner. Currently trained for -Oz only"), cl::values(clEnumValN(InliningAdvisorMode::Default, "default", "Heuristics-based inliner version."), clEnumValN(InliningAdvisorMode::Development, "development", "Use development mode (runtime-loadable model)."), clEnumValN(InliningAdvisorMode::Release, "release", "Use release mode (AOT-compiled model).")))
cl::opt< bool > EnableMSSALoopDependency
Enables memory ssa as a dependency for loop passes.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:529
static const OptimizationLevel O2
Optimize for fast execution as much as possible without triggering significant incremental compile ti...
Definition: PassBuilder.h:218
cl::opt< int > PreInlineThreshold
unsigned LicmMssaNoAccForPromotionCap
Tuning option to disable promotion to scalars in LICM with MemorySSA, if the number of access is too ...
Definition: PassBuilder.h:121
Pass which forces specific function attributes into the IR, primarily as a debugging tool.
LLVM_NODISCARD size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:413
cl::opt< AttributorRunOption > AttributorRun
The header file for the LowerExpectIntrinsic pass as used by the new pass manager.
Pass to outline cold regions.
ModulePassManager buildO0DefaultPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Build an O0 pipeline with the minimal semantically required passes.
cl::opt< bool > EnableMatrix
Provides passes for computing function attributes based on interprocedural analyses.
This file defines the interface for the loop cache analysis.
bool CallGraphProfile
Tuning option to enable/disable call graph profile.
Definition: PassBuilder.h:125
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Definition: PassManager.h:1206
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
Definition: StringRef.h:693
This header provides classes for managing passes over SCCs of the call graph.
static const OptimizationLevel O1
Optimize quickly without destroying debuggability.
Definition: PassBuilder.h:201
cl::opt< bool > EnableGVNSink
This file defines passes to print out IR in various granularities.
CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
void registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM)
Registers all available CGSCC analysis passes.
static bool startsWithDefaultPipelineAliasPrefix(StringRef Name)
Tests whether a pass name starts with a valid prefix for a default pipeline alias.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file provides the interface for LLVM's Call Graph Profile pass.
The profile size based optimization pass for memory intrinsics.
See the comments on JumpThreadingPass.
LLVM_NODISCARD char front() const
front - Get the first character in the string.
Definition: StringRef.h:164
LLVM-provided high-level optimization levels.
Definition: PassBuilder.h:166
An SCC of the call graph.
Performs Loop Inst Simplify Pass.
The LoopVectorize Pass.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks)
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
An optimization pass providing Scalar Replacement of Aggregates.
Definition: SROA.h:65
This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
Definition: PassManager.h:1252
print Print MemDeps of function
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1556
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
This is the interface for LLVM's primary stateless and local alias analysis.
A container for analyses that lazily runs them and caches their results.
ModulePassManager buildModuleOptimizationPipeline(OptimizationLevel Level, bool LTOPreLink=false)
Construct the core LLVM module optimization pipeline.
This pass exposes codegen information to IR-level passes.
ModulePassManager buildThinLTODefaultPipeline(OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary)
Build an ThinLTO default optimization pipeline to a pass manager.
static cl::opt< bool > PerformMandatoryInliningsFirst("mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Perform mandatory inlinings module-wide, before performing " "inlining."))
cl::opt< bool > EnableGVNHoist
This pass performs function-level constant propagation and merging.
Definition: SCCP.h:36
This header defines various interfaces for pass management in LLVM.
static void addAnnotationRemarksPass(ModulePassManager &MPM)
LoopUnrollOptions & setPeeling(bool Peeling)
Enables or disables loop peeling.
cl::opt< bool > EnableCHR
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:72
The header file for the LowerConstantIntrinsics pass as used by the new pass manager.
bool SLPVectorization
Tuning option to enable/disable slp loop vectorization, set based on opt level.
Definition: PassBuilder.h:100
This file provides the interface for the sampled PGO loader pass.
Optional< int > HotCallSiteThreshold
Threshold to use when the callsite is considered hot.
Definition: InlineCost.h:177
int DefaultThreshold
The default threshold to start with for a callee.
Definition: InlineCost.h:162
Optimize globals that never have their address taken.
Definition: GlobalOpt.h:25
Performs Loop Idiom Recognize Pass.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
Level
Definition: Debugify.cpp:34
static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks)