LLVM 24.0.0git
PassBuilderPipelines.cpp
Go to the documentation of this file.
1//===- Construction of pass pipelines -------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file provides the implementation of the PassBuilder based on our
11/// static pass registry as well as related functionality. It also provides
12/// helpers to aid in analyzing, debugging, and testing passes and pass
13/// pipelines.
14///
15//===----------------------------------------------------------------------===//
16
17#include "PassesOptions.h"
18#include "llvm/ADT/Statistic.h"
30#include "llvm/IR/PassManager.h"
31#include "llvm/IR/Verifier.h"
32#include "llvm/Pass.h"
161
162using namespace llvm;
163
164namespace llvm {
166
168
170} // namespace llvm
171
173 const cl::LibraryOptionsContext &LibraryOptions) const {
174 const PassesOptions &Opts = LibraryOptions.get<PassesOptions>();
175 PipelineTuningOptions R = *this;
176 R.ForgetAllSCEVInLoopUnroll = ForgetAllSCEVInLoopUnroll.value_or(
177 getForgetSCEVInLoopUnroll(LibraryOptions));
178 R.LicmMssaOptCap = LicmMssaOptCap.value_or(getLicmMssaOptCap(LibraryOptions));
179 R.LicmMssaNoAccForPromotionCap = LicmMssaNoAccForPromotionCap.value_or(
180 getLicmMssaNoAccForPromotionCap(LibraryOptions));
181 R.MergeFunctions = MergeFunctions.value_or(Opts.enable_merge_functions);
182 R.MaxDevirtIterations =
183 MaxDevirtIterations.value_or(getMaxDevirtIterations(LibraryOptions));
184 R.EagerlyInvalidateAnalyses =
185 EagerlyInvalidateAnalyses.value_or(Opts.eagerly_invalidate_analyses);
186 R.DevirtualizeSpeculatively = DevirtualizeSpeculatively.value_or(
187 Opts.enable_devirtualize_speculatively);
188 return R;
189}
190
192 OptimizationLevel Level) {
193 for (auto &C : PeepholeEPCallbacks)
194 C(FPM, Level);
195}
198 for (auto &C : LateLoopOptimizationsEPCallbacks)
199 C(LPM, Level);
200}
202 OptimizationLevel Level) {
203 for (auto &C : LoopOptimizerEndEPCallbacks)
204 C(LPM, Level);
205}
208 for (auto &C : ScalarOptimizerLateEPCallbacks)
209 C(FPM, Level);
210}
212 OptimizationLevel Level) {
213 for (auto &C : CGSCCOptimizerLateEPCallbacks)
214 C(CGPM, Level);
215}
217 OptimizationLevel Level) {
218 for (auto &C : VectorizerStartEPCallbacks)
219 C(FPM, Level);
220}
222 OptimizationLevel Level) {
223 for (auto &C : VectorizerEndEPCallbacks)
224 C(FPM, Level);
225}
227 OptimizationLevel Level,
229 for (auto &C : OptimizerEarlyEPCallbacks)
230 C(MPM, Level, Phase);
231}
233 OptimizationLevel Level,
235 for (auto &C : OptimizerLastEPCallbacks)
236 C(MPM, Level, Phase);
237}
240 for (auto &C : FullLinkTimeOptimizationEarlyEPCallbacks)
241 C(MPM, Level);
242}
245 for (auto &C : FullLinkTimeOptimizationLastEPCallbacks)
246 C(MPM, Level);
247}
250 for (auto &C : ThinLinkTimeOptimizationEarlyEPCallbacks)
251 C(MPM, Level);
252}
255 for (auto &C : ThinLinkTimeOptimizationLastEPCallbacks)
256 C(MPM, Level);
257}
259 OptimizationLevel Level) {
260 for (auto &C : PipelineStartEPCallbacks)
261 C(MPM, Level);
262}
265 for (auto &C : PipelineEarlySimplificationEPCallbacks)
266 C(MPM, Level, Phase);
267}
268
269// Get IR stats with InstCount before/after the optimization pipeline
271 bool IsPreOptimization) {
272 if (AreStatisticsEnabled()) {
273 MPM.addPass(
276 FunctionPropertiesStatisticsPass(IsPreOptimization)));
277 }
278}
279
280// Add passes to run at the start of the pipeline. This includes things like
281// statistics passes.
283 const PassesOptions &Opts) {
284 instructionCountersPass(MPM, /*IsPreOptimization=*/true);
287}
288
289// Add passes to run at the end of the pipeline. This includes things like
290// statistics passes.
292 const PassesOptions &Opts) {
293 instructionCountersPass(MPM, /*IsPreOptimization=*/false);
296}
297
298// Helper to add AnnotationRemarksPass.
302
303// Helper to check if the current compilation phase is preparing for LTO
308
309// Helper to check if the current compilation phase is preparing for FullLTO
310[[maybe_unused]] static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase) {
312}
313
314// Helper to check if the current compilation phase is preparing for ThinLTO
318
319// Helper to check if the current compilation phase is LTO backend
324
325// Helper to check if the current compilation phase is FullLTO backend
329
330// Helper to check if the current compilation phase is ThinLTO backend
334
335// Helper to wrap conditionally Coro passes.
337 // TODO: Skip passes according to Phase.
338 ModulePassManager CoroPM;
339 CoroPM.addPass(CoroEarlyPass());
340 CGSCCPassManager CGPM;
341 CGPM.addPass(CoroSplitPass());
342 CoroPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
343 CoroPM.addPass(CoroCleanupPass());
344 CoroPM.addPass(GlobalDCEPass());
345 return CoroConditionalWrapper(std::move(CoroPM));
346}
347
349 OptimizationLevel Level) {
350 return getInlineParamsFromOptLevel(Context, static_cast<unsigned>(Level));
351}
352
354 const LLVMContext &Context,
355 OptimizationLevel Level,
357 const PassesOptions &Opts = getPassesOptions(Context);
358 InlineParams IP = ::getInlineParamsFromOptLevel(Context, Level);
359 if (Opts.enable_module_inliner)
360 MPM.addPass(ModuleInlinerPass(IP, Opts.enable_ml_inliner, Phase));
361 else
363 IP,
364 /* MandatoryFirst */ true,
366}
367
368// TODO: Investigate the cost/benefit of tail call elimination on debugging.
370PassBuilder::buildO1FunctionSimplificationPipeline(OptimizationLevel Level,
372
374
376 FPM.addPass(CountVisitsPass());
377
378 // Form SSA out of local memory accesses after breaking apart aggregates into
379 // scalars.
380 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
381
382 // Catch trivial redundancies
383 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
384
385 // Hoisting of scalars and load expressions.
386 FPM.addPass(
387 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
388 FPM.addPass(InstCombinePass());
389
390 FPM.addPass(LibCallsShrinkWrapPass());
391
392 invokePeepholeEPCallbacks(FPM, Level);
393
394 FPM.addPass(
395 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
396
397 // Form canonically associated expression trees, and simplify the trees using
398 // basic mathematical properties. For example, this will form (nearly)
399 // minimal multiplication trees.
400 FPM.addPass(ReassociatePass());
401
402 // Add the primary loop simplification pipeline.
403 // FIXME: Currently this is split into two loop pass pipelines because we run
404 // some function passes in between them. These can and should be removed
405 // and/or replaced by scheduling the loop pass equivalents in the correct
406 // positions. But those equivalent passes aren't powerful enough yet.
407 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
408 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
409 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
410 // `LoopInstSimplify`.
411 LoopPassManager LPM1, LPM2;
412
413 // Simplify the loop body. We do this initially to clean up after other loop
414 // passes run, either when iterating on a loop or on inner loops with
415 // implications on the outer loop.
416 LPM1.addPass(LoopInstSimplifyPass());
417 LPM1.addPass(LoopSimplifyCFGPass());
418
419 // Try to remove as much code from the loop header as possible,
420 // to reduce amount of IR that will have to be duplicated. However,
421 // do not perform speculative hoisting the first time as LICM
422 // will destroy metadata that may not need to be destroyed if run
423 // after loop rotation.
424 // TODO: Investigate promotion cap for O1.
425 LPM1.addPass(LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
426 /*AllowSpeculation=*/false));
427
428 LPM1.addPass(
429 LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
430 // TODO: Investigate promotion cap for O1.
431 LPM1.addPass(LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
432 /*AllowSpeculation=*/true));
433 LPM1.addPass(SimpleLoopUnswitchPass());
434 if (Opts.enable_loop_flatten)
435 LPM1.addPass(LoopFlattenPass());
436
437 LPM2.addPass(LoopIdiomRecognizePass());
438 LPM2.addPass(IndVarSimplifyPass());
439
441
442 LPM2.addPass(LoopDeletionPass());
443
444 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
445 // because it changes IR to makes profile annotation in back compile
446 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
447 // attributes so we need to make sure and allow the full unroll pass to pay
448 // attention to it.
449 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
450 PGOOpt->Action != PGOOptions::SampleUse)
451 LPM2.addPass(LoopFullUnrollPass(static_cast<int>(Level),
452 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
453 *PTO.ForgetAllSCEVInLoopUnroll));
454
456
457 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
458 /*UseMemorySSA=*/true));
459 FPM.addPass(
460 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
461 FPM.addPass(InstCombinePass());
462 // The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
463 // *All* loop passes must preserve it, in order to be able to use it.
464 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
465 /*UseMemorySSA=*/false));
466
467 // Delete small array after loop unroll.
468 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
469
470 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
471 FPM.addPass(MemCpyOptPass());
472
473 // Sparse conditional constant propagation.
474 // FIXME: It isn't clear why we do this *after* loop passes rather than
475 // before...
476 FPM.addPass(SCCPPass());
477
478 // Delete dead bit computations (instcombine runs after to fold away the dead
479 // computations, and then ADCE will run later to exploit any new DCE
480 // opportunities that creates).
481 FPM.addPass(BDCEPass());
482
483 // Run instcombine after redundancy and dead bit elimination to exploit
484 // opportunities opened up by them.
485 FPM.addPass(InstCombinePass());
486 invokePeepholeEPCallbacks(FPM, Level);
487
488 FPM.addPass(CoroElidePass());
489
491
492 // Finally, do an expensive DCE pass to catch all the dead code exposed by
493 // the simplifications and basic cleanup after all the simplifications.
494 // TODO: Investigate if this is too expensive.
495 FPM.addPass(ADCEPass());
496 FPM.addPass(
497 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
498 FPM.addPass(InstCombinePass());
499 invokePeepholeEPCallbacks(FPM, Level);
500
501 return FPM;
502}
503
507 assert(Level != OptimizationLevel::O0 && "Must request optimizations!");
508
509 // The O1 pipeline has a separate pipeline creation function to simplify
510 // construction readability.
511 if (Level == OptimizationLevel::O1)
512 return buildO1FunctionSimplificationPipeline(Level, Phase);
513
515
518
519 // Form SSA out of local memory accesses after breaking apart aggregates into
520 // scalars.
522
523 // Catch trivial redundancies
524 FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
525
526 // Hoisting of scalars and load expressions.
527 if (Opts.enable_gvn_hoist)
528 FPM.addPass(GVNHoistPass());
529
530 // Global value numbering based sinking.
531 if (Opts.enable_gvn_sink) {
532 FPM.addPass(GVNSinkPass());
533 FPM.addPass(
534 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
535 }
536
537 // Speculative execution if the target has divergent branches; otherwise nop.
538 FPM.addPass(SpeculativeExecutionPass(/* OnlyIfDivergentTarget =*/true));
539
540 // Optimize based on known information about branches, and cleanup afterward.
543
544 // Jump table to switch conversion.
545 if (Opts.enable_jump_table_to_switch)
547
548 FPM.addPass(
549 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
553
554 invokePeepholeEPCallbacks(FPM, Level);
555
556 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
557 // using the size value profile. Don't perform this when optimizing for size.
558 if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse)
560
561 FPM.addPass(TailCallElimPass(/*UpdateFunctionEntryCount=*/
562 isInstrumentedPGOUse()));
563 FPM.addPass(
564 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
565
566 // Form canonically associated expression trees, and simplify the trees using
567 // basic mathematical properties. For example, this will form (nearly)
568 // minimal multiplication trees.
570
571 if (Opts.enable_constraint_elimination)
573
574 // Add the primary loop simplification pipeline.
575 // FIXME: Currently this is split into two loop pass pipelines because we run
576 // some function passes in between them. These can and should be removed
577 // and/or replaced by scheduling the loop pass equivalents in the correct
578 // positions. But those equivalent passes aren't powerful enough yet.
579 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
580 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
581 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
582 // `LoopInstSimplify`.
583 LoopPassManager LPM1, LPM2;
584
585 // Simplify the loop body. We do this initially to clean up after other loop
586 // passes run, either when iterating on a loop or on inner loops with
587 // implications on the outer loop.
588 LPM1.addPass(LoopInstSimplifyPass());
589 LPM1.addPass(LoopSimplifyCFGPass());
590
591 // Try to remove as much code from the loop header as possible,
592 // to reduce amount of IR that will have to be duplicated. However,
593 // do not perform speculative hoisting the first time as LICM
594 // will destroy metadata that may not need to be destroyed if run
595 // after loop rotation.
596 // TODO: Investigate promotion cap for O1.
597 LPM1.addPass(LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
598 /*AllowSpeculation=*/false));
599
600 LPM1.addPass(
601 LoopRotatePass(/*EnableHeaderDuplication=*/true, isLTOPreLink(Phase)));
602 // TODO: Investigate promotion cap for O1.
603 LPM1.addPass(LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
604 /*AllowSpeculation=*/true));
605 LPM1.addPass(
606 SimpleLoopUnswitchPass(/* NonTrivial */ Level == OptimizationLevel::O3));
607 if (Opts.enable_loop_flatten)
608 LPM1.addPass(LoopFlattenPass());
609
610 LPM2.addPass(LoopIdiomRecognizePass());
611 LPM2.addPass(IndVarSimplifyPass());
612
613 {
615 ExtraPasses.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
617 LPM2.addPass(std::move(ExtraPasses));
618 }
619
621
622 LPM2.addPass(LoopDeletionPass());
623
624 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
625 // because it changes IR to makes profile annotation in back compile
626 // inaccurate. The normal unroller doesn't pay attention to forced full unroll
627 // attributes so we need to make sure and allow the full unroll pass to pay
628 // attention to it.
629 if (!isThinLTOPreLink(Phase) || !PGOOpt ||
630 PGOOpt->Action != PGOOptions::SampleUse)
631 LPM2.addPass(LoopFullUnrollPass(static_cast<int>(Level),
632 /* OnlyWhenForced= */ !PTO.LoopUnrolling,
633 *PTO.ForgetAllSCEVInLoopUnroll));
634
636
637 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1),
638 /*UseMemorySSA=*/true));
639 FPM.addPass(
640 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
642 // The loop passes in LPM2 (LoopIdiomRecognizePass, IndVarSimplifyPass,
643 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
644 // *All* loop passes must preserve it, in order to be able to use it.
645 FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2),
646 /*UseMemorySSA=*/false));
647
648 // Delete small array after loop unroll.
650
651 // Try vectorization/scalarization transforms that are both improvements
652 // themselves and can allow further folds with GVN and InstCombine.
653 FPM.addPass(VectorCombinePass(/*TryEarlyFoldsOnly=*/true));
654
655 // Eliminate redundancies.
657 if (Opts.enable_newgvn)
658 FPM.addPass(NewGVNPass());
659 else
660 FPM.addPass(GVNPass());
661
662 // Sparse conditional constant propagation.
663 // FIXME: It isn't clear why we do this *after* loop passes rather than
664 // before...
665 FPM.addPass(SCCPPass());
666
667 // Delete dead bit computations (instcombine runs after to fold away the dead
668 // computations, and then ADCE will run later to exploit any new DCE
669 // opportunities that creates).
670 FPM.addPass(BDCEPass());
671
672 // Run instcombine after redundancy and dead bit elimination to exploit
673 // opportunities opened up by them.
675 invokePeepholeEPCallbacks(FPM, Level);
676
677 // Re-consider control flow based optimizations after redundancy elimination,
678 // redo DCE, etc.
679 if (Opts.enable_dfa_jump_thread)
681
684
685 // Finally, do an expensive DCE pass to catch all the dead code exposed by
686 // the simplifications and basic cleanup after all the simplifications.
687 // TODO: Investigate if this is too expensive.
688 FPM.addPass(ADCEPass());
689
690 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
691 FPM.addPass(MemCpyOptPass());
692
693 FPM.addPass(DSEPass());
695
697 LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
698 /*AllowSpeculation=*/true),
699 /*UseMemorySSA=*/true));
700
701 FPM.addPass(CoroElidePass());
702
704
706 .convertSwitchRangeToICmp(true)
707 .convertSwitchToArithmetic(true)
708 .hoistCommonInsts(true)
709 .sinkCommonInsts(true)));
711 invokePeepholeEPCallbacks(FPM, Level);
712
713 return FPM;
714}
715
716void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) {
719 MPM.addPass(AssignGUIDPass());
720}
721
722void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM,
723 OptimizationLevel Level,
724 ThinOrFullLTOPhase LTOPhase,
725 bool IgnoreProfile) {
726 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
727 if (Opts.disable_preinline)
728 return;
729 InlineParams IP;
730
731 IP.DefaultThreshold = Opts.preinline_threshold;
732 IP.IgnoreProfile = IgnoreProfile;
733
734 // FIXME: The hint threshold has the same value used by the regular inliner
735 // when not optimzing for size. This should probably be lowered after
736 // performance testing.
737 // FIXME: this comment is cargo culted from the old pass manager, revisit).
738 IP.HintThreshold = 325;
739 IP.OptSizeHintThreshold = Opts.preinline_threshold;
741 IP, /* MandatoryFirst */ true,
743 CGSCCPassManager &CGPipeline = MIWP.getPM();
744
746 FPM.addPass(SROAPass(SROAOptions::ModifyCFG));
747 FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
748 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(
749 true))); // Merge & remove basic blocks.
750 FPM.addPass(InstCombinePass()); // Combine silly sequences.
751 invokePeepholeEPCallbacks(FPM, Level);
752
753 CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
754 std::move(FPM), *PTO.EagerlyInvalidateAnalyses));
755
756 MPM.addPass(std::move(MIWP));
757
758 // Delete anything that is now dead to make sure that we don't instrument
759 // dead code. Instrumentation can end up keeping dead code around and
760 // dramatically increase code size.
761 MPM.addPass(GlobalDCEPass());
762}
763
764void PassBuilder::addPostPGOLoopRotation(ModulePassManager &MPM,
765 OptimizationLevel Level) {
766 if (Opts.enable_post_pgo_loop_rotation) {
767 // Disable header duplication in loop rotation at -Oz.
769 createFunctionToLoopPassAdaptor(LoopRotatePass(),
770 /*UseMemorySSA=*/false),
771 *PTO.EagerlyInvalidateAnalyses));
772 }
773}
774
775void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
776 OptimizationLevel Level, bool RunProfileGen,
777 bool IsCS, bool AtomicCounterUpdate,
778 std::string ProfileFile,
779 std::string ProfileRemappingFile) {
780 assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
781
782 if (!RunProfileGen) {
783 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
784 MPM.addPass(
785 PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
786 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
787 // RequireAnalysisPass for PSI before subsequent non-module passes.
788 MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
789 return;
790 }
791
792 // Perform PGO instrumentation.
793 MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
795
796 addPostPGOLoopRotation(MPM, Level);
797 // Add the profile lowering pass.
798 InstrProfOptions Options;
799 if (!ProfileFile.empty())
800 Options.InstrProfileOutput = ProfileFile;
801 // Do counter promotion at Level greater than O0.
802 Options.DoCounterPromotion = true;
803 Options.UseBFIInPromotion = IsCS;
804 if (Opts.enable_sampled_instrumentation) {
805 Options.Sampling = true;
806 // With sampling, there is little beneifit to enable counter promotion.
807 // But note that sampling does work with counter promotion.
808 Options.DoCounterPromotion = false;
809 }
810 Options.Atomic = AtomicCounterUpdate;
811 MPM.addPass(InstrProfilingLoweringPass(Options, IsCS));
812}
813
815 bool RunProfileGen, bool IsCS,
816 bool AtomicCounterUpdate,
817 std::string ProfileFile,
818 std::string ProfileRemappingFile) {
819 if (!RunProfileGen) {
820 assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
821 MPM.addPass(
822 PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
823 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
824 // RequireAnalysisPass for PSI before subsequent non-module passes.
826 return;
827 }
828
829 // Perform PGO instrumentation.
832 // Add the profile lowering pass.
834 if (!ProfileFile.empty())
835 Options.InstrProfileOutput = ProfileFile;
836 // Do not do counter promotion at O0.
837 Options.DoCounterPromotion = false;
838 Options.UseBFIInPromotion = IsCS;
839 Options.Atomic = AtomicCounterUpdate;
841}
842
846 InlineParams IP;
847 if (PTO.InlinerThreshold == -1)
848 IP = ::getInlineParamsFromOptLevel(Context, Level);
849 else
850 IP = getInlineParams(Context, PTO.InlinerThreshold);
851 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
852 // set hot-caller threshold to 0 to disable hot
853 // callsite inline (as much as possible [1]) because it makes
854 // profile annotation in the backend inaccurate.
855 //
856 // [1] Note the cost of a function could be below zero due to erased
857 // prologue / epilogue.
858 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
860
861 if (PGOOpt)
862 IP.EnableDeferral = Opts.enable_npm_pgo_inline_deferral;
863
864 ModuleInlinerWrapperPass MIWP(IP, Opts.mandatory_inlining_first,
865 InlineContext{Phase, InlinePass::CGSCCInliner},
866 Opts.enable_ml_inliner,
867 *PTO.MaxDevirtIterations);
868
869 // Require the GlobalsAA analysis for the module so we can query it within
870 // the CGSCC pipeline.
871 if (Opts.enable_global_analyses) {
873 // Invalidate AAManager so it can be recreated and pick up the newly
874 // available GlobalsAA.
875 MIWP.addModulePass(
877 }
878
879 // Require the ProfileSummaryAnalysis for the module so we can query it within
880 // the inliner pass.
882
883 // Now begin the main postorder CGSCC pipeline.
884 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
885 // manager and trying to emulate its precise behavior. Much of this doesn't
886 // make a lot of sense and we should revisit the core CGSCC structure.
887 CGSCCPassManager &MainCGPipeline = MIWP.getPM();
888
889 // Note: historically, the PruneEH pass was run first to deduce nounwind and
890 // generally clean up exception handling overhead. It isn't clear this is
891 // valuable as the inliner doesn't currently care whether it is inlining an
892 // invoke or a call.
893
894 if (Opts.attributor_enable & AttributorRunOption::CGSCC)
895 MainCGPipeline.addPass(AttributorCGSCCPass());
896
897 // Deduce function attributes. We do another run of this after the function
898 // simplification pipeline, so this only needs to run when it could affect the
899 // function simplification pipeline, which is only the case with recursive
900 // functions.
901 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT) {
902 MainCGPipeline.addPass(AttributorLightCGSCCPass());
903 } else {
904 MainCGPipeline.addPass(
905 PostOrderFunctionAttrsPass(/*SkipNonRecursive*/ true));
906 }
907
908 // When at O3 add argument promotion to the pass pipeline.
909 // FIXME: It isn't at all clear why this should be limited to O3.
910 if (Level == OptimizationLevel::O3)
911 MainCGPipeline.addPass(ArgumentPromotionPass());
912
913 // Try to perform OpenMP specific optimizations. This is a (quick!) no-op if
914 // there are no OpenMP runtime calls present in the module.
915 if (Level == OptimizationLevel::O2 || Level == OptimizationLevel::O3)
916 MainCGPipeline.addPass(OpenMPOptCGSCCPass(Phase));
917
918 invokeCGSCCOptimizerLateEPCallbacks(MainCGPipeline, Level);
919
920 // Add the core function simplification pipeline nested inside the
921 // CGSCC walk.
924 *PTO.EagerlyInvalidateAnalyses, /*NoRerun=*/true));
925
926 // Finally, deduce any function attributes based on the fully simplified
927 // function.
928 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT)
929 MainCGPipeline.addPass(AttributorLightCGSCCPass());
930 else
931 MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
932
933 // Mark that the function is fully simplified and that it shouldn't be
934 // simplified again if we somehow revisit it due to CGSCC mutations unless
935 // it's been modified since.
938
939 if (!isThinLTOPreLink(Phase)) {
940 MainCGPipeline.addPass(CoroSplitPass(Level != OptimizationLevel::O0));
941 MainCGPipeline.addPass(CoroAnnotationElidePass());
942 }
943
944 // Make sure we don't affect potential future NoRerun CGSCC adaptors.
947
948 return MIWP;
949}
950
955
956 InlineParams IP = ::getInlineParamsFromOptLevel(Context, Level);
957 // For PreLinkThinLTO + SamplePGO or PreLinkFullLTO + SamplePGO,
958 // set hot-caller threshold to 0 to disable hot
959 // callsite inline (as much as possible [1]) because it makes
960 // profile annotation in the backend inaccurate.
961 //
962 // [1] Note the cost of a function could be below zero due to erased
963 // prologue / epilogue.
964 if (isLTOPreLink(Phase) && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
966
967 if (PGOOpt)
968 IP.EnableDeferral = Opts.enable_npm_pgo_inline_deferral;
969
970 // The inline deferral logic is used to avoid losing some
971 // inlining chance in future. It is helpful in SCC inliner, in which
972 // inlining is processed in bottom-up order.
973 // While in module inliner, the inlining order is a priority-based order
974 // by default. The inline deferral is unnecessary there. So we disable the
975 // inline deferral logic in module inliner.
976 IP.EnableDeferral = false;
977
978 MPM.addPass(ModuleInlinerPass(IP, Opts.enable_ml_inliner, Phase));
980 MPM.addPass(GlobalOptPass());
981 MPM.addPass(GlobalDCEPass());
982 MPM.addPass(AssignGUIDPass());
983 MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false));
984 }
985
988 *PTO.EagerlyInvalidateAnalyses));
989
990 if (!isThinLTOPreLink(Phase)) {
993 MPM.addPass(
995 }
996
997 return MPM;
998}
999
1002 const ModuleSummaryIndex *ImportSummary) {
1003 assert(Level != OptimizationLevel::O0 &&
1004 "Should not be used for O0 pipeline");
1005
1007 "FullLTOPostLink shouldn't call buildModuleSimplificationPipeline!");
1008
1009 assert((!ImportSummary || isThinLTOPostLink(Phase)) &&
1010 "ImportSummary is only meaningful in the ThinLTO post link phase");
1011
1013
1014 // Place pseudo probe instrumentation as the first pass of the pipeline to
1015 // minimize the impact of optimization changes.
1016 if (PGOOpt && PGOOpt->PseudoProbeForProfiling && !isThinLTOPostLink(Phase))
1018
1019 bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
1020
1021 // In ThinLTO mode, when flattened profile is used, all the available
1022 // profile information will be annotated in PreLink phase so there is
1023 // no need to load the profile again in PostLink.
1024 bool LoadSampleProfile = HasSampleProfile && !(Opts.flattened_profile_used &&
1026
1027 // During the ThinLTO backend phase we perform early indirect call promotion
1028 // here, before globalopt. Otherwise imported available_externally functions
1029 // look unreferenced and are removed. If we are going to load the sample
1030 // profile then defer until later.
1031 // TODO: See if we can move later and consolidate with the location where
1032 // we perform ICP when we are loading a sample profile.
1033 // TODO: We pass HasSampleProfile (whether there was a sample profile file
1034 // passed to the compile) to the SamplePGO flag of ICP. This is used to
1035 // determine whether the new direct calls are annotated with prof metadata.
1036 // Ideally this should be determined from whether the IR is annotated with
1037 // sample profile, and not whether the a sample profile was provided on the
1038 // command line. E.g. for flattened profiles where we will not be reloading
1039 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
1040 // provide the sample profile file.
1041 if (isThinLTOPostLink(Phase) && !LoadSampleProfile)
1042 MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile,
1043 ImportSummary));
1044
1045 // Create an early function pass manager to cleanup the output of the
1046 // frontend. Not necessary with LTO post link pipelines since the pre link
1047 // pipeline already cleaned up the frontend output.
1048 if (!isThinLTOPostLink(Phase)) {
1049 // Do basic inference of function attributes from known properties of system
1050 // libraries and other oracles.
1052 MPM.addPass(CoroEarlyPass());
1053
1054 FunctionPassManager EarlyFPM;
1055 EarlyFPM.addPass(EntryExitInstrumenterPass(/*PostInlining=*/false));
1056 // Lower llvm.expect to metadata before attempting transforms.
1057 // Compare/branch metadata may alter the behavior of passes like
1058 // SimplifyCFG.
1060 EarlyFPM.addPass(SimplifyCFGPass());
1062 EarlyFPM.addPass(EarlyCSEPass());
1063 if (Level == OptimizationLevel::O3)
1064 EarlyFPM.addPass(CallSiteSplittingPass());
1066 std::move(EarlyFPM), *PTO.EagerlyInvalidateAnalyses));
1067 }
1068
1069 if (LoadSampleProfile) {
1070 // Annotate sample profile right after early FPM to ensure freshness of
1071 // the debug info.
1073 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile, Phase, FS));
1074 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1075 // RequireAnalysisPass for PSI before subsequent non-module passes.
1077 // Do not invoke ICP in the LTOPrelink phase as it makes it hard
1078 // for the profile annotation to be accurate in the LTO backend.
1079 if (!isLTOPreLink(Phase))
1080 // We perform early indirect call promotion here, before globalopt.
1081 // This is important for the ThinLTO backend phase because otherwise
1082 // imported available_externally functions look unreferenced and are
1083 // removed.
1085 true /* IsInLTO */, true /* SamplePGO */, ImportSummary));
1086 }
1087
1088 // Try to perform OpenMP specific optimizations on the module. This is a
1089 // (quick!) no-op if there are no OpenMP runtime calls present in the module.
1091
1092 if (Opts.attributor_enable & AttributorRunOption::MODULE)
1093 MPM.addPass(AttributorPass());
1094 else if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
1096
1097 // Lower type metadata and the type.test intrinsic in the ThinLTO post link
1098 // pipeline after ICP. This is to enable usage of the type tests in ICP
1099 // sequences. It also lets ICP (including the ICP performed by the sample
1100 // profile loader) see the original names of the CFI functions it wants to
1101 // promote to, which LowerTypeTests renames to <name>.cfi.
1102 if (isThinLTOPostLink(Phase)) {
1103 if (ImportSummary)
1104 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1106 }
1107
1109
1110 // Interprocedural constant propagation now that basic cleanup has occurred
1111 // and prior to optimizing globals.
1112 // FIXME: This position in the pipeline hasn't been carefully considered in
1113 // years, it should be re-analyzed.
1114 MPM.addPass(
1115 IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/!isLTOPreLink(Phase))));
1116
1117 // Attach metadata to indirect call sites indicating the set of functions
1118 // they may target at run-time. This should follow IPSCCP.
1120
1121 // Optimize globals to try and fold them into constants.
1122 MPM.addPass(GlobalOptPass());
1123
1124 // Create a small function pass pipeline to cleanup after all the global
1125 // optimizations.
1126 FunctionPassManager GlobalCleanupPM;
1127 // FIXME: Should this instead by a run of SROA?
1128 GlobalCleanupPM.addPass(PromotePass());
1129 GlobalCleanupPM.addPass(InstCombinePass());
1130 invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
1131 GlobalCleanupPM.addPass(
1132 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
1134 std::move(GlobalCleanupPM), *PTO.EagerlyInvalidateAnalyses));
1135
1136 // We already asserted this happens in non-FullLTOPostLink earlier.
1137 const bool IsPreLink = !isThinLTOPostLink(Phase);
1138 // Enable contextual profiling instrumentation.
1139 const bool IsCtxProfGen =
1141 Context.getLibraryOptions());
1142 const bool IsPGOPreLink = !IsCtxProfGen && PGOOpt && IsPreLink;
1143 const bool IsPGOInstrGen =
1144 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRInstr;
1145 const bool IsPGOInstrUse =
1146 IsPGOPreLink && PGOOpt->Action == PGOOptions::IRUse;
1147 const bool IsMemprofUse = IsPGOPreLink && !PGOOpt->MemoryProfile.empty();
1148 // We don't want to mix pgo ctx gen and pgo gen; we also don't currently
1149 // enable ctx profiling from the frontend.
1151 Context.getLibraryOptions())) &&
1152 "Enabling both instrumented PGO and contextual instrumentation is not "
1153 "supported.");
1154 const bool IsCtxProfUse = !UseCtxProfile.empty() && isThinLTOPreLink(Phase);
1155
1156 assert((Opts.instrument_cold_function_only_path.empty() ||
1157 isPGOInstrumentColdFunctionOnly(Context.getLibraryOptions())) &&
1158 "--instrument-cold-function-only-path is provided but "
1159 "--pgo-instrument-cold-function-only is not enabled");
1160 const bool IsColdFuncOnlyInstrGen =
1161 isPGOInstrumentColdFunctionOnly(Context.getLibraryOptions()) &&
1162 IsPGOPreLink && !Opts.instrument_cold_function_only_path.empty();
1163
1164 // When a memory profile is used together with a sample profile, the
1165 // pre-inliner runs after the sample profile has been loaded. By default make
1166 // it ignore the sample profile, so that the inlining seen by MemProfUsePass
1167 // doesn't depend on the sample profile. Otherwise, a different sample profile
1168 // than the one used to build the memory-profiled binary can cause additional
1169 // inlining (e.g. of newly hot callees) that leads to memprof matching
1170 // failures. This matches the IR PGO pipeline, where the pre-inliner runs
1171 // before the profile is annotated.
1172 const bool PreInlinerIgnoreProfile =
1173 IsMemprofUse && HasSampleProfile && Opts.memprof_preinline_ignore_profile;
1174 if (IsPGOInstrGen || IsPGOInstrUse || IsMemprofUse || IsCtxProfGen ||
1175 IsCtxProfUse || IsColdFuncOnlyInstrGen)
1176 addPreInlinerPasses(MPM, Level, Phase, PreInlinerIgnoreProfile);
1177
1178 // Add all the requested passes for instrumentation PGO, if requested.
1179 if (IsPGOInstrGen || IsPGOInstrUse) {
1180 addPGOInstrPasses(MPM, Level,
1181 /*RunProfileGen=*/IsPGOInstrGen,
1182 /*IsCS=*/false, PGOOpt->AtomicCounterUpdate,
1183 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1184 } else if (IsCtxProfGen || IsCtxProfUse) {
1186 // In pre-link, we just want the instrumented IR. We use the contextual
1187 // profile in the post-thinlink phase.
1188 // The instrumentation will be removed in post-thinlink after IPO.
1189 if (IsCtxProfUse) {
1190 MPM.addPass(AssignGUIDPass());
1191 MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true));
1192 return MPM;
1193 }
1194 // Block further inlining in the instrumented ctxprof case. This avoids
1195 // confusingly collecting profiles for the same GUID corresponding to
1196 // different variants of the function. We could do like PGO and identify
1197 // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for
1198 // thinlto to happen before performing any further optimizations, it's
1199 // unnecessary to collect profiles for non-prevailing copies.
1201 addPostPGOLoopRotation(MPM, Level);
1202 MPM.addPass(AssignGUIDPass());
1204 } else if (IsColdFuncOnlyInstrGen) {
1205 addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false,
1206 /* AtomicCounterUpdate */ false,
1207 Opts.instrument_cold_function_only_path.str(),
1208 /* ProfileRemappingFile */ "");
1209 }
1210
1211 if (IsPGOInstrGen || IsPGOInstrUse || IsCtxProfGen)
1212 MPM.addPass(PGOIndirectCallPromotion(false, false));
1213
1214 if (IsPGOPreLink && PGOOpt->CSAction == PGOOptions::CSIRInstr)
1216 PGOOpt->CSProfileGenFile, Opts.enable_sampled_instrumentation));
1217
1218 if (IsMemprofUse)
1219 MPM.addPass(MemProfUsePass(PGOOpt->MemoryProfile, FS));
1220
1221 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRUse ||
1222 PGOOpt->Action == PGOOptions::SampleUse))
1223 MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType));
1224
1225 MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true));
1226
1227 if (Opts.enable_module_inliner)
1229 else
1230 MPM.addPass(buildInlinerPipeline(Level, Phase));
1231
1232 // Remove any dead arguments exposed by cleanups, constant folding globals,
1233 // and argument promotion.
1235
1238
1239 if (!isThinLTOPreLink(Phase))
1240 MPM.addPass(CoroCleanupPass());
1241
1242 // Optimize globals now that functions are fully simplified.
1243 MPM.addPass(GlobalOptPass());
1244 MPM.addPass(GlobalDCEPass());
1245
1246 return MPM;
1247}
1248
1249/// TODO: Should LTO cause any differences to this set of passes?
1250void PassBuilder::addVectorPasses(OptimizationLevel Level,
1252 ThinOrFullLTOPhase LTOPhase) {
1255
1256 // Drop dereferenceable assumes after vectorization, as they are no longer
1257 // needed and can inhibit further optimization.
1258 if (!isLTOPreLink(LTOPhase))
1259 FPM.addPass(DropUnnecessaryAssumesPass(/*DropDereferenceable=*/true));
1260
1262 if (isFullLTOPostLink(LTOPhase)) {
1263 // The vectorizer may have significantly shortened a loop body; unroll
1264 // again. Unroll small loops to hide loop backedge latency and saturate any
1265 // parallel execution resources of an out-of-order processor. We also then
1266 // need to clean up redundancies and loop invariant code.
1267 // FIXME: It would be really good to use a loop-integrated instruction
1268 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1269 // across the loop nests.
1270 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1271 if (Opts.enable_unroll_and_jam && PTO.LoopUnrolling)
1273 LoopUnrollAndJamPass(static_cast<int>(Level))));
1275 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1278 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1279 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1280 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1281 // NOTE: we are very late in the pipeline, and we don't have any LICM
1282 // or SimplifyCFG passes scheduled after us, that would cleanup
1283 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1284
1285 // We also turn on struct to vector canonicalization here, which allows
1286 // converting allocas of homogeneous structs into vector allocas when the
1287 // allocas' users are all memory intrinsics. This allows promotion in some
1288 // cases because structs cannot promote to SSA values, but vectors can. We
1289 // only turn this on after memcpyopt runs because this might hinder
1290 // memcpyopt's optimizations if done before. Look at the documentation for
1291 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1293 /*AggregateToVector=*/true)));
1294 }
1295
1296 if (!isFullLTOPostLink(LTOPhase)) {
1297 // Eliminate loads by forwarding stores from the previous iteration to loads
1298 // of the current iteration.
1300 }
1301 // Cleanup after the loop optimization passes.
1302 FPM.addPass(InstCombinePass());
1303
1304 if (Level > OptimizationLevel::O1 && Opts.extra_vectorizer_passes) {
1305 ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
1306 // At higher optimization levels, try to clean up any runtime overlap and
1307 // alignment checks inserted by the vectorizer. We want to track correlated
1308 // runtime checks for two inner loops in the same outer loop, fold any
1309 // common computations, hoist loop-invariant aspects out of any outer loop,
1310 // and unswitch the runtime checks if possible. Once hoisted, we may have
1311 // dead (or speculatable) control flows or more combining opportunities.
1312 ExtraPasses.addPass(EarlyCSEPass());
1313 ExtraPasses.addPass(CorrelatedValuePropagationPass());
1314 ExtraPasses.addPass(InstCombinePass());
1315 LoopPassManager LPM;
1316 LPM.addPass(LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
1317 /*AllowSpeculation=*/true));
1318 LPM.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
1320 ExtraPasses.addPass(
1321 createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/true));
1322 ExtraPasses.addPass(
1323 SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
1324 ExtraPasses.addPass(InstCombinePass());
1325 FPM.addPass(std::move(ExtraPasses));
1326 }
1327
1328 // Now that we've formed fast to execute loop structures, we do further
1329 // optimizations. These are run afterward as they might block doing complex
1330 // analyses and transforms such as what are needed for loop vectorization.
1331
1332 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
1333 // GVN, loop transforms, and others have already run, so it's now better to
1334 // convert to more optimized IR using more aggressive simplify CFG options.
1335 // The extra sinking transform can create larger basic blocks, so do this
1336 // before SLP vectorization.
1337 FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions()
1338 .forwardSwitchCondToPhi(true)
1339 .convertSwitchRangeToICmp(true)
1340 .convertSwitchToArithmetic(true)
1341 .convertSwitchToLookupTable(true)
1342 .needCanonicalLoops(false)
1343 .hoistCommonInsts(true)
1344 .sinkCommonInsts(true)));
1345
1346 if (isFullLTOPostLink(LTOPhase)) {
1347 FPM.addPass(SCCPPass());
1348 FPM.addPass(InstCombinePass());
1349 FPM.addPass(BDCEPass());
1350 }
1351
1352 // Optimize parallel scalar instruction chains into SIMD instructions.
1353 if (PTO.SLPVectorization) {
1354 FPM.addPass(SLPVectorizerPass());
1355 if (Level >= OptimizationLevel::O2 && Opts.extra_vectorizer_passes) {
1356 FPM.addPass(EarlyCSEPass());
1357 }
1358 }
1359 // Enhance/cleanup vector code.
1360 FPM.addPass(VectorCombinePass());
1361
1362 if (!isFullLTOPostLink(LTOPhase)) {
1363 FPM.addPass(InstCombinePass());
1364 // Unroll small loops to hide loop backedge latency and saturate any
1365 // parallel execution resources of an out-of-order processor. We also then
1366 // need to clean up redundancies and loop invariant code.
1367 // FIXME: It would be really good to use a loop-integrated instruction
1368 // combiner for cleanup here so that the unrolling and LICM can be pipelined
1369 // across the loop nests.
1370 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
1371 if (Opts.enable_unroll_and_jam && PTO.LoopUnrolling) {
1373 LoopUnrollAndJamPass(static_cast<int>(Level))));
1374 }
1375 FPM.addPass(LoopUnrollPass(LoopUnrollOptions(
1376 static_cast<int>(Level), /*OnlyWhenForced=*/!PTO.LoopUnrolling,
1377 *PTO.ForgetAllSCEVInLoopUnroll)));
1378 FPM.addPass(WarnMissedTransformationsPass());
1379 // Now that we are done with loop unrolling, be it either by LoopVectorizer,
1380 // or LoopUnroll passes, some variable-offset GEP's into alloca's could have
1381 // become constant-offset, thus enabling SROA and alloca promotion. Do so.
1382 // NOTE: we are very late in the pipeline, and we don't have any LICM
1383 // or SimplifyCFG passes scheduled after us, that would cleanup
1384 // the CFG mess this may created if allowed to modify CFG, so forbid that.
1385
1386 // We also turn on struct to vector canonicalization here, which allows
1387 // converting allocas of homogeneous structs into vector allocas when the
1388 // allocas' users are all memory intrinsics. This allows promotion in some
1389 // cases because structs cannot promote to SSA values, but vectors can. We
1390 // only turn this on after memcpyopt runs because this might hinder
1391 // memcpyopt's optimizations if done before. Look at the documentation for
1392 // `tryCanonicalizeStructToVector` in SROA.cpp to see why.
1393 FPM.addPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
1394 /*AggregateToVector=*/true)));
1395 }
1396
1397 FPM.addPass(InferAlignmentPass());
1398 FPM.addPass(InstCombinePass());
1399
1400 // This is needed for two reasons:
1401 // 1. It works around problems that instcombine introduces, such as sinking
1402 // expensive FP divides into loops containing multiplications using the
1403 // divide result.
1404 // 2. It helps to clean up some loop-invariant code created by the loop
1405 // unroll pass when IsFullLTO=false.
1407 LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
1408 /*AllowSpeculation=*/true),
1409 /*UseMemorySSA=*/true));
1410
1411 // Now that we've vectorized and unrolled loops, we may have more refined
1412 // alignment information, try to re-derive it here.
1413 FPM.addPass(AlignmentFromAssumptionsPass());
1414}
1415
1418 ThinOrFullLTOPhase LTOPhase) {
1420
1421 // Run partial inlining pass to partially inline functions that have
1422 // large bodies.
1423 if (Opts.enable_partial_inlining)
1425
1426 // Remove avail extern fns and globals definitions since we aren't compiling
1427 // an object file for later LTO. For LTO we want to preserve these so they
1428 // are eligible for inlining at link-time. Note if they are unreferenced they
1429 // will be removed by GlobalDCE later, so this only impacts referenced
1430 // available externally globals. Eventually they will be suppressed during
1431 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
1432 // may make globals referenced by available external functions dead and saves
1433 // running remaining passes on the eliminated functions. These should be
1434 // preserved during prelinking for link-time inlining decisions.
1435 if (!isLTOPreLink(LTOPhase))
1437
1438 // Do RPO function attribute inference across the module to forward-propagate
1439 // attributes where applicable.
1440 // FIXME: Is this really an optimization rather than a canonicalization?
1441 if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
1443 else
1445
1446 // Do a post inline PGO instrumentation and use pass. This is a context
1447 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
1448 // cross-module inline has not been done yet. The context sensitive
1449 // instrumentation is after all the inlines are done.
1450 if (!isLTOPreLink(LTOPhase) && PGOOpt) {
1451 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1452 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
1453 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,
1454 PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile);
1455 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1456 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
1457 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,
1458 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
1459 }
1460
1461 // Re-compute GlobalsAA here prior to function passes. This is particularly
1462 // useful as the above will have inlined, DCE'ed, and function-attr
1463 // propagated everything. We should at this point have a reasonably minimal
1464 // and richly annotated call graph. By computing aliasing and mod/ref
1465 // information for all local globals here, the late loop passes and notably
1466 // the vectorizer will be able to use them to help recognize vectorizable
1467 // memory operations.
1468 if (Opts.enable_global_analyses)
1470
1471 invokeOptimizerEarlyEPCallbacks(MPM, Level, LTOPhase);
1472
1473 FunctionPassManager OptimizePM;
1474
1475 // Only drop unnecessary assumes post-inline and post-link, as otherwise
1476 // additional uses of the affected value may be introduced through inlining
1477 // and CSE.
1478 if (!isLTOPreLink(LTOPhase))
1479 OptimizePM.addPass(DropUnnecessaryAssumesPass());
1480
1481 // Scheduling LoopVersioningLICM when inlining is over, because after that
1482 // we may see more accurate aliasing. Reason to run this late is that too
1483 // early versioning may prevent further inlining due to increase of code
1484 // size. Other optimizations which runs later might get benefit of no-alias
1485 // assumption in clone loop.
1486 if (Opts.enable_loop_versioning_licm) {
1487 OptimizePM.addPass(
1489 // LoopVersioningLICM pass might increase new LICM opportunities.
1491 LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
1492 /*AllowSpeculation=*/true),
1493 /*USeMemorySSA=*/true));
1494 }
1495
1496 OptimizePM.addPass(Float2IntPass());
1497 // Defer until LTO post-link where some constants may become known.
1498 if (!isLTOPreLink(LTOPhase))
1500
1501 if (Opts.enable_matrix) {
1502 OptimizePM.addPass(LowerMatrixIntrinsicsPass());
1503 OptimizePM.addPass(EarlyCSEPass());
1504 }
1505
1506 // CHR pass should only be applied with the profile information.
1507 // The check is to check the profile summary information in CHR.
1508 if (Opts.enable_chr && Level == OptimizationLevel::O3)
1509 OptimizePM.addPass(ControlHeightReductionPass());
1510
1511 // FIXME: We need to run some loop optimizations to re-rotate loops after
1512 // simplifycfg and others undo their rotation.
1513
1514 // Optimize the loop execution. These passes operate on entire loop nests
1515 // rather than on each loop in an inside-out manner, and so they are actually
1516 // function passes.
1517
1518 invokeVectorizerStartEPCallbacks(OptimizePM, Level);
1519
1520 LoopPassManager LPM;
1521 // First rotate loops that may have been un-rotated by prior passes.
1522 // Disable header duplication at -Oz.
1523 LPM.addPass(LoopRotatePass(/*EnableLoopHeaderDuplication=*/true,
1524 isLTOPreLink(LTOPhase),
1525 /*CheckExitCount=*/true));
1526 // Some loops may have become dead by now. Try to delete them.
1527 // FIXME: see discussion in https://reviews.llvm.org/D112851,
1528 // this may need to be revisited once we run GVN before loop deletion
1529 // in the simplification pipeline.
1530 LPM.addPass(LoopDeletionPass());
1531
1532 if (PTO.LoopInterchange)
1533 LPM.addPass(LoopInterchangePass());
1534
1535 OptimizePM.addPass(
1536 createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/false));
1537
1538 // FIXME: This may not be the right place in the pipeline.
1539 // We need to have the data to support the right place.
1540 if (PTO.LoopFusion)
1541 OptimizePM.addPass(LoopFusePass());
1542
1543 // Distribute loops to allow partial vectorization. I.e. isolate dependences
1544 // into separate loop that would otherwise inhibit vectorization. This is
1545 // currently only performed for loops marked with the metadata
1546 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
1547 OptimizePM.addPass(LoopDistributePass());
1548
1549 // Populates the VFABI attribute with the scalar-to-vector mappings
1550 // from the TargetLibraryInfo.
1551 OptimizePM.addPass(InjectTLIMappings());
1552
1553 addVectorPasses(Level, OptimizePM, LTOPhase);
1554
1555 invokeVectorizerEndEPCallbacks(OptimizePM, Level);
1556
1557 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
1558 // canonicalization pass that enables other optimizations. As a result,
1559 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
1560 // result too early.
1561 OptimizePM.addPass(LoopSinkPass());
1562
1563 // And finally clean up LCSSA form before generating code.
1564 OptimizePM.addPass(InstSimplifyPass());
1565
1566 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
1567 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
1568 // flattening of blocks.
1569 OptimizePM.addPass(DivRemPairsPass());
1570
1571 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
1572 // TODO: move this further up so that it can be optimized by GVN, etc.
1573 if (Opts.enable_mergeicmps)
1574 OptimizePM.addPass(MergeICmpsPass());
1575 OptimizePM.addPass(ExpandMemCmpPass());
1576
1577 // Try to annotate calls that were created during optimization.
1578 OptimizePM.addPass(
1579 TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
1580
1581 // LoopSink (and other loop passes since the last simplifyCFG) might have
1582 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
1583 OptimizePM.addPass(
1585 .convertSwitchRangeToICmp(true)
1586 .convertSwitchToArithmetic(true)
1587 .speculateUnpredictables(true)
1588 .hoistLoadsStoresWithCondFaulting(true)));
1589
1590 // Add the core optimizing pipeline.
1592 std::move(OptimizePM), *PTO.EagerlyInvalidateAnalyses));
1593
1594 // AllocToken transforms heap allocation calls; this needs to run late after
1595 // other allocation call transformations (such as those in InstCombine).
1596 if (!isLTOPreLink(LTOPhase))
1597 MPM.addPass(AllocTokenPass());
1598
1599 invokeOptimizerLastEPCallbacks(MPM, Level, LTOPhase);
1600
1601 // Run the Instrumentor pass late.
1602 if (Opts.enable_instrumentor)
1603 MPM.addPass(InstrumentorPass(FS));
1604
1605 // Split out cold code. Splitting is done late to avoid hiding context from
1606 // other optimizations and inadvertently regressing performance. The tradeoff
1607 // is that this has a higher code size cost than splitting early.
1608 if (Opts.hot_cold_split && !isLTOPreLink(LTOPhase))
1610
1611 // Now we need to do some global optimization transforms.
1612 // FIXME: It would seem like these should come first in the optimization
1613 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1614 // ordering here.
1615 MPM.addPass(GlobalDCEPass());
1617
1618 // Merge functions if requested. It has a better chance to merge functions
1619 // after ConstantMerge folded jump tables.
1620 if (*PTO.MergeFunctions)
1622
1623 if (PTO.CallGraphProfile && !isLTOPreLink(LTOPhase))
1624 MPM.addPass(CGProfilePass(isLTOPostLink(LTOPhase)));
1625
1626 // RelLookupTableConverterPass runs later in LTO post-link pipeline.
1627 if (!isLTOPreLink(LTOPhase))
1629
1630 // Add devirtualization pass only when LTO is not enabled, as otherwise
1631 // the pass is already enabled in the LTO pipeline.
1632 if (*PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) {
1633 // TODO: explore a better pipeline configuration that can improve
1634 // compilation time overhead.
1635 // FIXME: move this earlier (lots of pass ordering tests will need fixing)
1637 MPM.addPass(AssignGUIDPass());
1639 /*ExportSummary*/ nullptr,
1640 /*ImportSummary*/ nullptr,
1641 /*DevirtSpeculatively*/ *PTO.DevirtualizeSpeculatively));
1643 // Given that the devirtualization creates more opportunities for inlining,
1644 // we run the Inliner again here to maximize the optimization gain we
1645 // get from devirtualization.
1646 // Also, we can't run devirtualization before inlining because the
1647 // devirtualization depends on the passes optimizing/eliminating vtable GVs
1648 // and those passes are only effective after inlining.
1649 addModuleInlinerPass(MPM, Context, Level, ThinOrFullLTOPhase::None);
1650 }
1651
1652 // Attach !implicit.ref metadata from all functions to copyright strings.
1654
1655 return MPM;
1656}
1657
1661 if (Level == OptimizationLevel::O0)
1662 return buildO0DefaultPipeline(Level, Phase);
1663
1665 addPipelineStartPasses(MPM, Opts);
1666 // Currently this pipeline is only invoked in an LTO pre link pass or when we
1667 // are not running LTO. If that changes the below checks may need updating.
1669
1670 // If we are invoking this in non-LTO mode, remove any MemProf related
1671 // attributes and metadata, as we don't know whether we are linking with
1672 // a library containing the necessary interfaces.
1675
1676 // Convert @llvm.global.annotations to !annotation metadata.
1678
1679 // Force any function attributes we want the rest of the pipeline to observe.
1681
1682 if (Opts.opt_pipeline_trigger_crash)
1684
1685 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1687
1688 // Apply module pipeline start EP callback.
1690
1691 // Add the core simplification pipeline.
1693
1694 // Now add the optimization pipeline.
1696
1697 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1698 PGOOpt->Action == PGOOptions::SampleUse)
1700
1701 // Emit annotation remarks.
1703
1704 if (isLTOPreLink(Phase))
1705 addRequiredLTOPreLinkPasses(MPM);
1706
1707 addPipelineEndPasses(MPM, Opts);
1708 return MPM;
1709}
1710
1713 bool EmitSummary, bool Verify) {
1715
1716 addPipelineStartPasses(MPM, Opts);
1717
1718 if (ThinLTO)
1720 else
1722 // AssignGUIDPass attaches !guid metadata (MD_unique_id) to global objects,
1723 // triggering the bitcode writer to emit a METADATA_KIND_BLOCK. Standard LTO
1724 // bitcode emission runs VerifierPass by default, which registers metadata
1725 // kind IDs in LLVMContext. Running VerifierPass here before EmbedBitcodePass
1726 // to get the same behavior.
1727 if (Verify)
1728 MPM.addPass(VerifierPass());
1729 MPM.addPass(EmbedBitcodePass(ThinLTO, EmitSummary));
1730
1731 // Perform any cleanups to the IR that aren't suitable for per TU compilation,
1732 // like removing CFI/WPD related instructions. Note, we reuse
1733 // DropTypeTestsPass to clean up type tests rather than duplicate that logic
1734 // in FatLtoCleanup; checked-load cleanup is shared the other way, with
1735 // FatLtoCleanup calling lowertypetests::cleanUpTypeCheckedLoad.
1736 MPM.addPass(FatLtoCleanup());
1737
1738 // If we're doing FatLTO w/ CFI enabled, we don't want the type tests in the
1739 // object code, only in the bitcode section, so drop it before we run
1740 // module optimization and generate machine code. If llvm.type.test() isn't in
1741 // the IR, this won't do anything.
1743
1744 // Use the ThinLTO post-link pipeline with sample profiling
1745 if (ThinLTO && PGOOpt && PGOOpt->Action == PGOOptions::SampleUse)
1746 MPM.addPass(buildThinLTODefaultPipeline(Level, /*ImportSummary=*/nullptr));
1747 else {
1748 // ModuleSimplification does not run the coroutine passes for
1749 // ThinLTOPreLink, so we need the coroutine passes to run for ThinLTO
1750 // builds, otherwise they will miscompile.
1751 if (ThinLTO) {
1752 // TODO: replace w/ buildCoroWrapper() when it takes phase and level into
1753 // consideration.
1754 CGSCCPassManager CGPM;
1758 MPM.addPass(CoroCleanupPass());
1759 }
1760
1761 // otherwise, just use module optimization
1762 MPM.addPass(
1764 // Emit annotation remarks.
1766 }
1767
1768 addPipelineEndPasses(MPM, Opts);
1769
1770 return MPM;
1771}
1772
1775 if (Level == OptimizationLevel::O0)
1777
1779
1780 addPipelineStartPasses(MPM, Opts);
1781
1782 // Convert @llvm.global.annotations to !annotation metadata.
1784
1785 // Force any function attributes we want the rest of the pipeline to observe.
1787
1788 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
1790
1791 // Apply module pipeline start EP callback.
1793
1794 // If we are planning to perform ThinLTO later, we don't bloat the code with
1795 // unrolling/vectorization/... now. Just simplify the module as much as we
1796 // can.
1799 // In pre-link, for ctx prof use, we stop here with an instrumented IR. We let
1800 // thinlto use the contextual info to perform imports; then use the contextual
1801 // profile in the post-thinlink phase.
1802 if (!UseCtxProfile.empty()) {
1803 addRequiredLTOPreLinkPasses(MPM);
1804 return MPM;
1805 }
1806
1807 // Run partial inlining pass to partially inline functions that have
1808 // large bodies.
1809 // FIXME: It isn't clear whether this is really the right place to run this
1810 // in ThinLTO. Because there is another canonicalization and simplification
1811 // phase that will run after the thin link, running this here ends up with
1812 // less information than will be available later and it may grow functions in
1813 // ways that aren't beneficial.
1814 if (Opts.enable_partial_inlining)
1816
1817 if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
1818 PGOOpt->Action == PGOOptions::SampleUse)
1820
1821 // Handle Optimizer{Early,Last}EPCallbacks added by clang on PreLink. Actual
1822 // optimization is going to be done in PostLink stage, but clang can't add
1823 // callbacks there in case of in-process ThinLTO called by linker.
1828
1829 // Emit annotation remarks.
1831
1832 // Attach !implicit.ref metadata from all functions to copyright strings.
1834
1835 addRequiredLTOPreLinkPasses(MPM);
1836
1837 addPipelineEndPasses(MPM, Opts);
1838
1839 return MPM;
1840}
1841
1843 OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary) {
1845
1846 addPipelineStartPasses(MPM, Opts);
1847
1849
1850 // If we are invoking this without a summary index noting that we are linking
1851 // with a library containing the necessary APIs, remove any MemProf related
1852 // attributes and metadata.
1853 if (!ImportSummary || !ImportSummary->withSupportsHotColdNew())
1855
1856 if (ImportSummary) {
1857 // For ThinLTO we must apply the context disambiguation decisions early, to
1858 // ensure we can correctly match the callsites to summary data.
1861 ImportSummary, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1862
1863 // WholeProgramDevirt imports type identifier resolutions for whole-program
1864 // devirtualization. It must run early because later passes may disturb the
1865 // specific instruction patterns that it looks for, creating dependencies on
1866 // resolutions that may not appear in the summary.
1867 //
1868 // For example, GVN may transform the pattern assume(type.test) appearing in
1869 // two basic blocks into assume(phi(type.test, type.test)), which would
1870 // transform a dependency on a WPD resolution into a dependency on a type
1871 // identifier resolution for CFI.
1872 //
1873 // Also, WPD has access to more precise information than ICP and can
1874 // devirtualize more effectively, so it should operate on the IR first.
1875 //
1876 // It also needs to run at -O0 to lower type metadata and intrinsics.
1877 MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1878 }
1879
1880 // LowerTypeTests renames CFI functions to <name>.cfi, so with optimization
1881 // it runs in buildModuleSimplificationPipeline after ICP. Otherwise, lower
1882 // type tests here.
1883 if (Level == OptimizationLevel::O0) {
1884 if (ImportSummary)
1885 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1886 // Run a second time to clean up any type tests left behind by WPD for use
1887 // in ICP.
1890
1891 // AllocToken transforms heap allocation calls; this needs to run late after
1892 // other allocation call transformations (such as those in InstCombine).
1893 MPM.addPass(AllocTokenPass());
1894
1895 // Drop available_externally and unreferenced globals. This is necessary
1896 // with ThinLTO in order to avoid leaving undefined references to dead
1897 // globals in the object file.
1899 MPM.addPass(GlobalDCEPass());
1900
1902
1903 return MPM;
1904 }
1905 if (!UseCtxProfile.empty()) {
1906 // FIXME: Move LowerTypeTestsPass (and DropTypeTestsPass) into
1907 // buildModuleInlinerPipeline after ModuleInlinerPass, which performs ICP.
1908 if (ImportSummary)
1909 MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1910 MPM.addPass(
1912 } else {
1913 // Add the core simplification pipeline.
1915 Level, ThinOrFullLTOPhase::ThinLTOPostLink, ImportSummary));
1916 }
1917 // Now add the optimization pipeline.
1920
1922
1923 // Emit annotation remarks.
1925
1926 addPipelineEndPasses(MPM, Opts);
1927
1928 return MPM;
1929}
1930
1933 // FIXME: We should use a customized pre-link pipeline!
1934 return buildPerModuleDefaultPipeline(Level,
1936}
1937
1940 ModuleSummaryIndex *ExportSummary) {
1942
1943 addPipelineStartPasses(MPM, Opts);
1944
1946
1947 // If we are invoking this without a summary index noting that we are linking
1948 // with a library containing the necessary APIs, remove any MemProf related
1949 // attributes and metadata.
1950 if (!ExportSummary || !ExportSummary->withSupportsHotColdNew())
1952
1953 // Create a function that performs CFI checks for cross-DSO calls with targets
1954 // in the current module.
1955 MPM.addPass(CrossDSOCFIPass());
1956
1957 if (Level == OptimizationLevel::O0) {
1958 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1959 // metadata and intrinsics.
1960 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1961 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1962 // Run a second time to clean up any type tests left behind by WPD for use
1963 // in ICP.
1965
1967
1968 // AllocToken transforms heap allocation calls; this needs to run late after
1969 // other allocation call transformations (such as those in InstCombine).
1970 MPM.addPass(AllocTokenPass());
1971
1973
1974 // Emit annotation remarks.
1976
1977 return MPM;
1978 }
1979
1980 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1981 // Load sample profile before running the LTO optimization pipeline.
1982 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1983 PGOOpt->ProfileRemappingFile,
1985 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1986 // RequireAnalysisPass for PSI before subsequent non-module passes.
1988 }
1989
1990 // Try to run OpenMP optimizations, quick no-op if no OpenMP metadata present.
1992
1993 // Remove unused virtual tables to improve the quality of code generated by
1994 // whole-program devirtualization and bitset lowering.
1995 MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));
1996
1997 // Do basic inference of function attributes from known properties of system
1998 // libraries and other oracles.
2000
2001 if (Level >= OptimizationLevel::O2) {
2003 CallSiteSplittingPass(), *PTO.EagerlyInvalidateAnalyses));
2004
2005 // Indirect call promotion. This should promote all the targets that are
2006 // left by the earlier promotion pass that promotes intra-module targets.
2007 // This two-step promotion is to save the compile time. For LTO, it should
2008 // produce the same result as if we only do promotion here.
2010 true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2011
2012 // Promoting by-reference arguments to by-value exposes more constants to
2013 // IPSCCP.
2014 CGSCCPassManager CGPM;
2015 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT)
2017 else
2020 CGPM.addPass(
2023
2024 // Propagate constants at call sites into the functions they call. This
2025 // opens opportunities for globalopt (and inlining) by substituting function
2026 // pointers passed as arguments to direct uses of functions.
2027 MPM.addPass(IPSCCPPass(IPSCCPOptions(/*AllowFuncSpec=*/true)));
2028
2029 // Attach metadata to indirect call sites indicating the set of functions
2030 // they may target at run-time. This should follow IPSCCP.
2032 }
2033
2034 // Do RPO function attribute inference across the module to forward-propagate
2035 // attributes where applicable.
2036 // FIXME: Is this really an optimization rather than a canonicalization?
2037 if (Opts.attributor_enable & AttributorRunOption::MODULE_LIGHT)
2039 else
2041
2042 // Use in-range annotations on GEP indices to split globals where beneficial.
2043 MPM.addPass(GlobalSplitPass());
2044
2045 // Run whole program optimization of virtual call when the list of callees
2046 // is fixed.
2047 MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
2048
2050 // Stop here at -O1.
2051 if (Level == OptimizationLevel::O1) {
2053 LowerConstantIntrinsicsPass(), *PTO.EagerlyInvalidateAnalyses));
2054
2055 // The LowerTypeTestsPass needs to run to lower type metadata and the
2056 // type.test intrinsics. The pass does nothing if CFI is disabled.
2057 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
2058 // Run a second time to clean up any type tests left behind by WPD for use
2059 // in ICP (which is performed earlier than this in the regular LTO
2060 // pipeline).
2062
2064
2065 // AllocToken transforms heap allocation calls; this needs to run late after
2066 // other allocation call transformations (such as those in InstCombine).
2067 MPM.addPass(AllocTokenPass());
2068
2070
2071 // Emit annotation remarks.
2073
2074 addPipelineEndPasses(MPM, Opts);
2075
2076 return MPM;
2077 }
2078
2079 // TODO: Skip to match buildCoroWrapper.
2080 MPM.addPass(CoroEarlyPass());
2081
2082 // Optimize globals to try and fold them into constants.
2083 MPM.addPass(GlobalOptPass());
2084
2085 // Promote any localized globals to SSA registers.
2087
2088 // Linking modules together can lead to duplicate global constant, only
2089 // keep one copy of each constant.
2091
2092 // Remove unused arguments from functions.
2094
2095 // Reduce the code after globalopt and ipsccp. Both can open up significant
2096 // simplification opportunities, and both can propagate functions through
2097 // function pointers. When this happens, we often have to resolve varargs
2098 // calls, etc, so let instcombine do this.
2099 FunctionPassManager PeepholeFPM;
2100 PeepholeFPM.addPass(InstCombinePass());
2101 if (Level >= OptimizationLevel::O2)
2102 PeepholeFPM.addPass(AggressiveInstCombinePass());
2103 invokePeepholeEPCallbacks(PeepholeFPM, Level);
2104
2106 std::move(PeepholeFPM), *PTO.EagerlyInvalidateAnalyses));
2107
2108 // Lower variadic functions for supported targets prior to inlining.
2110
2111 // Note: historically, the PruneEH pass was run first to deduce nounwind and
2112 // generally clean up exception handling overhead. It isn't clear this is
2113 // valuable as the inliner doesn't currently care whether it is inlining an
2114 // invoke or a call.
2115 // Run the inliner now.
2116 addModuleInlinerPass(MPM, Context, Level,
2118
2119 // Perform context disambiguation after inlining, since that would reduce the
2120 // amount of additional cloning required to distinguish the allocation
2121 // contexts.
2124 /*Summary=*/nullptr,
2125 PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
2126
2127 // Optimize globals again after we ran the inliner.
2128 MPM.addPass(GlobalOptPass());
2129
2130 // Run the OpenMPOpt pass again after global optimizations.
2132
2133 // Garbage collect dead functions.
2134 MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));
2135
2136 // If we didn't decide to inline a function, check to see if we can
2137 // transform it to pass arguments by value instead of by reference.
2138 CGSCCPassManager CGPM;
2144
2146 // The IPO Passes may leave cruft around. Clean up after them.
2147 FPM.addPass(InstCombinePass());
2148 invokePeepholeEPCallbacks(FPM, Level);
2149
2150 if (Opts.enable_constraint_elimination)
2152
2154
2155 // Do a post inline PGO instrumentation and use pass. This is a context
2156 // sensitive PGO pass.
2157 if (PGOOpt) {
2158 if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
2159 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/true,
2160 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,
2161 PGOOpt->CSProfileGenFile, PGOOpt->ProfileRemappingFile);
2162 else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
2163 addPGOInstrPasses(MPM, Level, /*RunProfileGen=*/false,
2164 /*IsCS=*/true, PGOOpt->AtomicCounterUpdate,
2165 PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
2166 }
2167
2168 // Break up allocas
2170
2171 // LTO provides additional opportunities for tailcall elimination due to
2172 // link-time inlining, and visibility of nocapture attribute.
2173 FPM.addPass(
2174 TailCallElimPass(/*UpdateFunctionEntryCount=*/isInstrumentedPGOUse()));
2175
2176 // Run a few AA driver optimizations here and now to cleanup the code.
2178 std::move(FPM), *PTO.EagerlyInvalidateAnalyses));
2179
2180 if (Opts.attributor_enable & AttributorRunOption::CGSCC_LIGHT) {
2181 MPM.addPass(
2183 } else {
2184 MPM.addPass(
2186 }
2187
2188 // Require the GlobalsAA analysis for the module so we can query it within
2189 // MainFPM.
2190 if (Opts.enable_global_analyses) {
2192 // Invalidate AAManager so it can be recreated and pick up the newly
2193 // available GlobalsAA.
2194 MPM.addPass(
2196 }
2197
2198 FunctionPassManager MainFPM;
2200 LICMPass(*PTO.LicmMssaOptCap, *PTO.LicmMssaNoAccForPromotionCap,
2201 /*AllowSpeculation=*/true),
2202 /*USeMemorySSA=*/true));
2203
2204 if (Opts.enable_newgvn)
2205 MainFPM.addPass(NewGVNPass());
2206 else
2207 MainFPM.addPass(GVNPass());
2208
2209 // Remove dead memcpy()'s.
2210 MainFPM.addPass(MemCpyOptPass());
2211
2212 // Nuke dead stores.
2213 MainFPM.addPass(DSEPass());
2214 MainFPM.addPass(MoveAutoInitPass());
2216
2218
2219 invokeVectorizerStartEPCallbacks(MainFPM, Level);
2220
2221 LoopPassManager LPM;
2222 if (Opts.enable_loop_flatten && Level >= OptimizationLevel::O2)
2223 LPM.addPass(LoopFlattenPass());
2224 LPM.addPass(IndVarSimplifyPass());
2225 LPM.addPass(LoopDeletionPass());
2226 // FIXME: Add loop interchange.
2227
2228 // The loop passes in LPM (LoopFullUnrollPass) do not preserve MemorySSA.
2229 // *All* loop passes must preserve it, in order to be able to use it.
2230 // TODO: Explore enabling MemorySSA for loop passes after we remove the
2231 // LoopFullUnrollPass.
2232 MainFPM.addPass(
2233 createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA=*/false));
2234
2235 MainFPM.addPass(LoopDistributePass());
2236
2237 addVectorPasses(Level, MainFPM, ThinOrFullLTOPhase::FullLTOPostLink);
2238
2239 invokeVectorizerEndEPCallbacks(MainFPM, Level);
2240
2241 // Run the OpenMPOpt CGSCC pass again late.
2244
2245 invokePeepholeEPCallbacks(MainFPM, Level);
2246 MainFPM.addPass(JumpThreadingPass());
2248 std::move(MainFPM), *PTO.EagerlyInvalidateAnalyses));
2249
2250 // Lower type metadata and the type.test intrinsic. This pass supports
2251 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
2252 // to be run at link time if CFI is enabled. This pass does nothing if
2253 // CFI is disabled.
2254 MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
2255 // Run a second time to clean up any type tests left behind by WPD for use
2256 // in ICP (which is performed earlier than this in the regular LTO pipeline).
2258
2259 // Enable splitting late in the FullLTO post-link pipeline.
2260 if (Opts.hot_cold_split)
2262
2263 // Add late LTO optimization passes.
2264 FunctionPassManager LateFPM;
2265
2266 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
2267 // canonicalization pass that enables other optimizations. As a result,
2268 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
2269 // result too early.
2270 LateFPM.addPass(LoopSinkPass());
2271
2272 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
2273 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
2274 // flattening of blocks.
2275 LateFPM.addPass(DivRemPairsPass());
2276
2277 // Merge adjacent icmps into memcmp, then expand memcmp to loads/compares.
2278 // TODO: move this further up so that it can be optimized by GVN, etc.
2279 if (Opts.enable_mergeicmps)
2280 LateFPM.addPass(MergeICmpsPass());
2281 LateFPM.addPass(ExpandMemCmpPass());
2282
2283 // Delete basic blocks, which optimization passes may have killed.
2285 .convertSwitchRangeToICmp(true)
2286 .convertSwitchToArithmetic(true)
2287 .hoistCommonInsts(true)
2288 .speculateUnpredictables(true)));
2289 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(LateFPM)));
2290
2291 // Drop bodies of available eternally objects to improve GlobalDCE.
2293
2294 // Now that we have optimized the program, discard unreachable functions.
2295 MPM.addPass(GlobalDCEPass(/*InLTOPostLink=*/true));
2296
2297 if (*PTO.MergeFunctions)
2299
2301
2302 if (PTO.CallGraphProfile)
2303 MPM.addPass(CGProfilePass(/*InLTOPostLink=*/true));
2304
2305 MPM.addPass(CoroCleanupPass());
2306
2307 // AllocToken transforms heap allocation calls; this needs to run late after
2308 // other allocation call transformations (such as those in InstCombine).
2309 MPM.addPass(AllocTokenPass());
2310
2312
2313 // Emit annotation remarks.
2315
2316 addPipelineEndPasses(MPM, Opts);
2317
2318 return MPM;
2319}
2320
2324 assert(Level == OptimizationLevel::O0 &&
2325 "buildO0DefaultPipeline should only be used with O0");
2326
2328
2329 addPipelineStartPasses(MPM, Opts);
2330
2331 // Perform pseudo probe instrumentation in O0 mode. This is for the
2332 // consistency between different build modes. For example, a LTO build can be
2333 // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in
2334 // the postlink will require pseudo probe instrumentation in the prelink.
2335 if (PGOOpt && PGOOpt->PseudoProbeForProfiling)
2337
2338 if (PGOOpt && (PGOOpt->Action == PGOOptions::IRInstr ||
2339 PGOOpt->Action == PGOOptions::IRUse))
2341 MPM,
2342 /*RunProfileGen=*/(PGOOpt->Action == PGOOptions::IRInstr),
2343 /*IsCS=*/false, PGOOpt->AtomicCounterUpdate, PGOOpt->ProfileFile,
2344 PGOOpt->ProfileRemappingFile);
2345
2346 // Instrument function entry and exit before all inlining.
2348 EntryExitInstrumenterPass(/*PostInlining=*/false)));
2349
2351
2352 if (PGOOpt && PGOOpt->DebugInfoForProfiling)
2354
2355 if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
2356 // Explicitly disable sample loader inlining and use flattened profile in O0
2357 // pipeline.
2358 MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
2359 PGOOpt->ProfileRemappingFile,
2361 /*DisableSampleProfileInlining=*/true,
2362 /*UseFlattenedProfile=*/true));
2363 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
2364 // RequireAnalysisPass for PSI before subsequent non-module passes.
2366 }
2367
2369
2370 // Build a minimal pipeline based on the semantics required by LLVM,
2371 // which is just that always inlining occurs. Further, disable generating
2372 // lifetime intrinsics to avoid enabling further optimizations during
2373 // code generation.
2375 /*InsertLifetimeIntrinsics=*/false));
2376
2377 if (*PTO.MergeFunctions)
2379
2380 if (Opts.enable_matrix)
2381 MPM.addPass(
2383
2384 if (!CGSCCOptimizerLateEPCallbacks.empty()) {
2385 CGSCCPassManager CGPM;
2387 if (!CGPM.isEmpty())
2389 }
2390 if (!LateLoopOptimizationsEPCallbacks.empty()) {
2391 LoopPassManager LPM;
2393 if (!LPM.isEmpty()) {
2395 createFunctionToLoopPassAdaptor(std::move(LPM))));
2396 }
2397 }
2398 if (!LoopOptimizerEndEPCallbacks.empty()) {
2399 LoopPassManager LPM;
2401 if (!LPM.isEmpty()) {
2403 createFunctionToLoopPassAdaptor(std::move(LPM))));
2404 }
2405 }
2406 if (!ScalarOptimizerLateEPCallbacks.empty()) {
2409 if (!FPM.isEmpty())
2410 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2411 }
2412
2414
2415 if (!VectorizerStartEPCallbacks.empty()) {
2418 if (!FPM.isEmpty())
2419 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2420 }
2421
2422 if (!VectorizerEndEPCallbacks.empty()) {
2425 if (!FPM.isEmpty())
2426 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
2427 }
2428
2430
2431 // AllocToken transforms heap allocation calls; this needs to run late after
2432 // other allocation call transformations (such as those in InstCombine).
2433 if (!isLTOPreLink(Phase))
2434 MPM.addPass(AllocTokenPass());
2435
2437
2438 if (Opts.enable_instrumentor)
2439 MPM.addPass(InstrumentorPass(FS));
2440
2441 // Attach !implicit.ref metadata from all functions to copyright strings.
2443
2444 if (isLTOPreLink(Phase))
2445 addRequiredLTOPreLinkPasses(MPM);
2446
2447 // Emit annotation remarks.
2449
2450 addPipelineEndPasses(MPM, Opts);
2451
2452 return MPM;
2453}
2454
2456 AAManager AA;
2457
2458 // The order in which these are registered determines their priority when
2459 // being queried.
2460
2461 // Add any target-specific alias analyses that should be run early.
2462 if (TM)
2463 TM->registerEarlyDefaultAliasAnalyses(AA);
2464
2465 // First we register the basic alias analysis that provides the majority of
2466 // per-function local AA logic. This is a stateless, on-demand local set of
2467 // AA techniques.
2468 AA.registerFunctionAnalysis<BasicAA>();
2469
2470 // Next we query fast, specialized alias analyses that wrap IR-embedded
2471 // information about aliasing.
2472 AA.registerFunctionAnalysis<ScopedNoAliasAA>();
2473 AA.registerFunctionAnalysis<TypeBasedAA>();
2474
2475 // Add support for querying global aliasing information when available.
2476 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
2477 // analysis, all that the `AAManager` can do is query for any *cached*
2478 // results from `GlobalsAA` through a readonly proxy.
2479 if (Opts.enable_global_analyses)
2480 AA.registerModuleAnalysis<GlobalsAA>();
2481
2482 // Add target-specific alias analyses.
2483 if (TM)
2484 TM->registerDefaultAliasAnalyses(AA);
2485
2486 return AA;
2487}
2488
2489bool PassBuilder::isInstrumentedPGOUse() const {
2490 return (PGOOpt && PGOOpt->Action == PGOOptions::IRUse) ||
2491 !UseCtxProfile.empty();
2492}
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AggressiveInstCombiner - Combine expression patterns to form expressions with fewer,...
Provides passes to inlining "always_inline" functions.
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
This file provides the interface for LLVM's Call Graph Profile pass.
This header provides classes for managing passes over SCCs of the call graph.
This file provides the interface for a simple, fast CSE pass.
This file provides a pass which clones the current module and runs the provided pass pipeline on the ...
This file provides a pass manager that only runs its passes if the provided marker analysis has been ...
Super simple passes to force specific function attrs from the commandline into the IR for debugging p...
Provides passes for computing function attributes based on interprocedural analyses.
This file provides the interface for the GVNHoist pass.
This file provides the interface for the GVNSink pass.
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
This is the interface for a simple mod/ref and alias analysis over globals.
AcceleratorCodeSelection - Identify all functions reachable from a kernel, removing those that are un...
This header defines various interfaces for pass management in LLVM.
Interfaces for passes which infer implicit function attributes from the name and signature of functio...
This file provides the primary interface to the instcombine pass.
Defines passes for running instruction simplification across chunks of IR.
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
See the comments on JumpThreadingPass.
static LVOptions Options
Definition LVOptions.cpp:25
This file implements the Loop Fusion pass.
This header defines the LoopLoadEliminationPass object.
This header provides classes for managing a pipeline of passes over loops in LLVM IR.
The header file for the LowerConstantIntrinsics pass as used by the new pass manager.
The header file for the LowerExpectIntrinsic pass as used by the new pass manager.
This pass performs merges of loads and stores on both sides of a.
This file provides the interface for LLVM's Global Value Numbering pass.
This header enumerates the LLVM-provided high-level optimization levels.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
Define option tunables for PGO.
ppc ctr loops PowerPC CTR Loops Verify
static void addPipelineEndPasses(ModulePassManager &MPM, const PassesOptions &Opts)
static void addPipelineStartPasses(ModulePassManager &MPM, const PassesOptions &Opts)
static bool isThinLTOPostLink(ThinOrFullLTOPhase Phase)
static void addModuleInlinerPass(ModulePassManager &MPM, const LLVMContext &Context, OptimizationLevel Level, ThinOrFullLTOPhase Phase)
static void addAnnotationRemarksPass(ModulePassManager &MPM)
static CoroConditionalWrapper buildCoroWrapper(ThinOrFullLTOPhase Phase)
static bool isFullLTOPostLink(ThinOrFullLTOPhase Phase)
static bool isThinLTOPreLink(ThinOrFullLTOPhase Phase)
static bool isLTOPreLink(ThinOrFullLTOPhase Phase)
static void instructionCountersPass(ModulePassManager &MPM, bool IsPreOptimization)
static bool isFullLTOPreLink(ThinOrFullLTOPhase Phase)
static bool isLTOPostLink(ThinOrFullLTOPhase Phase)
This file implements relative lookup table converter that converts lookup tables to relative lookup t...
This file provides the interface for LLVM's Scalar Replacement of Aggregates pass.
This file provides the interface for the pseudo probe implementation for AutoFDO.
This file provides the interface for the sampled PGO loader pass.
This is the interface for a metadata-based scoped no-alias analysis.
This file provides the interface for the pass responsible for both simplifying and canonicalizing the...
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
This is the interface for a metadata-based TBAA.
A manager for alias analyses.
A module pass that rewrites heap allocations to use token-enabled allocation functions based on vario...
Definition AllocToken.h:36
Inlines functions marked as "always_inline".
Argument promotion pass.
Analysis pass providing a never-invalidated alias analysis result.
Simple pass that canonicalizes aliases.
A pass that merges duplicate global constants into a single constant.
Eliminate dead arguments (and return values) from functions.
A pass that transforms external global definitions into declarations.
Pass embeds a copy of the module optimized with the provided pass pipeline into a global variable.
A pass manager to run a set of extra loop passes if the MarkerTy analysis is present.
Statistics pass for the FunctionPropertiesAnalysis results.
Pass to remove unused function declarations.
Definition GlobalDCE.h:39
Optimize globals that never have their address taken.
Definition GlobalOpt.h:25
Pass to perform split of global variables.
Definition GlobalSplit.h:26
Analysis pass providing a never-invalidated alias analysis result.
Pass to outline cold regions.
Pass to perform interprocedural constant propagation.
Definition SCCP.h:48
Run instruction simplification across each instruction in the function.
Instrumentation based profiling lowering pass.
The Instrumentor pass.
This pass performs 'jump threading', which looks at blocks that have multiple predecessors and multip...
Performs Loop Invariant Code Motion Pass.
Definition LICM.h:71
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Loop unroll pass that only does full loop unrolling and peeling.
Performs Loop Idiom Recognize Pass.
Performs Loop Inst Simplify Pass.
A simple loop rotation transformation.
Performs basic CFG simplifications to assist other loop passes.
A pass that does profile-guided sinking of instructions into loops.
Definition LoopSink.h:33
A simple loop rotation transformation.
Loop unroll pass that will support both full and partial unrolling.
Strips MemProf attributes and metadata.
Merge identical functions.
The module inliner pass for the new pass manager.
Module pass, wrapping the inliner pass.
Definition Inliner.h:65
void addModulePass(T Pass)
Add a module pass that runs before the CGSCC passes.
Definition Inliner.h:80
CGSCCPassManager & getPM()
Allow adding more CGSCC passes, besides inlining.
Definition Inliner.h:77
void addLateModulePass(T Pass)
Add a module pass that runs after the CGSCC passes.
Definition Inliner.h:85
Class to hold module path string table and global value map, and encapsulate methods for operating on...
Simple pass that provides a name to every anonymous globals.
Additional 'norecurse' attribute deduction during postlink LTO phase.
OpenMP optimizations pass.
Definition OpenMPOpt.h:42
static LLVM_ABI bool isCtxIRPGOInstrEnabled(const cl::LibraryOptionsContext &LibraryOptions)
The indirect function call promotion pass.
The instrumentation (profile-instr-gen) pass for IR based PGO.
The instrumentation (profile-instr-gen) pass for IR based PGO.
The profile annotation (profile-instr-use) pass for IR based PGO.
The profile size based optimization pass for memory intrinsics.
Pass to remove unused function declarations.
LLVM_ABI void invokeFullLinkTimeOptimizationLastEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level)
LLVM_ABI ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the module pipeline that performs inlining as well as the inlining-driven cleanups.
LLVM_ABI void invokeOptimizerEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase)
LLVM_ABI ModulePassManager buildFatLTODefaultPipeline(OptimizationLevel Level, bool ThinLTO, bool EmitSummary, bool Verify=true)
Build a fat object default optimization pipeline.
LLVM_ABI void invokeVectorizerStartEPCallbacks(FunctionPassManager &FPM, OptimizationLevel Level)
LLVM_ABI AAManager buildDefaultAAPipeline()
Build the default AAManager with the default alias analysis pipeline registered.
LLVM_ABI void invokeCGSCCOptimizerLateEPCallbacks(CGSCCPassManager &CGPM, OptimizationLevel Level)
LLVM_ABI ModulePassManager buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, ThinLTO-targeting default optimization pipeline to a pass manager.
LLVM_ABI void addPGOInstrPassesForO0(ModulePassManager &MPM, bool RunProfileGen, bool IsCS, bool AtomicCounterUpdate, std::string ProfileFile, std::string ProfileRemappingFile)
Add PGOInstrumenation passes for O0 only.
LLVM_ABI void invokeScalarOptimizerLateEPCallbacks(FunctionPassManager &FPM, OptimizationLevel Level)
LLVM_ABI ModulePassManager buildModuleSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase, const ModuleSummaryIndex *ImportSummary=nullptr)
Construct the core LLVM module canonicalization and simplification pipeline.
LLVM_ABI ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase=ThinOrFullLTOPhase::None)
Build a per-module default optimization pipeline.
LLVM_ABI void invokePipelineStartEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level)
LLVM_ABI void invokeVectorizerEndEPCallbacks(FunctionPassManager &FPM, OptimizationLevel Level)
LLVM_ABI void invokeThinLinkTimeOptimizationLastEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level)
LLVM_ABI ModulePassManager buildO0DefaultPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase=ThinOrFullLTOPhase::None)
Build an O0 pipeline with the minimal semantically required passes.
LLVM_ABI FunctionPassManager buildFunctionSimplificationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the core LLVM function canonicalization and simplification pipeline.
LLVM_ABI void invokePeepholeEPCallbacks(FunctionPassManager &FPM, OptimizationLevel Level)
LLVM_ABI void invokePipelineEarlySimplificationEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase)
LLVM_ABI void invokeLoopOptimizerEndEPCallbacks(LoopPassManager &LPM, OptimizationLevel Level)
LLVM_ABI ModulePassManager buildLTODefaultPipeline(OptimizationLevel Level, ModuleSummaryIndex *ExportSummary)
Build an LTO default optimization pipeline to a pass manager.
LLVM_ABI ModulePassManager buildModuleInlinerPipeline(OptimizationLevel Level, ThinOrFullLTOPhase Phase)
Construct the module pipeline that performs inlining with module inliner pass.
LLVM_ABI ModulePassManager buildThinLTODefaultPipeline(OptimizationLevel Level, const ModuleSummaryIndex *ImportSummary)
Build a ThinLTO default optimization pipeline to a pass manager.
LLVM_ABI void invokeLateLoopOptimizationsEPCallbacks(LoopPassManager &LPM, OptimizationLevel Level)
LLVM_ABI void invokeFullLinkTimeOptimizationEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level)
LLVM_ABI ModulePassManager buildModuleOptimizationPipeline(OptimizationLevel Level, ThinOrFullLTOPhase LTOPhase)
Construct the core LLVM module optimization pipeline.
LLVM_ABI void invokeThinLinkTimeOptimizationEarlyEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level)
LLVM_ABI void invokeOptimizerLastEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level, ThinOrFullLTOPhase Phase)
LLVM_ABI ModulePassManager buildLTOPreLinkDefaultPipeline(OptimizationLevel Level)
Build a pre-link, LTO-targeting default optimization pipeline to a pass manager.
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same_v< PassT, PassManager > > addPass(PassT &&Pass)
bool isEmpty() const
Returns if the pass manager contains any passes.
Tunable parameters for passes in the default pipelines.
Definition PassBuilder.h:48
std::optional< bool > EagerlyInvalidateAnalyses
std::optional< bool > ForgetAllSCEVInLoopUnroll
Tuning option to forget all SCEV loops in LoopUnroll.
Definition PassBuilder.h:75
std::optional< unsigned > MaxDevirtIterations
Tuning option to cap how many times the inliner pipeline reruns on an SCC after devirtualizing a call...
Definition PassBuilder.h:99
std::optional< unsigned > LicmMssaNoAccForPromotionCap
Tuning option to disable promotion to scalars in LICM with MemorySSA, if the number of access is too ...
Definition PassBuilder.h:83
std::optional< bool > DevirtualizeSpeculatively
std::optional< unsigned > LicmMssaOptCap
Tuning option to cap the number of calls to retrive clobbering accesses in MemorySSA,...
Definition PassBuilder.h:79
LLVM_ABI PipelineTuningOptions resolve(const cl::LibraryOptionsContext &LibraryOptions) const
Return a copy whose unset members are set from LibraryOptions.
bool LoopInterleaving
Tuning option to set loop interleaving on/off, set based on opt level.
Definition PassBuilder.h:55
std::optional< bool > MergeFunctions
Tuning option to enable/disable function merging.
Definition PassBuilder.h:92
bool LoopUnrolling
Tuning option to enable/disable loop unrolling.
Definition PassBuilder.h:66
bool LoopVectorization
Tuning option to enable/disable loop vectorization, set based on opt level.
Definition PassBuilder.h:59
Inject MD_prof metadata where it's missing.
Checks that MD_prof is present on every instruction that supports it.
Reassociate commutative expressions.
Definition Reassociate.h:75
A pass to do RPO deduction and propagation of function attributes.
This pass performs function-level constant propagation and merging.
Definition SCCP.h:30
The sample profiler data loader pass.
Analysis pass providing a never-invalidated alias analysis result.
This pass transforms loops that contain branches or switches on loop- invariant conditions to have mu...
A pass to simplify and canonicalize the CFG of a function.
Definition SimplifyCFG.h:30
Analysis pass providing a never-invalidated alias analysis result.
Optimize scalar/vector interactions in IR using target cost models.
Create a verifier pass.
Definition Verifier.h:134
Interfaces for registering analysis passes, producing common pass manager configurations,...
Abstract Attribute helper functions.
Definition Attributor.h:166
@ All
Drop only llvm.assumes using type test value.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI InlineParams getInlineParams(const LLVMContext &C)
Generate the parameters to tune the inline cost analysis based only on the commandline options.
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
cl::opt< bool > EnableProfcheckInPipeline("enable-profcheck-in-pipeline", cl::init(false), cl::desc("Adds profile verification to the pipeline, erroring if profile " "information is dropped."))
LLVM_ABI unsigned getLicmMssaOptCap(const cl::LibraryOptionsContext &LibraryOptions)
Return -licm-mssa-optimization-cap.
Definition LICM.cpp:120
cl::opt< std::string > UseCtxProfile("use-ctx-profile", cl::init(""), cl::Hidden, cl::desc("Use the specified contextual profile file"))
@ O1
Optimize quickly without destroying debuggability.
@ O0
Disable as many optimizations as possible.
@ O3
Optimize for fast execution as much as possible.
@ O2
Optimize for fast execution as much as possible without triggering significant incremental compile ti...
PassManager< LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, CGSCCUpdateResult & > CGSCCPassManager
The CGSCC pass manager.
@ CGSCC_LIGHT
@ MODULE_LIGHT
ThinOrFullLTOPhase
This enumerates the LLVM full LTO or ThinLTO optimization phases.
Definition Pass.h:75
@ FullLTOPreLink
Full LTO prelink phase.
Definition Pass.h:83
@ ThinLTOPostLink
ThinLTO postlink (backend compile) phase.
Definition Pass.h:81
@ None
No LTO/ThinLTO behavior needed.
Definition Pass.h:77
@ FullLTOPostLink
Full LTO postlink (backend compile) phase.
Definition Pass.h:85
@ ThinLTOPreLink
ThinLTO prelink (summary) phase.
Definition Pass.h:79
PassManager< Loop, LoopAnalysisManager, LoopStandardAnalysisResults &, LPMUpdater & > LoopPassManager
The Loop pass manager.
ModuleToPostOrderCGSCCPassAdaptor createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT &&Pass)
A function to deduce a function pass type and wrap it in the templated adaptor.
LLVM_ABI InlineParams getInlineParamsFromOptLevel(const LLVMContext &C, unsigned OptLevel)
Generate the parameters to tune the inline cost analysis based on command line options.
FunctionToLoopPassAdaptor createFunctionToLoopPassAdaptor(LoopPassT &&Pass, bool UseMemorySSA=false)
A function to deduce a loop pass type and wrap it in the templated adaptor.
CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false, bool NoRerun=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
LLVM_ABI bool getForgetSCEVInLoopUnroll(const cl::LibraryOptionsContext &LibraryOptions)
Return -forget-scev-loop-unroll.
PassManager< Module > ModulePassManager
Convenience typedef for a pass manager over modules.
LLVM_ABI bool AreStatisticsEnabled()
Check if statistics are enabled.
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
PassManager< Function > FunctionPassManager
Convenience typedef for a pass manager over functions.
LLVM_ABI unsigned getLicmMssaNoAccForPromotionCap(const cl::LibraryOptionsContext &LibraryOptions)
Return -licm-mssa-max-acc-promotion.
Definition LICM.cpp:124
LLVM_ABI unsigned getMaxDevirtIterations(const cl::LibraryOptionsContext &)
const PassesOptions & getPassesOptions(const LLVMContext &C)
LLVM_ABI bool isPGOInstrumentColdFunctionOnly(const cl::LibraryOptionsContext &LibraryOptions)
Return the value of -pgo-instrument-cold-function-only in LibraryOptions.
A DCE pass that assumes instructions are dead until proven otherwise.
Definition ADCE.h:31
Pass to convert @llvm.global.annotations to !annotation metadata.
A more lightweight version of the Attributor which only runs attribute inference but no simplificatio...
A more lightweight version of the Attributor which only runs attribute inference but no simplificatio...
Hoist/decompose integer division and remainder instructions to enable CFG improvements and better cod...
Definition DivRemPairs.h:23
A simple and fast domtree-based CSE pass.
Definition EarlyCSE.h:31
Pass which forces specific function attributes into the IR, primarily as a debugging tool.
A simple and fast domtree-based GVN pass to hoist common expressions from sibling branches.
Definition GVNHoist.h:23
Uses an "inverted" value numbering to decide the similarity of expressions and sinks similar expressi...
Definition GVNSink.h:23
A set of parameters to control various transforms performed by IPSCCP pass.
Definition SCCP.h:35
A pass which infers function attributes from the names and signatures of function declarations in a m...
Provides context on when an inline advisor is constructed in the pipeline (e.g., link phase,...
Thresholds to tune inline cost analysis.
Definition InlineCost.h:210
std::optional< int > OptSizeHintThreshold
Threshold to use for callees with inline hint, when the caller is optimized for size.
Definition InlineCost.h:219
std::optional< int > HotCallSiteThreshold
Threshold to use when the callsite is considered hot.
Definition InlineCost.h:231
std::optional< bool > IgnoreProfile
Indicate whether to ignore profile information.
Definition InlineCost.h:252
int DefaultThreshold
The default threshold to start with for a callee.
Definition InlineCost.h:212
std::optional< bool > EnableDeferral
Indicate whether we should allow inline deferral.
Definition InlineCost.h:244
std::optional< int > HintThreshold
Threshold to use for callees with inline hint.
Definition InlineCost.h:215
Options for the frontend instrumentation based profiling pass.
A no-op pass template which simply forces a specific analysis result to be invalidated.
Pass to forward loads in a loop around the backedge to subsequent iterations.
A set of parameters used to control various transforms performed by the LoopUnroll pass.
The LoopVectorize Pass.
Computes function attributes in post-order over the call graph.
A utility pass template to force an analysis result to be available.