Line data Source code
1 : //===- Parsing, selection, and construction of pass pipelines -------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : /// \file
10 : ///
11 : /// This file provides the implementation of the PassBuilder based on our
12 : /// static pass registry as well as related functionality. It also provides
13 : /// helpers to aid in analyzing, debugging, and testing passes and pass
14 : /// pipelines.
15 : ///
16 : //===----------------------------------------------------------------------===//
17 :
18 : #include "llvm/Passes/PassBuilder.h"
19 : #include "llvm/ADT/StringSwitch.h"
20 : #include "llvm/Analysis/AliasAnalysis.h"
21 : #include "llvm/Analysis/AliasAnalysisEvaluator.h"
22 : #include "llvm/Analysis/AssumptionCache.h"
23 : #include "llvm/Analysis/BasicAliasAnalysis.h"
24 : #include "llvm/Analysis/BlockFrequencyInfo.h"
25 : #include "llvm/Analysis/BranchProbabilityInfo.h"
26 : #include "llvm/Analysis/CFGPrinter.h"
27 : #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
28 : #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
29 : #include "llvm/Analysis/CGSCCPassManager.h"
30 : #include "llvm/Analysis/CallGraph.h"
31 : #include "llvm/Analysis/DemandedBits.h"
32 : #include "llvm/Analysis/DependenceAnalysis.h"
33 : #include "llvm/Analysis/DominanceFrontier.h"
34 : #include "llvm/Analysis/GlobalsModRef.h"
35 : #include "llvm/Analysis/IVUsers.h"
36 : #include "llvm/Analysis/LazyCallGraph.h"
37 : #include "llvm/Analysis/LazyValueInfo.h"
38 : #include "llvm/Analysis/LoopAccessAnalysis.h"
39 : #include "llvm/Analysis/LoopInfo.h"
40 : #include "llvm/Analysis/MemoryDependenceAnalysis.h"
41 : #include "llvm/Analysis/MemorySSA.h"
42 : #include "llvm/Analysis/ModuleSummaryAnalysis.h"
43 : #include "llvm/Analysis/OptimizationRemarkEmitter.h"
44 : #include "llvm/Analysis/PhiValues.h"
45 : #include "llvm/Analysis/PostDominators.h"
46 : #include "llvm/Analysis/ProfileSummaryInfo.h"
47 : #include "llvm/Analysis/RegionInfo.h"
48 : #include "llvm/Analysis/ScalarEvolution.h"
49 : #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
50 : #include "llvm/Analysis/ScopedNoAliasAA.h"
51 : #include "llvm/Analysis/TargetLibraryInfo.h"
52 : #include "llvm/Analysis/TargetTransformInfo.h"
53 : #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
54 : #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
55 : #include "llvm/CodeGen/UnreachableBlockElim.h"
56 : #include "llvm/IR/Dominators.h"
57 : #include "llvm/IR/IRPrintingPasses.h"
58 : #include "llvm/IR/PassManager.h"
59 : #include "llvm/IR/Verifier.h"
60 : #include "llvm/Support/Debug.h"
61 : #include "llvm/Support/FormatVariadic.h"
62 : #include "llvm/Support/Regex.h"
63 : #include "llvm/Target/TargetMachine.h"
64 : #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
65 : #include "llvm/Transforms/IPO/AlwaysInliner.h"
66 : #include "llvm/Transforms/IPO/ArgumentPromotion.h"
67 : #include "llvm/Transforms/IPO/CalledValuePropagation.h"
68 : #include "llvm/Transforms/IPO/ConstantMerge.h"
69 : #include "llvm/Transforms/IPO/CrossDSOCFI.h"
70 : #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
71 : #include "llvm/Transforms/IPO/ElimAvailExtern.h"
72 : #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
73 : #include "llvm/Transforms/IPO/FunctionAttrs.h"
74 : #include "llvm/Transforms/IPO/FunctionImport.h"
75 : #include "llvm/Transforms/IPO/GlobalDCE.h"
76 : #include "llvm/Transforms/IPO/GlobalOpt.h"
77 : #include "llvm/Transforms/IPO/GlobalSplit.h"
78 : #include "llvm/Transforms/IPO/HotColdSplitting.h"
79 : #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
80 : #include "llvm/Transforms/IPO/Inliner.h"
81 : #include "llvm/Transforms/IPO/Internalize.h"
82 : #include "llvm/Transforms/IPO/LowerTypeTests.h"
83 : #include "llvm/Transforms/IPO/PartialInlining.h"
84 : #include "llvm/Transforms/IPO/SCCP.h"
85 : #include "llvm/Transforms/IPO/SampleProfile.h"
86 : #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
87 : #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
88 : #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
89 : #include "llvm/Transforms/InstCombine/InstCombine.h"
90 : #include "llvm/Transforms/Instrumentation/AddressSanitizerPass.h"
91 : #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
92 : #include "llvm/Transforms/Instrumentation/CGProfile.h"
93 : #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
94 : #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
95 : #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
96 : #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
97 : #include "llvm/Transforms/Scalar/ADCE.h"
98 : #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
99 : #include "llvm/Transforms/Scalar/BDCE.h"
100 : #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
101 : #include "llvm/Transforms/Scalar/ConstantHoisting.h"
102 : #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
103 : #include "llvm/Transforms/Scalar/DCE.h"
104 : #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
105 : #include "llvm/Transforms/Scalar/DivRemPairs.h"
106 : #include "llvm/Transforms/Scalar/EarlyCSE.h"
107 : #include "llvm/Transforms/Scalar/Float2Int.h"
108 : #include "llvm/Transforms/Scalar/GVN.h"
109 : #include "llvm/Transforms/Scalar/GuardWidening.h"
110 : #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
111 : #include "llvm/Transforms/Scalar/IndVarSimplify.h"
112 : #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
113 : #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
114 : #include "llvm/Transforms/Scalar/JumpThreading.h"
115 : #include "llvm/Transforms/Scalar/LICM.h"
116 : #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
117 : #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
118 : #include "llvm/Transforms/Scalar/LoopDeletion.h"
119 : #include "llvm/Transforms/Scalar/LoopDistribute.h"
120 : #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
121 : #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
122 : #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
123 : #include "llvm/Transforms/Scalar/LoopPassManager.h"
124 : #include "llvm/Transforms/Scalar/LoopPredication.h"
125 : #include "llvm/Transforms/Scalar/LoopRotation.h"
126 : #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
127 : #include "llvm/Transforms/Scalar/LoopSink.h"
128 : #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
129 : #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
130 : #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
131 : #include "llvm/Transforms/Scalar/LowerAtomic.h"
132 : #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
133 : #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
134 : #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
135 : #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
136 : #include "llvm/Transforms/Scalar/NaryReassociate.h"
137 : #include "llvm/Transforms/Scalar/NewGVN.h"
138 : #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
139 : #include "llvm/Transforms/Scalar/Reassociate.h"
140 : #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
141 : #include "llvm/Transforms/Scalar/SCCP.h"
142 : #include "llvm/Transforms/Scalar/SROA.h"
143 : #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
144 : #include "llvm/Transforms/Scalar/SimplifyCFG.h"
145 : #include "llvm/Transforms/Scalar/Sink.h"
146 : #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
147 : #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
148 : #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
149 : #include "llvm/Transforms/Utils/AddDiscriminators.h"
150 : #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
151 : #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
152 : #include "llvm/Transforms/Utils/LCSSA.h"
153 : #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
154 : #include "llvm/Transforms/Utils/LoopSimplify.h"
155 : #include "llvm/Transforms/Utils/LowerInvoke.h"
156 : #include "llvm/Transforms/Utils/Mem2Reg.h"
157 : #include "llvm/Transforms/Utils/NameAnonGlobals.h"
158 : #include "llvm/Transforms/Utils/SymbolRewriter.h"
159 : #include "llvm/Transforms/Vectorize/LoopVectorize.h"
160 : #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
161 :
162 : using namespace llvm;
163 :
164 : static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
165 : cl::ReallyHidden, cl::init(4));
166 : static cl::opt<bool>
167 : RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
168 : cl::Hidden, cl::ZeroOrMore,
169 : cl::desc("Run Partial inlinining pass"));
170 :
171 : static cl::opt<bool>
172 : RunNewGVN("enable-npm-newgvn", cl::init(false),
173 : cl::Hidden, cl::ZeroOrMore,
174 : cl::desc("Run NewGVN instead of GVN"));
175 :
176 : static cl::opt<bool> EnableEarlyCSEMemSSA(
177 : "enable-npm-earlycse-memssa", cl::init(true), cl::Hidden,
178 : cl::desc("Enable the EarlyCSE w/ MemorySSA pass for the new PM (default = on)"));
179 :
180 : static cl::opt<bool> EnableGVNHoist(
181 : "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
182 : cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
183 :
184 : static cl::opt<bool> EnableGVNSink(
185 : "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
186 : cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
187 :
188 : static cl::opt<bool> EnableUnrollAndJam(
189 : "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
190 : cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
191 :
192 : static cl::opt<bool> EnableSyntheticCounts(
193 : "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
194 : cl::desc("Run synthetic function entry count generation "
195 : "pass"));
196 :
197 : static Regex DefaultAliasRegex(
198 : "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
199 :
200 : static cl::opt<bool>
201 : EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
202 : cl::desc("Enable control height reduction optimization (CHR)"));
203 :
204 : extern cl::opt<bool> EnableHotColdSplit;
205 :
206 : static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
207 2 : switch (Level) {
208 : case PassBuilder::O0:
209 : case PassBuilder::O1:
210 : case PassBuilder::O2:
211 : case PassBuilder::O3:
212 : return false;
213 :
214 : case PassBuilder::Os:
215 : case PassBuilder::Oz:
216 : return true;
217 : }
218 0 : llvm_unreachable("Invalid optimization level!");
219 : }
220 :
221 : namespace {
222 :
223 : /// No-op module pass which does nothing.
224 : struct NoOpModulePass {
225 0 : PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
226 0 : return PreservedAnalyses::all();
227 : }
228 : static StringRef name() { return "NoOpModulePass"; }
229 : };
230 :
231 : /// No-op module analysis.
232 : class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
233 : friend AnalysisInfoMixin<NoOpModuleAnalysis>;
234 : static AnalysisKey Key;
235 :
236 : public:
237 : struct Result {};
238 0 : Result run(Module &, ModuleAnalysisManager &) { return Result(); }
239 : static StringRef name() { return "NoOpModuleAnalysis"; }
240 : };
241 :
242 : /// No-op CGSCC pass which does nothing.
243 : struct NoOpCGSCCPass {
244 0 : PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
245 : LazyCallGraph &, CGSCCUpdateResult &UR) {
246 0 : return PreservedAnalyses::all();
247 : }
248 : static StringRef name() { return "NoOpCGSCCPass"; }
249 : };
250 :
251 : /// No-op CGSCC analysis.
252 : class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
253 : friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
254 : static AnalysisKey Key;
255 :
256 : public:
257 : struct Result {};
258 0 : Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
259 0 : return Result();
260 : }
261 : static StringRef name() { return "NoOpCGSCCAnalysis"; }
262 : };
263 :
264 : /// No-op function pass which does nothing.
265 : struct NoOpFunctionPass {
266 0 : PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
267 0 : return PreservedAnalyses::all();
268 : }
269 : static StringRef name() { return "NoOpFunctionPass"; }
270 : };
271 :
272 : /// No-op function analysis.
273 : class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
274 : friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
275 : static AnalysisKey Key;
276 :
277 : public:
278 : struct Result {};
279 0 : Result run(Function &, FunctionAnalysisManager &) { return Result(); }
280 : static StringRef name() { return "NoOpFunctionAnalysis"; }
281 : };
282 :
283 : /// No-op loop pass which does nothing.
284 : struct NoOpLoopPass {
285 0 : PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
286 : LoopStandardAnalysisResults &, LPMUpdater &) {
287 0 : return PreservedAnalyses::all();
288 : }
289 : static StringRef name() { return "NoOpLoopPass"; }
290 : };
291 :
292 : /// No-op loop analysis.
293 : class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
294 : friend AnalysisInfoMixin<NoOpLoopAnalysis>;
295 : static AnalysisKey Key;
296 :
297 : public:
298 : struct Result {};
299 0 : Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
300 0 : return Result();
301 : }
302 : static StringRef name() { return "NoOpLoopAnalysis"; }
303 : };
304 :
305 : AnalysisKey NoOpModuleAnalysis::Key;
306 : AnalysisKey NoOpCGSCCAnalysis::Key;
307 : AnalysisKey NoOpFunctionAnalysis::Key;
308 : AnalysisKey NoOpLoopAnalysis::Key;
309 :
310 : } // End anonymous namespace.
311 :
312 346 : void PassBuilder::invokePeepholeEPCallbacks(
313 : FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
314 353 : for (auto &C : PeepholeEPCallbacks)
315 7 : C(FPM, Level);
316 346 : }
317 :
318 1031 : void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
319 : #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
320 : MAM.registerPass([&] { return CREATE_PASS; });
321 : #include "PassRegistry.def"
322 :
323 1036 : for (auto &C : ModuleAnalysisRegistrationCallbacks)
324 : C(MAM);
325 1031 : }
326 :
327 1027 : void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
328 : #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
329 : CGAM.registerPass([&] { return CREATE_PASS; });
330 : #include "PassRegistry.def"
331 :
332 1031 : for (auto &C : CGSCCAnalysisRegistrationCallbacks)
333 : C(CGAM);
334 1027 : }
335 :
336 1031 : void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
337 : #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
338 : FAM.registerPass([&] { return CREATE_PASS; });
339 : #include "PassRegistry.def"
340 :
341 1970 : for (auto &C : FunctionAnalysisRegistrationCallbacks)
342 : C(FAM);
343 1031 : }
344 :
345 1027 : void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
346 : #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
347 : LAM.registerPass([&] { return CREATE_PASS; });
348 : #include "PassRegistry.def"
349 :
350 1031 : for (auto &C : LoopAnalysisRegistrationCallbacks)
351 : C(LAM);
352 1027 : }
353 :
354 : FunctionPassManager
355 73 : PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
356 : ThinLTOPhase Phase,
357 : bool DebugLogging) {
358 : assert(Level != O0 && "Must request optimizations!");
359 73 : FunctionPassManager FPM(DebugLogging);
360 :
361 : // Form SSA out of local memory accesses after breaking apart aggregates into
362 : // scalars.
363 73 : FPM.addPass(SROA());
364 :
365 : // Catch trivial redundancies
366 73 : FPM.addPass(EarlyCSEPass(EnableEarlyCSEMemSSA));
367 :
368 : // Hoisting of scalars and load expressions.
369 73 : if (EnableGVNHoist)
370 0 : FPM.addPass(GVNHoistPass());
371 :
372 : // Global value numbering based sinking.
373 73 : if (EnableGVNSink) {
374 0 : FPM.addPass(GVNSinkPass());
375 0 : FPM.addPass(SimplifyCFGPass());
376 : }
377 :
378 : // Speculative execution if the target has divergent branches; otherwise nop.
379 73 : FPM.addPass(SpeculativeExecutionPass());
380 :
381 : // Optimize based on known information about branches, and cleanup afterward.
382 73 : FPM.addPass(JumpThreadingPass());
383 73 : FPM.addPass(CorrelatedValuePropagationPass());
384 73 : FPM.addPass(SimplifyCFGPass());
385 73 : if (Level == O3)
386 18 : FPM.addPass(AggressiveInstCombinePass());
387 146 : FPM.addPass(InstCombinePass());
388 :
389 : if (!isOptimizingForSize(Level))
390 62 : FPM.addPass(LibCallsShrinkWrapPass());
391 :
392 73 : invokePeepholeEPCallbacks(FPM, Level);
393 :
394 : // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
395 : // using the size value profile. Don't perform this when optimizing for size.
396 73 : if (PGOOpt && !PGOOpt->ProfileUseFile.empty() &&
397 : !isOptimizingForSize(Level))
398 2 : FPM.addPass(PGOMemOPSizeOpt());
399 :
400 73 : FPM.addPass(TailCallElimPass());
401 73 : FPM.addPass(SimplifyCFGPass());
402 :
403 : // Form canonically associated expression trees, and simplify the trees using
404 : // basic mathematical properties. For example, this will form (nearly)
405 : // minimal multiplication trees.
406 73 : FPM.addPass(ReassociatePass());
407 :
408 : // Add the primary loop simplification pipeline.
409 : // FIXME: Currently this is split into two loop pass pipelines because we run
410 : // some function passes in between them. These can and should be removed
411 : // and/or replaced by scheduling the loop pass equivalents in the correct
412 : // positions. But those equivalent passes aren't powerful enough yet.
413 : // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
414 : // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
415 : // fully replace `SimplifyCFGPass`, and the closest to the other we have is
416 : // `LoopInstSimplify`.
417 : LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
418 :
419 : // Simplify the loop body. We do this initially to clean up after other loop
420 : // passes run, either when iterating on a loop or on inner loops with
421 : // implications on the outer loop.
422 73 : LPM1.addPass(LoopInstSimplifyPass());
423 73 : LPM1.addPass(LoopSimplifyCFGPass());
424 :
425 : // Rotate Loop - disable header duplication at -Oz
426 73 : LPM1.addPass(LoopRotatePass(Level != Oz));
427 73 : LPM1.addPass(LICMPass());
428 73 : LPM1.addPass(SimpleLoopUnswitchPass());
429 73 : LPM2.addPass(IndVarSimplifyPass());
430 73 : LPM2.addPass(LoopIdiomRecognizePass());
431 :
432 74 : for (auto &C : LateLoopOptimizationsEPCallbacks)
433 1 : C(LPM2, Level);
434 :
435 73 : LPM2.addPass(LoopDeletionPass());
436 : // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
437 : // because it changes IR to makes profile annotation in back compile
438 : // inaccurate.
439 15 : if (Phase != ThinLTOPhase::PreLink ||
440 73 : !PGOOpt || PGOOpt->SampleProfileFile.empty())
441 138 : LPM2.addPass(LoopFullUnrollPass(Level));
442 :
443 74 : for (auto &C : LoopOptimizerEndEPCallbacks)
444 1 : C(LPM2, Level);
445 :
446 : // We provide the opt remark emitter pass for LICM to use. We only need to do
447 : // this once as it is immutable.
448 73 : FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
449 146 : FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging));
450 73 : FPM.addPass(SimplifyCFGPass());
451 146 : FPM.addPass(InstCombinePass());
452 146 : FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging));
453 :
454 : // Eliminate redundancies.
455 73 : if (Level != O1) {
456 : // These passes add substantial compile time so skip them at O1.
457 64 : FPM.addPass(MergedLoadStoreMotionPass());
458 64 : if (RunNewGVN)
459 0 : FPM.addPass(NewGVNPass());
460 : else
461 64 : FPM.addPass(GVN());
462 : }
463 :
464 : // Specially optimize memory movement as it doesn't look like dataflow in SSA.
465 146 : FPM.addPass(MemCpyOptPass());
466 :
467 : // Sparse conditional constant propagation.
468 : // FIXME: It isn't clear why we do this *after* loop passes rather than
469 : // before...
470 73 : FPM.addPass(SCCPPass());
471 :
472 : // Delete dead bit computations (instcombine runs after to fold away the dead
473 : // computations, and then ADCE will run later to exploit any new DCE
474 : // opportunities that creates).
475 73 : FPM.addPass(BDCEPass());
476 :
477 : // Run instcombine after redundancy and dead bit elimination to exploit
478 : // opportunities opened up by them.
479 73 : FPM.addPass(InstCombinePass());
480 73 : invokePeepholeEPCallbacks(FPM, Level);
481 :
482 : // Re-consider control flow based optimizations after redundancy elimination,
483 : // redo DCE, etc.
484 73 : FPM.addPass(JumpThreadingPass());
485 73 : FPM.addPass(CorrelatedValuePropagationPass());
486 73 : FPM.addPass(DSEPass());
487 146 : FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging));
488 :
489 74 : for (auto &C : ScalarOptimizerLateEPCallbacks)
490 1 : C(FPM, Level);
491 :
492 : // Finally, do an expensive DCE pass to catch all the dead code exposed by
493 : // the simplifications and basic cleanup after all the simplifications.
494 73 : FPM.addPass(ADCEPass());
495 73 : FPM.addPass(SimplifyCFGPass());
496 73 : FPM.addPass(InstCombinePass());
497 73 : invokePeepholeEPCallbacks(FPM, Level);
498 :
499 73 : if (EnableCHR && Level == O3 && PGOOpt &&
500 0 : (!PGOOpt->ProfileUseFile.empty() || !PGOOpt->SampleProfileFile.empty()))
501 0 : FPM.addPass(ControlHeightReductionPass());
502 :
503 73 : return FPM;
504 : }
505 :
506 3 : void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
507 : PassBuilder::OptimizationLevel Level,
508 : bool RunProfileGen,
509 : std::string ProfileGenFile,
510 : std::string ProfileUseFile,
511 : std::string ProfileRemappingFile) {
512 : // Generally running simplification passes and the inliner with an high
513 : // threshold results in smaller executables, but there may be cases where
514 : // the size grows, so let's be conservative here and skip this simplification
515 : // at -Os/Oz.
516 : if (!isOptimizingForSize(Level)) {
517 : InlineParams IP;
518 :
519 : // In the old pass manager, this is a cl::opt. Should still this be one?
520 3 : IP.DefaultThreshold = 75;
521 :
522 : // FIXME: The hint threshold has the same value used by the regular inliner.
523 : // This should probably be lowered after performance testing.
524 : // FIXME: this comment is cargo culted from the old pass manager, revisit).
525 : IP.HintThreshold = 325;
526 :
527 : CGSCCPassManager CGPipeline(DebugLogging);
528 :
529 6 : CGPipeline.addPass(InlinerPass(IP));
530 :
531 : FunctionPassManager FPM;
532 3 : FPM.addPass(SROA());
533 3 : FPM.addPass(EarlyCSEPass()); // Catch trivial redundancies.
534 3 : FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
535 3 : FPM.addPass(InstCombinePass()); // Combine silly sequences.
536 3 : invokePeepholeEPCallbacks(FPM, Level);
537 :
538 6 : CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
539 :
540 6 : MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
541 : }
542 :
543 : // Delete anything that is now dead to make sure that we don't instrument
544 : // dead code. Instrumentation can end up keeping dead code around and
545 : // dramatically increase code size.
546 3 : MPM.addPass(GlobalDCEPass());
547 :
548 3 : if (RunProfileGen) {
549 1 : MPM.addPass(PGOInstrumentationGen());
550 :
551 : FunctionPassManager FPM;
552 1 : FPM.addPass(
553 2 : createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
554 2 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
555 :
556 : // Add the profile lowering pass.
557 : InstrProfOptions Options;
558 1 : if (!ProfileGenFile.empty())
559 : Options.InstrProfileOutput = ProfileGenFile;
560 1 : Options.DoCounterPromotion = true;
561 1 : MPM.addPass(InstrProfiling(Options));
562 : }
563 :
564 3 : if (!ProfileUseFile.empty())
565 2 : MPM.addPass(PGOInstrumentationUse(ProfileUseFile, ProfileRemappingFile));
566 3 : }
567 :
568 : static InlineParams
569 : getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
570 : auto O3 = PassBuilder::O3;
571 73 : unsigned OptLevel = Level > O3 ? 2 : Level;
572 90 : unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
573 90 : return getInlineParams(OptLevel, SizeLevel);
574 : }
575 :
576 : ModulePassManager
577 73 : PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
578 : ThinLTOPhase Phase,
579 : bool DebugLogging) {
580 73 : ModulePassManager MPM(DebugLogging);
581 :
582 : // Do basic inference of function attributes from known properties of system
583 : // libraries and other oracles.
584 73 : MPM.addPass(InferFunctionAttrsPass());
585 :
586 : // Create an early function pass manager to cleanup the output of the
587 : // frontend.
588 : FunctionPassManager EarlyFPM(DebugLogging);
589 73 : EarlyFPM.addPass(SimplifyCFGPass());
590 73 : EarlyFPM.addPass(SROA());
591 73 : EarlyFPM.addPass(EarlyCSEPass());
592 73 : EarlyFPM.addPass(LowerExpectIntrinsicPass());
593 73 : if (Level == O3)
594 18 : EarlyFPM.addPass(CallSiteSplittingPass());
595 :
596 : // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
597 : // to convert bitcast to direct calls so that they can be inlined during the
598 : // profile annotation prepration step.
599 : // More details about SamplePGO design can be found in:
600 : // https://research.google.com/pubs/pub45290.html
601 : // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
602 73 : if (PGOOpt && !PGOOpt->SampleProfileFile.empty() &&
603 : Phase == ThinLTOPhase::PostLink)
604 4 : EarlyFPM.addPass(InstCombinePass());
605 146 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
606 :
607 73 : if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) {
608 : // Annotate sample profile right after early FPM to ensure freshness of
609 : // the debug info.
610 27 : MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile,
611 : PGOOpt->ProfileRemappingFile,
612 : Phase == ThinLTOPhase::PreLink));
613 : // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
614 : // for the profile annotation to be accurate in the ThinLTO backend.
615 9 : if (Phase != ThinLTOPhase::PreLink)
616 : // We perform early indirect call promotion here, before globalopt.
617 : // This is important for the ThinLTO backend phase because otherwise
618 : // imported available_externally functions look unreferenced and are
619 : // removed.
620 10 : MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
621 : true));
622 : }
623 :
624 73 : if (EnableHotColdSplit)
625 0 : MPM.addPass(HotColdSplittingPass());
626 :
627 : // Interprocedural constant propagation now that basic cleanup has occurred
628 : // and prior to optimizing globals.
629 : // FIXME: This position in the pipeline hasn't been carefully considered in
630 : // years, it should be re-analyzed.
631 73 : MPM.addPass(IPSCCPPass());
632 :
633 : // Attach metadata to indirect call sites indicating the set of functions
634 : // they may target at run-time. This should follow IPSCCP.
635 73 : MPM.addPass(CalledValuePropagationPass());
636 :
637 : // Optimize globals to try and fold them into constants.
638 73 : MPM.addPass(GlobalOptPass());
639 :
640 : // Promote any localized globals to SSA registers.
641 : // FIXME: Should this instead by a run of SROA?
642 : // FIXME: We should probably run instcombine and simplify-cfg afterward to
643 : // delete control flows that are dead once globals have been folded to
644 : // constants.
645 73 : MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
646 :
647 : // Remove any dead arguments exposed by cleanups and constand folding
648 : // globals.
649 73 : MPM.addPass(DeadArgumentEliminationPass());
650 :
651 : // Create a small function pass pipeline to cleanup after all the global
652 : // optimizations.
653 : FunctionPassManager GlobalCleanupPM(DebugLogging);
654 73 : GlobalCleanupPM.addPass(InstCombinePass());
655 73 : invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
656 :
657 73 : GlobalCleanupPM.addPass(SimplifyCFGPass());
658 146 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
659 :
660 : // Add all the requested passes for instrumentation PGO, if requested.
661 73 : if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
662 11 : (!PGOOpt->ProfileGenFile.empty() || !PGOOpt->ProfileUseFile.empty())) {
663 9 : addPGOInstrPasses(MPM, DebugLogging, Level, PGOOpt->RunProfileGen,
664 : PGOOpt->ProfileGenFile, PGOOpt->ProfileUseFile,
665 : PGOOpt->ProfileRemappingFile);
666 3 : MPM.addPass(PGOIndirectCallPromotion(false, false));
667 : }
668 :
669 : // Synthesize function entry counts for non-PGO compilation.
670 73 : if (EnableSyntheticCounts && !PGOOpt)
671 0 : MPM.addPass(SyntheticCountsPropagation());
672 :
673 : // Require the GlobalsAA analysis for the module so we can query it within
674 : // the CGSCC pipeline.
675 73 : MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
676 :
677 : // Require the ProfileSummaryAnalysis for the module so we can query it within
678 : // the inliner pass.
679 73 : MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
680 :
681 : // Now begin the main postorder CGSCC pipeline.
682 : // FIXME: The current CGSCC pipeline has its origins in the legacy pass
683 : // manager and trying to emulate its precise behavior. Much of this doesn't
684 : // make a lot of sense and we should revisit the core CGSCC structure.
685 : CGSCCPassManager MainCGPipeline(DebugLogging);
686 :
687 : // Note: historically, the PruneEH pass was run first to deduce nounwind and
688 : // generally clean up exception handling overhead. It isn't clear this is
689 : // valuable as the inliner doesn't currently care whether it is inlining an
690 : // invoke or a call.
691 :
692 : // Run the inliner first. The theory is that we are walking bottom-up and so
693 : // the callees have already been fully optimized, and we want to inline them
694 : // into the callers so that our optimizations can reflect that.
695 : // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
696 : // because it makes profile annotation in the backend inaccurate.
697 : InlineParams IP = getInlineParamsFromOptLevel(Level);
698 15 : if (Phase == ThinLTOPhase::PreLink &&
699 73 : PGOOpt && !PGOOpt->SampleProfileFile.empty())
700 : IP.HotCallSiteThreshold = 0;
701 146 : MainCGPipeline.addPass(InlinerPass(IP));
702 :
703 : // Now deduce any function attributes based in the current code.
704 73 : MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
705 :
706 : // When at O3 add argument promotion to the pass pipeline.
707 : // FIXME: It isn't at all clear why this should be limited to O3.
708 73 : if (Level == O3)
709 18 : MainCGPipeline.addPass(ArgumentPromotionPass());
710 :
711 : // Lastly, add the core function simplification pipeline nested inside the
712 : // CGSCC walk.
713 146 : MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
714 146 : buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
715 :
716 74 : for (auto &C : CGSCCOptimizerLateEPCallbacks)
717 1 : C(MainCGPipeline, Level);
718 :
719 : // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
720 : // to detect when we devirtualize indirect calls and iterate the SCC passes
721 : // in that case to try and catch knock-on inlining or function attrs
722 : // opportunities. Then we add it to the module pipeline by walking the SCCs
723 : // in postorder (or bottom-up).
724 73 : MPM.addPass(
725 146 : createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
726 : std::move(MainCGPipeline), MaxDevirtIterations)));
727 :
728 73 : return MPM;
729 : }
730 :
731 : ModulePassManager
732 58 : PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
733 : bool DebugLogging) {
734 : ModulePassManager MPM(DebugLogging);
735 :
736 : // Optimize globals now that the module is fully simplified.
737 58 : MPM.addPass(GlobalOptPass());
738 58 : MPM.addPass(GlobalDCEPass());
739 :
740 : // Run partial inlining pass to partially inline functions that have
741 : // large bodies.
742 58 : if (RunPartialInlining)
743 0 : MPM.addPass(PartialInlinerPass());
744 :
745 : // Remove avail extern fns and globals definitions since we aren't compiling
746 : // an object file for later LTO. For LTO we want to preserve these so they
747 : // are eligible for inlining at link-time. Note if they are unreferenced they
748 : // will be removed by GlobalDCE later, so this only impacts referenced
749 : // available externally globals. Eventually they will be suppressed during
750 : // codegen, but eliminating here enables more opportunity for GlobalDCE as it
751 : // may make globals referenced by available external functions dead and saves
752 : // running remaining passes on the eliminated functions.
753 58 : MPM.addPass(EliminateAvailableExternallyPass());
754 :
755 : // Do RPO function attribute inference across the module to forward-propagate
756 : // attributes where applicable.
757 : // FIXME: Is this really an optimization rather than a canonicalization?
758 58 : MPM.addPass(ReversePostOrderFunctionAttrsPass());
759 :
760 : // Re-require GloblasAA here prior to function passes. This is particularly
761 : // useful as the above will have inlined, DCE'ed, and function-attr
762 : // propagated everything. We should at this point have a reasonably minimal
763 : // and richly annotated call graph. By computing aliasing and mod/ref
764 : // information for all local globals here, the late loop passes and notably
765 : // the vectorizer will be able to use them to help recognize vectorizable
766 : // memory operations.
767 58 : MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
768 :
769 : FunctionPassManager OptimizePM(DebugLogging);
770 58 : OptimizePM.addPass(Float2IntPass());
771 : // FIXME: We need to run some loop optimizations to re-rotate loops after
772 : // simplify-cfg and others undo their rotation.
773 :
774 : // Optimize the loop execution. These passes operate on entire loop nests
775 : // rather than on each loop in an inside-out manner, and so they are actually
776 : // function passes.
777 :
778 89 : for (auto &C : VectorizerStartEPCallbacks)
779 31 : C(OptimizePM, Level);
780 :
781 : // First rotate loops that may have been un-rotated by prior passes.
782 58 : OptimizePM.addPass(
783 58 : createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
784 :
785 : // Distribute loops to allow partial vectorization. I.e. isolate dependences
786 : // into separate loop that would otherwise inhibit vectorization. This is
787 : // currently only performed for loops marked with the metadata
788 : // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
789 58 : OptimizePM.addPass(LoopDistributePass());
790 :
791 : // Now run the core loop vectorizer.
792 58 : OptimizePM.addPass(LoopVectorizePass());
793 :
794 : // Eliminate loads by forwarding stores from the previous iteration to loads
795 : // of the current iteration.
796 58 : OptimizePM.addPass(LoopLoadEliminationPass());
797 :
798 : // Cleanup after the loop optimization passes.
799 116 : OptimizePM.addPass(InstCombinePass());
800 :
801 : // Now that we've formed fast to execute loop structures, we do further
802 : // optimizations. These are run afterward as they might block doing complex
803 : // analyses and transforms such as what are needed for loop vectorization.
804 :
805 : // Cleanup after loop vectorization, etc. Simplification passes like CVP and
806 : // GVN, loop transforms, and others have already run, so it's now better to
807 : // convert to more optimized IR using more aggressive simplify CFG options.
808 : // The extra sinking transform can create larger basic blocks, so do this
809 : // before SLP vectorization.
810 58 : OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
811 : forwardSwitchCondToPhi(true).
812 : convertSwitchToLookupTable(true).
813 : needCanonicalLoops(false).
814 : sinkCommonInsts(true)));
815 :
816 : // Optimize parallel scalar instruction chains into SIMD instructions.
817 58 : OptimizePM.addPass(SLPVectorizerPass());
818 :
819 116 : OptimizePM.addPass(InstCombinePass());
820 :
821 : // Unroll small loops to hide loop backedge latency and saturate any parallel
822 : // execution resources of an out-of-order processor. We also then need to
823 : // clean up redundancies and loop invariant code.
824 : // FIXME: It would be really good to use a loop-integrated instruction
825 : // combiner for cleanup here so that the unrolling and LICM can be pipelined
826 : // across the loop nests.
827 : // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
828 58 : if (EnableUnrollAndJam) {
829 0 : OptimizePM.addPass(
830 0 : createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level)));
831 : }
832 116 : OptimizePM.addPass(LoopUnrollPass(Level));
833 58 : OptimizePM.addPass(InstCombinePass());
834 58 : OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
835 58 : OptimizePM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging));
836 :
837 : // Now that we've vectorized and unrolled loops, we may have more refined
838 : // alignment information, try to re-derive it here.
839 58 : OptimizePM.addPass(AlignmentFromAssumptionsPass());
840 :
841 : // LoopSink pass sinks instructions hoisted by LICM, which serves as a
842 : // canonicalization pass that enables other optimizations. As a result,
843 : // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
844 : // result too early.
845 58 : OptimizePM.addPass(LoopSinkPass());
846 :
847 : // And finally clean up LCSSA form before generating code.
848 58 : OptimizePM.addPass(InstSimplifyPass());
849 :
850 : // This hoists/decomposes div/rem ops. It should run after other sink/hoist
851 : // passes to avoid re-sinking, but before SimplifyCFG because it can allow
852 : // flattening of blocks.
853 58 : OptimizePM.addPass(DivRemPairsPass());
854 :
855 : // LoopSink (and other loop passes since the last simplifyCFG) might have
856 : // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
857 58 : OptimizePM.addPass(SimplifyCFGPass());
858 :
859 : // Optimize PHIs by speculating around them when profitable. Note that this
860 : // pass needs to be run after any PRE or similar pass as it is essentially
861 : // inserting redudnancies into the progrem. This even includes SimplifyCFG.
862 58 : OptimizePM.addPass(SpeculateAroundPHIsPass());
863 :
864 : // Add the core optimizing pipeline.
865 116 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
866 :
867 58 : MPM.addPass(CGProfilePass());
868 :
869 : // Now we need to do some global optimization transforms.
870 : // FIXME: It would seem like these should come first in the optimization
871 : // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
872 : // ordering here.
873 58 : MPM.addPass(GlobalDCEPass());
874 58 : MPM.addPass(ConstantMergePass());
875 :
876 58 : return MPM;
877 : }
878 :
879 : ModulePassManager
880 40 : PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
881 : bool DebugLogging) {
882 : assert(Level != O0 && "Must request optimizations for the default pipeline!");
883 :
884 40 : ModulePassManager MPM(DebugLogging);
885 :
886 : // Force any function attributes we want the rest of the pipeline to observe.
887 40 : MPM.addPass(ForceFunctionAttrsPass());
888 :
889 : // Apply module pipeline start EP callback.
890 43 : for (auto &C : PipelineStartEPCallbacks)
891 : C(MPM);
892 :
893 40 : if (PGOOpt && PGOOpt->SamplePGOSupport)
894 4 : MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
895 :
896 : // Add the core simplification pipeline.
897 40 : MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
898 : DebugLogging));
899 :
900 : // Now add the optimization pipeline.
901 40 : MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
902 :
903 40 : return MPM;
904 : }
905 :
906 : ModulePassManager
907 15 : PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
908 : bool DebugLogging) {
909 : assert(Level != O0 && "Must request optimizations for the default pipeline!");
910 :
911 15 : ModulePassManager MPM(DebugLogging);
912 :
913 : // Force any function attributes we want the rest of the pipeline to observe.
914 15 : MPM.addPass(ForceFunctionAttrsPass());
915 :
916 15 : if (PGOOpt && PGOOpt->SamplePGOSupport)
917 5 : MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
918 :
919 : // Apply module pipeline start EP callback.
920 16 : for (auto &C : PipelineStartEPCallbacks)
921 : C(MPM);
922 :
923 : // If we are planning to perform ThinLTO later, we don't bloat the code with
924 : // unrolling/vectorization/... now. Just simplify the module as much as we
925 : // can.
926 30 : MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
927 : DebugLogging));
928 :
929 : // Run partial inlining pass to partially inline functions that have
930 : // large bodies.
931 : // FIXME: It isn't clear whether this is really the right place to run this
932 : // in ThinLTO. Because there is another canonicalization and simplification
933 : // phase that will run after the thin link, running this here ends up with
934 : // less information than will be available later and it may grow functions in
935 : // ways that aren't beneficial.
936 15 : if (RunPartialInlining)
937 0 : MPM.addPass(PartialInlinerPass());
938 :
939 : // Reduce the size of the IR as much as possible.
940 15 : MPM.addPass(GlobalOptPass());
941 :
942 15 : return MPM;
943 : }
944 :
945 18 : ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
946 : OptimizationLevel Level, bool DebugLogging,
947 : const ModuleSummaryIndex *ImportSummary) {
948 18 : ModulePassManager MPM(DebugLogging);
949 :
950 18 : if (ImportSummary) {
951 : // These passes import type identifier resolutions for whole-program
952 : // devirtualization and CFI. They must run early because other passes may
953 : // disturb the specific instruction patterns that these passes look for,
954 : // creating dependencies on resolutions that may not appear in the summary.
955 : //
956 : // For example, GVN may transform the pattern assume(type.test) appearing in
957 : // two basic blocks into assume(phi(type.test, type.test)), which would
958 : // transform a dependency on a WPD resolution into a dependency on a type
959 : // identifier resolution for CFI.
960 : //
961 : // Also, WPD has access to more precise information than ICP and can
962 : // devirtualize more effectively, so it should operate on the IR first.
963 11 : MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
964 11 : MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
965 : }
966 :
967 : // Force any function attributes we want the rest of the pipeline to observe.
968 18 : MPM.addPass(ForceFunctionAttrsPass());
969 :
970 : // During the ThinLTO backend phase we perform early indirect call promotion
971 : // here, before globalopt. Otherwise imported available_externally functions
972 : // look unreferenced and are removed.
973 : // FIXME: move this into buildModuleSimplificationPipeline to merge the logic
974 : // with SamplePGO.
975 18 : if (!PGOOpt || PGOOpt->SampleProfileFile.empty())
976 16 : MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */,
977 : false /* SamplePGO */));
978 :
979 : // Add the core simplification pipeline.
980 18 : MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
981 : DebugLogging));
982 :
983 : // Now add the optimization pipeline.
984 18 : MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
985 :
986 18 : return MPM;
987 : }
988 :
989 : ModulePassManager
990 7 : PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
991 : bool DebugLogging) {
992 : assert(Level != O0 && "Must request optimizations for the default pipeline!");
993 : // FIXME: We should use a customized pre-link pipeline!
994 7 : return buildPerModuleDefaultPipeline(Level, DebugLogging);
995 : }
996 :
997 : ModulePassManager
998 18 : PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
999 : ModuleSummaryIndex *ExportSummary) {
1000 : assert(Level != O0 && "Must request optimizations for the default pipeline!");
1001 : ModulePassManager MPM(DebugLogging);
1002 :
1003 : // Remove unused virtual tables to improve the quality of code generated by
1004 : // whole-program devirtualization and bitset lowering.
1005 18 : MPM.addPass(GlobalDCEPass());
1006 :
1007 : // Force any function attributes we want the rest of the pipeline to observe.
1008 18 : MPM.addPass(ForceFunctionAttrsPass());
1009 :
1010 : // Do basic inference of function attributes from known properties of system
1011 : // libraries and other oracles.
1012 18 : MPM.addPass(InferFunctionAttrsPass());
1013 :
1014 18 : if (Level > 1) {
1015 : FunctionPassManager EarlyFPM(DebugLogging);
1016 17 : EarlyFPM.addPass(CallSiteSplittingPass());
1017 34 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1018 :
1019 : // Indirect call promotion. This should promote all the targets that are
1020 : // left by the earlier promotion pass that promotes intra-module targets.
1021 : // This two-step promotion is to save the compile time. For LTO, it should
1022 : // produce the same result as if we only do promotion here.
1023 17 : MPM.addPass(PGOIndirectCallPromotion(
1024 17 : true /* InLTO */, PGOOpt && !PGOOpt->SampleProfileFile.empty()));
1025 : // Propagate constants at call sites into the functions they call. This
1026 : // opens opportunities for globalopt (and inlining) by substituting function
1027 : // pointers passed as arguments to direct uses of functions.
1028 17 : MPM.addPass(IPSCCPPass());
1029 :
1030 : // Attach metadata to indirect call sites indicating the set of functions
1031 : // they may target at run-time. This should follow IPSCCP.
1032 17 : MPM.addPass(CalledValuePropagationPass());
1033 : }
1034 :
1035 : // Now deduce any function attributes based in the current code.
1036 18 : MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1037 : PostOrderFunctionAttrsPass()));
1038 :
1039 : // Do RPO function attribute inference across the module to forward-propagate
1040 : // attributes where applicable.
1041 : // FIXME: Is this really an optimization rather than a canonicalization?
1042 18 : MPM.addPass(ReversePostOrderFunctionAttrsPass());
1043 :
1044 : // Use inragne annotations on GEP indices to split globals where beneficial.
1045 18 : MPM.addPass(GlobalSplitPass());
1046 :
1047 : // Run whole program optimization of virtual call when the list of callees
1048 : // is fixed.
1049 18 : MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1050 :
1051 : // Stop here at -O1.
1052 18 : if (Level == 1) {
1053 : // The LowerTypeTestsPass needs to run to lower type metadata and the
1054 : // type.test intrinsics. The pass does nothing if CFI is disabled.
1055 1 : MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1056 1 : return MPM;
1057 : }
1058 :
1059 : // Optimize globals to try and fold them into constants.
1060 17 : MPM.addPass(GlobalOptPass());
1061 :
1062 : // Promote any localized globals to SSA registers.
1063 17 : MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1064 :
1065 : // Linking modules together can lead to duplicate global constant, only
1066 : // keep one copy of each constant.
1067 17 : MPM.addPass(ConstantMergePass());
1068 :
1069 : // Remove unused arguments from functions.
1070 17 : MPM.addPass(DeadArgumentEliminationPass());
1071 :
1072 : // Reduce the code after globalopt and ipsccp. Both can open up significant
1073 : // simplification opportunities, and both can propagate functions through
1074 : // function pointers. When this happens, we often have to resolve varargs
1075 : // calls, etc, so let instcombine do this.
1076 : FunctionPassManager PeepholeFPM(DebugLogging);
1077 17 : if (Level == O3)
1078 2 : PeepholeFPM.addPass(AggressiveInstCombinePass());
1079 17 : PeepholeFPM.addPass(InstCombinePass());
1080 17 : invokePeepholeEPCallbacks(PeepholeFPM, Level);
1081 :
1082 34 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1083 :
1084 : // Note: historically, the PruneEH pass was run first to deduce nounwind and
1085 : // generally clean up exception handling overhead. It isn't clear this is
1086 : // valuable as the inliner doesn't currently care whether it is inlining an
1087 : // invoke or a call.
1088 : // Run the inliner now.
1089 17 : MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1090 17 : InlinerPass(getInlineParamsFromOptLevel(Level))));
1091 :
1092 : // Optimize globals again after we ran the inliner.
1093 17 : MPM.addPass(GlobalOptPass());
1094 :
1095 : // Garbage collect dead functions.
1096 : // FIXME: Add ArgumentPromotion pass after once it's ported.
1097 17 : MPM.addPass(GlobalDCEPass());
1098 :
1099 : FunctionPassManager FPM(DebugLogging);
1100 : // The IPO Passes may leave cruft around. Clean up after them.
1101 17 : FPM.addPass(InstCombinePass());
1102 17 : invokePeepholeEPCallbacks(FPM, Level);
1103 :
1104 17 : FPM.addPass(JumpThreadingPass());
1105 :
1106 : // Break up allocas
1107 17 : FPM.addPass(SROA());
1108 :
1109 : // Run a few AA driver optimizations here and now to cleanup the code.
1110 34 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1111 :
1112 17 : MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1113 : PostOrderFunctionAttrsPass()));
1114 : // FIXME: here we run IP alias analysis in the legacy PM.
1115 :
1116 : FunctionPassManager MainFPM;
1117 :
1118 : // FIXME: once we fix LoopPass Manager, add LICM here.
1119 : // FIXME: once we provide support for enabling MLSM, add it here.
1120 : // FIXME: once we provide support for enabling NewGVN, add it here.
1121 17 : if (RunNewGVN)
1122 0 : MainFPM.addPass(NewGVNPass());
1123 : else
1124 17 : MainFPM.addPass(GVN());
1125 :
1126 : // Remove dead memcpy()'s.
1127 34 : MainFPM.addPass(MemCpyOptPass());
1128 :
1129 : // Nuke dead stores.
1130 17 : MainFPM.addPass(DSEPass());
1131 :
1132 : // FIXME: at this point, we run a bunch of loop passes:
1133 : // indVarSimplify, loopDeletion, loopInterchange, loopUnrool,
1134 : // loopVectorize. Enable them once the remaining issue with LPM
1135 : // are sorted out.
1136 :
1137 34 : MainFPM.addPass(InstCombinePass());
1138 17 : MainFPM.addPass(SimplifyCFGPass());
1139 17 : MainFPM.addPass(SCCPPass());
1140 17 : MainFPM.addPass(InstCombinePass());
1141 17 : MainFPM.addPass(BDCEPass());
1142 :
1143 : // FIXME: We may want to run SLPVectorizer here.
1144 : // After vectorization, assume intrinsics may tell us more
1145 : // about pointer alignments.
1146 : #if 0
1147 : MainFPM.add(AlignmentFromAssumptionsPass());
1148 : #endif
1149 :
1150 : // FIXME: Conditionally run LoadCombine here, after it's ported
1151 : // (in case we still have this pass, given its questionable usefulness).
1152 :
1153 17 : MainFPM.addPass(InstCombinePass());
1154 17 : invokePeepholeEPCallbacks(MainFPM, Level);
1155 17 : MainFPM.addPass(JumpThreadingPass());
1156 34 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1157 :
1158 : // Create a function that performs CFI checks for cross-DSO calls with
1159 : // targets in the current module.
1160 17 : MPM.addPass(CrossDSOCFIPass());
1161 :
1162 : // Lower type metadata and the type.test intrinsic. This pass supports
1163 : // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1164 : // to be run at link time if CFI is enabled. This pass does nothing if
1165 : // CFI is disabled.
1166 17 : MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1167 :
1168 : // Add late LTO optimization passes.
1169 : // Delete basic blocks, which optimization passes may have killed.
1170 17 : MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1171 :
1172 : // Drop bodies of available eternally objects to improve GlobalDCE.
1173 17 : MPM.addPass(EliminateAvailableExternallyPass());
1174 :
1175 : // Now that we have optimized the program, discard unreachable functions.
1176 17 : MPM.addPass(GlobalDCEPass());
1177 :
1178 : // FIXME: Enable MergeFuncs, conditionally, after ported, maybe.
1179 : return MPM;
1180 : }
1181 :
1182 70 : AAManager PassBuilder::buildDefaultAAPipeline() {
1183 : AAManager AA;
1184 :
1185 : // The order in which these are registered determines their priority when
1186 : // being queried.
1187 :
1188 : // First we register the basic alias analysis that provides the majority of
1189 : // per-function local AA logic. This is a stateless, on-demand local set of
1190 : // AA techniques.
1191 : AA.registerFunctionAnalysis<BasicAA>();
1192 :
1193 : // Next we query fast, specialized alias analyses that wrap IR-embedded
1194 : // information about aliasing.
1195 : AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1196 : AA.registerFunctionAnalysis<TypeBasedAA>();
1197 :
1198 : // Add support for querying global aliasing information when available.
1199 : // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1200 : // analysis, all that the `AAManager` can do is query for any *cached*
1201 : // results from `GlobalsAA` through a readonly proxy.
1202 : AA.registerModuleAnalysis<GlobalsAA>();
1203 :
1204 70 : return AA;
1205 : }
1206 :
1207 1788 : static Optional<int> parseRepeatPassName(StringRef Name) {
1208 : if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1209 : return None;
1210 : int Count;
1211 5 : if (Name.getAsInteger(0, Count) || Count <= 0)
1212 : return None;
1213 : return Count;
1214 : }
1215 :
1216 517 : static Optional<int> parseDevirtPassName(StringRef Name) {
1217 : if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1218 : return None;
1219 : int Count;
1220 3 : if (Name.getAsInteger(0, Count) || Count <= 0)
1221 : return None;
1222 : return Count;
1223 : }
1224 :
1225 : /// Tests whether a pass name starts with a valid prefix for a default pipeline
1226 : /// alias.
1227 1345 : static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1228 : return Name.startswith("default") || Name.startswith("thinlto") ||
1229 1345 : Name.startswith("lto");
1230 : }
1231 :
1232 : /// Tests whether registered callbacks will accept a given pass name.
1233 : ///
1234 : /// When parsing a pipeline text, the type of the outermost pipeline may be
1235 : /// omitted, in which case the type is automatically determined from the first
1236 : /// pass name in the text. This may be a name that is handled through one of the
1237 : /// callbacks. We check this through the oridinary parsing callbacks by setting
1238 : /// up a dummy PassManager in order to not force the client to also handle this
1239 : /// type of query.
1240 : template <typename PassManagerT, typename CallbacksT>
1241 1011 : static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1242 1011 : if (!Callbacks.empty()) {
1243 : PassManagerT DummyPM;
1244 1038 : for (auto &CB : Callbacks)
1245 538 : if (CB(Name, DummyPM, {}))
1246 : return true;
1247 : }
1248 : return false;
1249 : }
1250 11 :
1251 11 : template <typename CallbacksT>
1252 : static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1253 4 : // Manually handle aliases for pre-configured pipeline fragments.
1254 4 : if (startsWithDefaultPipelineAliasPrefix(Name))
1255 : return DefaultAliasRegex.match(Name);
1256 :
1257 : // Explicitly handle pass manager names.
1258 : if (Name == "module")
1259 36 : return true;
1260 36 : if (Name == "cgscc")
1261 : return true;
1262 54 : if (Name == "function")
1263 38 : return true;
1264 :
1265 : // Explicitly handle custom-parsed pass names.
1266 : if (parseRepeatPassName(Name))
1267 : return true;
1268 446 :
1269 446 : #define MODULE_PASS(NAME, CREATE_PASS) \
1270 : if (Name == NAME) \
1271 4 : return true;
1272 4 : #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1273 : if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1274 : return true;
1275 : #include "PassRegistry.def"
1276 :
1277 518 : return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1278 518 : }
1279 :
1280 976 : template <typename CallbacksT>
1281 492 : static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
1282 : // Explicitly handle pass manager names.
1283 : if (Name == "cgscc")
1284 : return true;
1285 : if (Name == "function")
1286 : return true;
1287 :
1288 965 : // Explicitly handle custom-parsed pass names.
1289 : if (parseRepeatPassName(Name))
1290 965 : return true;
1291 46 : if (parseDevirtPassName(Name))
1292 : return true;
1293 :
1294 : #define CGSCC_PASS(NAME, CREATE_PASS) \
1295 : if (Name == NAME) \
1296 : return true;
1297 : #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1298 : if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1299 : return true;
1300 : #include "PassRegistry.def"
1301 :
1302 791 : return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
1303 : }
1304 :
1305 : template <typename CallbacksT>
1306 : static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
1307 : // Explicitly handle pass manager names.
1308 : if (Name == "function")
1309 : return true;
1310 : if (Name == "loop")
1311 : return true;
1312 :
1313 518 : // Explicitly handle custom-parsed pass names.
1314 : if (parseRepeatPassName(Name))
1315 : return true;
1316 :
1317 513 : #define FUNCTION_PASS(NAME, CREATE_PASS) \
1318 : if (Name == NAME) \
1319 : return true;
1320 : #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
1321 : if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1322 : return true;
1323 : #include "PassRegistry.def"
1324 :
1325 513 : return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
1326 : }
1327 513 :
1328 : template <typename CallbacksT>
1329 : static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
1330 : // Explicitly handle pass manager names.
1331 : if (Name == "loop")
1332 : return true;
1333 :
1334 : // Explicitly handle custom-parsed pass names.
1335 : if (parseRepeatPassName(Name))
1336 : return true;
1337 :
1338 446 : #define LOOP_PASS(NAME, CREATE_PASS) \
1339 : if (Name == NAME) \
1340 : return true;
1341 : #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
1342 457 : if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1343 : return true;
1344 : #include "PassRegistry.def"
1345 :
1346 : return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
1347 : }
1348 :
1349 : Optional<std::vector<PassBuilder::PipelineElement>>
1350 443 : PassBuilder::parsePipelineText(StringRef Text) {
1351 : std::vector<PipelineElement> ResultPipeline;
1352 :
1353 : SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
1354 : &ResultPipeline};
1355 : for (;;) {
1356 : std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
1357 : size_t Pos = Text.find_first_of(",()");
1358 : Pipeline.push_back({Text.substr(0, Pos), {}});
1359 :
1360 : // If we have a single terminating name, we're done.
1361 36 : if (Pos == Text.npos)
1362 : break;
1363 :
1364 : char Sep = Text[Pos];
1365 27 : Text = Text.substr(Pos + 1);
1366 : if (Sep == ',')
1367 : // Just a name ending in a comma, continue.
1368 : continue;
1369 :
1370 : if (Sep == '(') {
1371 27 : // Push the inner pipeline onto the stack to continue processing.
1372 : PipelineStack.push_back(&Pipeline.back().InnerPipeline);
1373 : continue;
1374 : }
1375 :
1376 : assert(Sep == ')' && "Bogus separator!");
1377 : // When handling the close parenthesis, we greedily consume them to avoid
1378 : // empty strings in the pipeline.
1379 : do {
1380 : // If we try to pop the outer pipeline we have unbalanced parentheses.
1381 : if (PipelineStack.size() == 1)
1382 11 : return None;
1383 :
1384 : PipelineStack.pop_back();
1385 : } while (Text.consume_front(")"));
1386 1001 :
1387 1001 : // Check if we've finished parsing.
1388 : if (Text.empty())
1389 : break;
1390 1001 :
1391 : // Otherwise, the end of an inner pipeline always has to be followed by
1392 1932 : // a comma, and then we can continue.
1393 1932 : if (!Text.consume_front(","))
1394 1932 : return None;
1395 : }
1396 :
1397 1932 : if (PipelineStack.size() > 1)
1398 : // Unbalanced paretheses.
1399 : return None;
1400 1229 :
1401 1229 : assert(PipelineStack.back() == &ResultPipeline &&
1402 1229 : "Wrong pipeline at the bottom of the stack!");
1403 : return {std::move(ResultPipeline)};
1404 : }
1405 :
1406 730 : Error PassBuilder::parseModulePass(ModulePassManager &MPM,
1407 : const PipelineElement &E,
1408 380 : bool VerifyEachPass, bool DebugLogging) {
1409 380 : auto &Name = E.Name;
1410 : auto &InnerPipeline = E.InnerPipeline;
1411 :
1412 : // First handle complex passes like the pass managers which carry pipelines.
1413 : if (!InnerPipeline.empty()) {
1414 : if (Name == "module") {
1415 : ModulePassManager NestedMPM(DebugLogging);
1416 : if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1417 384 : VerifyEachPass, DebugLogging))
1418 : return Err;
1419 : MPM.addPass(std::move(NestedMPM));
1420 : return Error::success();
1421 : }
1422 : if (Name == "cgscc") {
1423 : CGSCCPassManager CGPM(DebugLogging);
1424 343 : if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
1425 : DebugLogging))
1426 : return Err;
1427 : MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1428 : return Error::success();
1429 : }
1430 : if (Name == "function") {
1431 : FunctionPassManager FPM(DebugLogging);
1432 : if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1433 993 : VerifyEachPass, DebugLogging))
1434 : return Err;
1435 : MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1436 : return Error::success();
1437 : }
1438 : if (auto Count = parseRepeatPassName(Name)) {
1439 : ModulePassManager NestedMPM(DebugLogging);
1440 : if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1441 : VerifyEachPass, DebugLogging))
1442 1063 : return Err;
1443 : MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
1444 : return Error::success();
1445 1063 : }
1446 :
1447 : for (auto &C : ModulePipelineParsingCallbacks)
1448 : if (C(Name, MPM, InnerPipeline))
1449 1063 : return Error::success();
1450 :
1451 12 : // Normal passes can't have pipelines.
1452 12 : return make_error<StringError>(
1453 24 : formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
1454 : inconvertibleErrorCode());
1455 20 : ;
1456 : }
1457 :
1458 : // Manually handle aliases for pre-configured pipeline fragments.
1459 178 : if (startsWithDefaultPipelineAliasPrefix(Name)) {
1460 178 : SmallVector<StringRef, 3> Matches;
1461 356 : if (!DefaultAliasRegex.match(Name, &Matches))
1462 : return make_error<StringError>(
1463 350 : formatv("unknown default pipeline alias '{0}'", Name).str(),
1464 : inconvertibleErrorCode());
1465 :
1466 : assert(Matches.size() == 3 && "Must capture two matched strings!");
1467 491 :
1468 491 : OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
1469 982 : .Case("O0", O0)
1470 : .Case("O1", O1)
1471 964 : .Case("O2", O2)
1472 : .Case("O3", O3)
1473 : .Case("Os", Os)
1474 2 : .Case("Oz", Oz);
1475 1 : if (L == O0)
1476 1 : // At O0 we do nothing at all!
1477 2 : return Error::success();
1478 :
1479 2 : if (Matches[1] == "default") {
1480 : MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
1481 : } else if (Matches[1] == "thinlto-pre-link") {
1482 : MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
1483 2 : } else if (Matches[1] == "thinlto") {
1484 1 : MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
1485 : } else if (Matches[1] == "lto-pre-link") {
1486 : MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
1487 : } else {
1488 : assert(Matches[1] == "lto" && "Not one of the matched options!");
1489 2 : MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
1490 3 : }
1491 : return Error::success();
1492 : }
1493 :
1494 : // Finally expand the basic registered passes from the .inc file.
1495 380 : #define MODULE_PASS(NAME, CREATE_PASS) \
1496 : if (Name == NAME) { \
1497 46 : MPM.addPass(CREATE_PASS); \
1498 : return Error::success(); \
1499 0 : }
1500 0 : #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1501 : if (Name == "require<" NAME ">") { \
1502 : MPM.addPass( \
1503 : RequireAnalysisPass< \
1504 46 : std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
1505 : return Error::success(); \
1506 : } \
1507 : if (Name == "invalidate<" NAME ">") { \
1508 : MPM.addPass(InvalidateAnalysisPass< \
1509 : std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1510 : return Error::success(); \
1511 46 : }
1512 : #include "PassRegistry.def"
1513 :
1514 : for (auto &C : ModulePipelineParsingCallbacks)
1515 : if (C(Name, MPM, InnerPipeline))
1516 42 : return Error::success();
1517 : return make_error<StringError>(
1518 18 : formatv("unknown module pass '{0}'", Name).str(),
1519 : inconvertibleErrorCode());
1520 14 : }
1521 :
1522 4 : Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
1523 : const PipelineElement &E, bool VerifyEachPass,
1524 : bool DebugLogging) {
1525 12 : auto &Name = E.Name;
1526 : auto &InnerPipeline = E.InnerPipeline;
1527 :
1528 : // First handle complex passes like the pass managers which carry pipelines.
1529 : if (!InnerPipeline.empty()) {
1530 : if (Name == "cgscc") {
1531 : CGSCCPassManager NestedCGPM(DebugLogging);
1532 : if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1533 : VerifyEachPass, DebugLogging))
1534 : return Err;
1535 : // Add the nested pass manager with the appropriate adaptor.
1536 : CGPM.addPass(std::move(NestedCGPM));
1537 : return Error::success();
1538 : }
1539 : if (Name == "function") {
1540 : FunctionPassManager FPM(DebugLogging);
1541 : if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1542 : VerifyEachPass, DebugLogging))
1543 : return Err;
1544 : // Add the nested pass manager with the appropriate adaptor.
1545 : CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
1546 : return Error::success();
1547 : }
1548 : if (auto Count = parseRepeatPassName(Name)) {
1549 : CGSCCPassManager NestedCGPM(DebugLogging);
1550 14 : if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1551 13 : VerifyEachPass, DebugLogging))
1552 : return Err;
1553 : CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
1554 2 : return Error::success();
1555 3 : }
1556 : if (auto MaxRepetitions = parseDevirtPassName(Name)) {
1557 : CGSCCPassManager NestedCGPM(DebugLogging);
1558 226 : if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1559 : VerifyEachPass, DebugLogging))
1560 : return Err;
1561 226 : CGPM.addPass(
1562 : createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
1563 : return Error::success();
1564 : }
1565 226 :
1566 : for (auto &C : CGSCCPipelineParsingCallbacks)
1567 0 : if (C(Name, CGPM, InnerPipeline))
1568 0 : return Error::success();
1569 0 :
1570 : // Normal passes can't have pipelines.
1571 : return make_error<StringError>(
1572 0 : formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
1573 : inconvertibleErrorCode());
1574 : }
1575 :
1576 31 : // Now expand the basic registered passes from the .inc file.
1577 31 : #define CGSCC_PASS(NAME, CREATE_PASS) \
1578 62 : if (Name == NAME) { \
1579 : CGPM.addPass(CREATE_PASS); \
1580 : return Error::success(); \
1581 60 : }
1582 : #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1583 : if (Name == "require<" NAME ">") { \
1584 5 : CGPM.addPass(RequireAnalysisPass< \
1585 1 : std::remove_reference<decltype(CREATE_PASS)>::type, \
1586 1 : LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
1587 2 : CGSCCUpdateResult &>()); \
1588 : return Error::success(); \
1589 2 : } \
1590 : if (Name == "invalidate<" NAME ">") { \
1591 : CGPM.addPass(InvalidateAnalysisPass< \
1592 4 : std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1593 3 : return Error::success(); \
1594 3 : }
1595 6 : #include "PassRegistry.def"
1596 :
1597 3 : for (auto &C : CGSCCPipelineParsingCallbacks)
1598 6 : if (C(Name, CGPM, InnerPipeline))
1599 : return Error::success();
1600 : return make_error<StringError>(
1601 : formatv("unknown cgscc pass '{0}'", Name).str(),
1602 1 : inconvertibleErrorCode());
1603 0 : }
1604 :
1605 : Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
1606 : const PipelineElement &E,
1607 : bool VerifyEachPass, bool DebugLogging) {
1608 2 : auto &Name = E.Name;
1609 3 : auto &InnerPipeline = E.InnerPipeline;
1610 :
1611 : // First handle complex passes like the pass managers which carry pipelines.
1612 : if (!InnerPipeline.empty()) {
1613 : if (Name == "function") {
1614 : FunctionPassManager NestedFPM(DebugLogging);
1615 : if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1616 : VerifyEachPass, DebugLogging))
1617 : return Err;
1618 : // Add the nested pass manager with the appropriate adaptor.
1619 : FPM.addPass(std::move(NestedFPM));
1620 : return Error::success();
1621 : }
1622 : if (Name == "loop") {
1623 : LoopPassManager LPM(DebugLogging);
1624 : if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
1625 : DebugLogging))
1626 : return Err;
1627 : // Add the nested pass manager with the appropriate adaptor.
1628 : FPM.addPass(
1629 : createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging));
1630 : return Error::success();
1631 : }
1632 : if (auto Count = parseRepeatPassName(Name)) {
1633 6 : FunctionPassManager NestedFPM(DebugLogging);
1634 5 : if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1635 : VerifyEachPass, DebugLogging))
1636 : return Err;
1637 2 : FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
1638 3 : return Error::success();
1639 : }
1640 :
1641 926 : for (auto &C : FunctionPipelineParsingCallbacks)
1642 : if (C(Name, FPM, InnerPipeline))
1643 : return Error::success();
1644 926 :
1645 : // Normal passes can't have pipelines.
1646 : return make_error<StringError>(
1647 : formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
1648 926 : inconvertibleErrorCode());
1649 : }
1650 1 :
1651 1 : // Now expand the basic registered passes from the .inc file.
1652 2 : #define FUNCTION_PASS(NAME, CREATE_PASS) \
1653 : if (Name == NAME) { \
1654 : FPM.addPass(CREATE_PASS); \
1655 2 : return Error::success(); \
1656 : }
1657 : #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
1658 : if (Name == "require<" NAME ">") { \
1659 165 : FPM.addPass( \
1660 165 : RequireAnalysisPass< \
1661 330 : std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
1662 : return Error::success(); \
1663 : } \
1664 163 : if (Name == "invalidate<" NAME ">") { \
1665 326 : FPM.addPass(InvalidateAnalysisPass< \
1666 : std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1667 : return Error::success(); \
1668 5 : }
1669 1 : #include "PassRegistry.def"
1670 1 :
1671 2 : for (auto &C : FunctionPipelineParsingCallbacks)
1672 : if (C(Name, FPM, InnerPipeline))
1673 2 : return Error::success();
1674 : return make_error<StringError>(
1675 : formatv("unknown function pass '{0}'", Name).str(),
1676 : inconvertibleErrorCode());
1677 10 : }
1678 8 :
1679 : Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
1680 : bool VerifyEachPass, bool DebugLogging) {
1681 : StringRef Name = E.Name;
1682 : auto &InnerPipeline = E.InnerPipeline;
1683 4 :
1684 6 : // First handle complex passes like the pass managers which carry pipelines.
1685 : if (!InnerPipeline.empty()) {
1686 : if (Name == "loop") {
1687 : LoopPassManager NestedLPM(DebugLogging);
1688 : if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
1689 : VerifyEachPass, DebugLogging))
1690 : return Err;
1691 : // Add the nested pass manager with the appropriate adaptor.
1692 : LPM.addPass(std::move(NestedLPM));
1693 : return Error::success();
1694 : }
1695 : if (auto Count = parseRepeatPassName(Name)) {
1696 : LoopPassManager NestedLPM(DebugLogging);
1697 : if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
1698 : VerifyEachPass, DebugLogging))
1699 : return Err;
1700 : LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
1701 : return Error::success();
1702 : }
1703 :
1704 : for (auto &C : LoopPipelineParsingCallbacks)
1705 : if (C(Name, LPM, InnerPipeline))
1706 : return Error::success();
1707 25 :
1708 19 : // Normal passes can't have pipelines.
1709 : return make_error<StringError>(
1710 : formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
1711 12 : inconvertibleErrorCode());
1712 18 : }
1713 :
1714 : // Now expand the basic registered passes from the .inc file.
1715 185 : #define LOOP_PASS(NAME, CREATE_PASS) \
1716 : if (Name == NAME) { \
1717 185 : LPM.addPass(CREATE_PASS); \
1718 : return Error::success(); \
1719 : }
1720 : #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
1721 185 : if (Name == "require<" NAME ">") { \
1722 : LPM.addPass(RequireAnalysisPass< \
1723 0 : std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
1724 0 : LoopAnalysisManager, LoopStandardAnalysisResults &, \
1725 0 : LPMUpdater &>()); \
1726 : return Error::success(); \
1727 : } \
1728 0 : if (Name == "invalidate<" NAME ">") { \
1729 : LPM.addPass(InvalidateAnalysisPass< \
1730 : std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1731 2 : return Error::success(); \
1732 1 : }
1733 1 : #include "PassRegistry.def"
1734 2 :
1735 : for (auto &C : LoopPipelineParsingCallbacks)
1736 2 : if (C(Name, LPM, InnerPipeline))
1737 : return Error::success();
1738 : return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
1739 : inconvertibleErrorCode());
1740 1 : }
1741 0 :
1742 : bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
1743 : #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
1744 : if (Name == NAME) { \
1745 : AA.registerModuleAnalysis< \
1746 2 : std::remove_reference<decltype(CREATE_PASS)>::type>(); \
1747 3 : return true; \
1748 : }
1749 : #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
1750 : if (Name == NAME) { \
1751 : AA.registerFunctionAnalysis< \
1752 : std::remove_reference<decltype(CREATE_PASS)>::type>(); \
1753 : return true; \
1754 : }
1755 : #include "PassRegistry.def"
1756 :
1757 : for (auto &C : AAParsingCallbacks)
1758 : if (C(Name, AA))
1759 : return true;
1760 : return false;
1761 : }
1762 :
1763 : Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
1764 : ArrayRef<PipelineElement> Pipeline,
1765 : bool VerifyEachPass,
1766 : bool DebugLogging) {
1767 : for (const auto &Element : Pipeline) {
1768 : if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
1769 : return Err;
1770 : // FIXME: No verifier support for Loop passes!
1771 8 : }
1772 5 : return Error::success();
1773 : }
1774 6 :
1775 9 : Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
1776 : ArrayRef<PipelineElement> Pipeline,
1777 : bool VerifyEachPass,
1778 155 : bool DebugLogging) {
1779 : for (const auto &Element : Pipeline) {
1780 : if (auto Err =
1781 : parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
1782 : return Err;
1783 : if (VerifyEachPass)
1784 : FPM.addPass(VerifierPass());
1785 : }
1786 : return Error::success();
1787 : }
1788 :
1789 : Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
1790 : ArrayRef<PipelineElement> Pipeline,
1791 : bool VerifyEachPass,
1792 : bool DebugLogging) {
1793 3 : for (const auto &Element : Pipeline) {
1794 0 : if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
1795 : return Err;
1796 : // FIXME: No verifier support for CGSCC passes!
1797 : }
1798 : return Error::success();
1799 172 : }
1800 :
1801 : void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
1802 : FunctionAnalysisManager &FAM,
1803 353 : CGSCCAnalysisManager &CGAM,
1804 370 : ModuleAnalysisManager &MAM) {
1805 : MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
1806 : MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
1807 : CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
1808 : FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
1809 : FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
1810 : FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
1811 537 : LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
1812 : }
1813 :
1814 : Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
1815 1453 : ArrayRef<PipelineElement> Pipeline,
1816 926 : bool VerifyEachPass,
1817 926 : bool DebugLogging) {
1818 : for (const auto &Element : Pipeline) {
1819 916 : if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
1820 18 : return Err;
1821 : if (VerifyEachPass)
1822 : MPM.addPass(VerifierPass());
1823 : }
1824 : return Error::success();
1825 184 : }
1826 :
1827 : // Primary pass pipeline description parsing routine for a \c ModulePassManager
1828 : // FIXME: Should this routine accept a TargetMachine or require the caller to
1829 407 : // pre-populate the analysis managers with target-specific stuff?
1830 452 : Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
1831 : StringRef PipelineText,
1832 : bool VerifyEachPass, bool DebugLogging) {
1833 : auto Pipeline = parsePipelineText(PipelineText);
1834 : if (!Pipeline || Pipeline->empty())
1835 : return make_error<StringError>(
1836 : formatv("invalid pipeline '{0}'", PipelineText).str(),
1837 1027 : inconvertibleErrorCode());
1838 :
1839 : // If the first name isn't at the module layer, wrap the pipeline up
1840 : // automatically.
1841 1027 : StringRef FirstName = Pipeline->front().Name;
1842 1027 :
1843 1027 : if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
1844 1027 : if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
1845 1027 : Pipeline = {{"cgscc", std::move(*Pipeline)}};
1846 1027 : } else if (isFunctionPassName(FirstName,
1847 1027 : FunctionPipelineParsingCallbacks)) {
1848 1027 : Pipeline = {{"function", std::move(*Pipeline)}};
1849 : } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
1850 971 : Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
1851 : } else {
1852 : for (auto &C : TopLevelPipelineParsingCallbacks)
1853 : if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
1854 2017 : return Error::success();
1855 2125 :
1856 : // Unknown pass or pipeline name!
1857 1046 : auto &InnerPipeline = Pipeline->front().InnerPipeline;
1858 29 : return make_error<StringError>(
1859 : formatv("unknown {0} name '{1}'",
1860 : (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
1861 : .str(),
1862 : inconvertibleErrorCode());
1863 : }
1864 : }
1865 :
1866 976 : if (auto Err =
1867 : parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
1868 : return Err;
1869 976 : return Error::success();
1870 976 : }
1871 :
1872 22 : // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
1873 33 : Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
1874 : StringRef PipelineText,
1875 : bool VerifyEachPass, bool DebugLogging) {
1876 : auto Pipeline = parsePipelineText(PipelineText);
1877 965 : if (!Pipeline || Pipeline->empty())
1878 : return make_error<StringError>(
1879 965 : formatv("invalid pipeline '{0}'", PipelineText).str(),
1880 510 : inconvertibleErrorCode());
1881 138 :
1882 441 : StringRef FirstName = Pipeline->front().Name;
1883 441 : if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
1884 828 : return make_error<StringError>(
1885 27 : formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
1886 60 : PipelineText)
1887 : .str(),
1888 8 : inconvertibleErrorCode());
1889 2 :
1890 : if (auto Err =
1891 : parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
1892 : return Err;
1893 : return Error::success();
1894 : }
1895 6 :
1896 6 : // Primary pass pipeline description parsing routine for a \c
1897 12 : // FunctionPassManager
1898 18 : Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
1899 : StringRef PipelineText,
1900 : bool VerifyEachPass, bool DebugLogging) {
1901 : auto Pipeline = parsePipelineText(PipelineText);
1902 957 : if (!Pipeline || Pipeline->empty())
1903 1916 : return make_error<StringError>(
1904 : formatv("invalid pipeline '{0}'", PipelineText).str(),
1905 : inconvertibleErrorCode());
1906 :
1907 : StringRef FirstName = Pipeline->front().Name;
1908 : if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
1909 3 : return make_error<StringError>(
1910 : formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
1911 : PipelineText)
1912 3 : .str(),
1913 3 : inconvertibleErrorCode());
1914 :
1915 0 : if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
1916 0 : DebugLogging))
1917 : return Err;
1918 3 : return Error::success();
1919 3 : }
1920 :
1921 1 : // Primary pass pipeline description parsing routine for a \c LoopPassManager
1922 : Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
1923 2 : StringRef PipelineText,
1924 3 : bool VerifyEachPass, bool DebugLogging) {
1925 : auto Pipeline = parsePipelineText(PipelineText);
1926 2 : if (!Pipeline || Pipeline->empty())
1927 4 : return make_error<StringError>(
1928 : formatv("invalid pipeline '{0}'", PipelineText).str(),
1929 : inconvertibleErrorCode());
1930 :
1931 : if (auto Err =
1932 : parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
1933 : return Err;
1934 16 :
1935 : return Error::success();
1936 : }
1937 16 :
1938 16 : Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
1939 : // If the pipeline just consists of the word 'default' just replace the AA
1940 0 : // manager with our default one.
1941 0 : if (PipelineText == "default") {
1942 : AA = buildDefaultAAPipeline();
1943 16 : return Error::success();
1944 16 : }
1945 :
1946 3 : while (!PipelineText.empty()) {
1947 : StringRef Name;
1948 6 : std::tie(Name, PipelineText) = PipelineText.split(',');
1949 9 : if (!parseAAPassName(AA, Name))
1950 : return make_error<StringError>(
1951 13 : formatv("unknown alias analysis name '{0}'", Name).str(),
1952 26 : inconvertibleErrorCode());
1953 : }
1954 :
1955 : return Error::success();
1956 : }
|
