LLVM 24.0.0git
NVPTXCodeGenPassBuilder.cpp
Go to the documentation of this file.
1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains the NVPTX CodeGen pipeline builder. It mirrors
10/// NVPTXPassConfig in NVPTXTargetMachine.cpp; the two must be kept in sync
11/// until the legacy pass manager path is removed.
12//===----------------------------------------------------------------------===//
13
14#include "NVPTX.h"
15#include "NVPTXAliasAnalysis.h"
16#include "NVPTXAsmPrinter.h"
17#include "NVPTXSubtarget.h"
18#include "NVPTXTargetMachine.h"
27#include "llvm/CodeGen/PEI.h"
40#include "llvm/MC/MCStreamer.h"
54
55using namespace llvm;
56
57namespace {
58
59class NVPTXCodeGenPassBuilder : public CodeGenPassBuilder {
60 using Base = CodeGenPassBuilder;
61
62 const NVPTXOptions &CLOpts;
63
64 NVPTXTargetMachine &getTM() const {
65 return static_cast<NVPTXTargetMachine &>(TM);
66 }
67
68public:
69 explicit NVPTXCodeGenPassBuilder(NVPTXTargetMachine &TM,
70 const CGPassBuilderOption &Opts,
71 PassInstrumentationCallbacks *PIC)
72 : CodeGenPassBuilder(TM, Opts, PIC), CLOpts(TM.getCLOpts()) {
73 // The following passes are known to not play well with virtual regs
74 // hanging around after register allocation (which in our case, is *all*
75 // registers). We explicitly disable them here. We do, however, need some
76 // functionality of the PrologEpilogCodeInserter pass, so we emulate that
77 // behavior in the NVPTXPrologEpilog pass (see NVPTXPrologEpilogPass.cpp).
78 disablePass<PrologEpilogInserterPass, MachineLateInstrsCleanupPass,
79 MachineCopyPropagationPass, TailDuplicatePass,
80 StackMapLivenessPass, PostRAMachineSinkingPass,
81 PostRASchedulerPass, FuncletLayoutPass, PatchableFunctionPass,
82 ShrinkWrapPass, RemoveLoadsIntoFakeUsesPass,
83 MachineBlockPlacementPass>();
84 }
85
86protected:
87 void addIRPasses(PassManagerWrapper &PMW) override;
88 Error addInstSelector(PassManagerWrapper &PMW) override;
89 void addPreRegAlloc(PassManagerWrapper &PMW) override;
90 void addPostRegAlloc(PassManagerWrapper &PMW) override;
91
92 // NVPTX has no register allocation; virtual registers are emitted directly.
93 void addTargetRegisterAllocator(PassManagerWrapper &PMW, bool) override {}
94 Error addFastRegAlloc(PassManagerWrapper &PMW) override;
95 Error addOptimizedRegAlloc(PassManagerWrapper &PMW) override;
96
97 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
98 void addAsmPrinter(PassManagerWrapper &PMW) override;
99 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
100
101private:
102 // If the opt level is aggressive, add GVN; otherwise, add EarlyCSE.
103 void addEarlyCSEOrGVNPass(PassManagerWrapper &PMW);
104
105 // Add passes that propagate special memory spaces.
106 void addAddressSpaceInferencePasses(PassManagerWrapper &PMW);
107
108 // Add passes that perform straight-line scalar optimizations.
109 void addStraightLineScalarOptimizationPasses(PassManagerWrapper &PMW);
110};
111
112} // namespace
113
114void NVPTXCodeGenPassBuilder::addEarlyCSEOrGVNPass(PassManagerWrapper &PMW) {
115 if (getOptLevel() == CodeGenOptLevel::Aggressive)
116 // Disable scalar PRE due to Register Pressure increase
117 addFunctionPass(GVNPass(GVNOptions().setScalarPRE(false)), PMW);
118 else
119 addFunctionPass(EarlyCSEPass(), PMW);
120}
121
122void NVPTXCodeGenPassBuilder::addAddressSpaceInferencePasses(
123 PassManagerWrapper &PMW) {
124 // NVPTXLowerArgs emits alloca for byval parameters which can often
125 // be eliminated by SROA.
126 addFunctionPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
127 /*AggregateToVector=*/true)),
128 PMW);
129 addFunctionPass(NVPTXLowerAllocaPass(), PMW);
130 // TODO: Consider running InferAddressSpaces during opt, earlier in the
131 // compilation flow.
132 addFunctionPass(InferAddressSpacesPass(), PMW);
133 addFunctionPass(NVPTXAtomicLowerPass(), PMW);
134}
135
136void NVPTXCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(
137 PassManagerWrapper &PMW) {
138 addFunctionPass(SeparateConstOffsetFromGEPPass(), PMW);
139 addFunctionPass(SpeculativeExecutionPass(), PMW);
140 // ReassociateGEPs exposes more opportunites for SLSR. See
141 // the example in reassociate-geps-and-slsr.ll.
142 addFunctionPass(StraightLineStrengthReducePass(), PMW);
143 // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN
144 // or EarlyCSE can reuse. GVN generates significantly better code than
145 // EarlyCSE for some of our benchmarks.
146 addEarlyCSEOrGVNPass(PMW);
147 // Run NaryReassociate after EarlyCSE/GVN to be more effective.
148 addFunctionPass(NaryReassociatePass(), PMW);
149 // NaryReassociate on GEPs creates redundant common expressions, so run
150 // EarlyCSE after it.
151 addFunctionPass(EarlyCSEPass(), PMW);
152}
153
154void NVPTXCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
155 const NVPTXSubtarget &ST = *getTM().getSubtargetImpl();
156
157 // NVVMReflectPass is added in the pipeline-start extension point, so
158 // hopefully running it here does nothing. But since we need it for
159 // correctness when lowering to NVPTX, run it here too, in case whoever built
160 // our pass pipeline didn't add it.
161 flushFPMsToMPM(PMW);
162 addModulePass(NVVMReflectPass(ST.getSmVersion()), PMW);
163 addModulePass(NVPTXCanonicalizeDynamicSharedPass(), PMW);
164
165 if (getOptLevel() != CodeGenOptLevel::None)
166 addFunctionPass(NVPTXImageOptimizerPass(), PMW);
167 flushFPMsToMPM(PMW);
168 addModulePass(NVPTXAssignValidGlobalNamesPass(), PMW);
169 addModulePass(GenericToNVVMPass(), PMW);
170
171 // Lower variadic calls before address space inference.
172 addModulePass(ExpandVariadicsPass(ExpandVariadicsMode::Lowering), PMW);
173
174 // NVPTXLowerArgs is required for correctness and should be run right
175 // before the address space inference passes.
176 if (getTM().getDrvInterface() == NVPTX::CUDA) {
177 addFunctionPass(NVPTXMarkKernelPtrsGlobalPass(), PMW);
178 flushFPMsToMPM(PMW);
179 }
180 addModulePass(NVPTXPromoteParamAlignPass(), PMW);
181 addModulePass(NVPTXLowerArgsPass(TM), PMW);
182 if (getOptLevel() != CodeGenOptLevel::None) {
183 addAddressSpaceInferencePasses(PMW);
184 addStraightLineScalarOptimizationPasses(PMW);
185 } else {
186 // Required for correct stack lowering
187 addFunctionPass(NVPTXLowerAllocaPass(), PMW);
188 }
189
190 addFunctionPass(AtomicExpandPass(TM), PMW);
191 flushFPMsToMPM(PMW);
192 addModulePass(NVPTXCtorDtorLoweringPass(), PMW);
193
194 // === LSR and other generic IR passes ===
195 Base::addIRPasses(PMW);
196 // EarlyCSE is not always strong enough to clean up what LSR produces. For
197 // example, GVN can combine
198 //
199 // %0 = add %a, %b
200 // %1 = add %b, %a
201 //
202 // and
203 //
204 // %0 = shl nsw %a, 2
205 // %1 = shl %a, 2
206 //
207 // but EarlyCSE can do neither of them.
208 if (getOptLevel() != CodeGenOptLevel::None) {
209 addEarlyCSEOrGVNPass(PMW);
210 if (!CLOpts.disable_nvptx_load_store_vectorizer)
211 addFunctionPass(LoadStoreVectorizerPass(), PMW);
212 addFunctionPass(SROAPass(SROAOptions(SROAOptions::PreserveCFG,
213 /*AggregateToVector=*/true)),
214 PMW);
215 addFunctionPass(NVPTXTagInvariantLoadsPass(), PMW);
216 if (!CLOpts.disable_nvptx_ir_peephole)
217 addFunctionPass(NVPTXIRPeepholePass(), PMW);
218 }
219
220 if (ST.hasPTXASUnreachableBug()) {
221 // Run LowerUnreachable to WAR a ptxas bug. See the commit description of
222 // 1ee4d880e8760256c606fe55b7af85a4f70d006d for more details.
223 addFunctionPass(NVPTXLowerUnreachablePass(TM.Options.TrapUnreachable,
224 TM.Options.NoTrapAfterNoreturn),
225 PMW);
226 }
227}
228
229Error NVPTXCodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
230 addFunctionPass(NVPTXLowerAggrCopiesPass(), PMW);
231 addFunctionPass(NVPTXAllocaHoistingPass(), PMW);
232 addMachineFunctionPass(NVPTXISelDAGToDAGPass(getTM(), getOptLevel()), PMW);
233 addMachineFunctionPass(NVPTXReplaceImageHandlesPass(), PMW);
234 return Error::success();
235}
236
237void NVPTXCodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
238 addMachineFunctionPass(NVPTXForwardParamsPass(), PMW);
239 if (getOptLevel() != CodeGenOptLevel::None)
240 addMachineFunctionPass(NVPTXAddressFolderPass(), PMW);
241 // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
242 addMachineFunctionPass(NVPTXProxyRegErasurePass(), PMW);
243}
244
245void NVPTXCodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
246 addMachineFunctionPass(NVPTXPrologEpilogPass(), PMW);
247 if (getOptLevel() != CodeGenOptLevel::None) {
248 // NVPTXPrologEpilogPass calculates frame object offset and replaces frame
249 // index with VRFrame register. NVPTXPeephole needs to be run after that
250 // and will replace VRFrame with VRFrameLocal when possible.
251 addMachineFunctionPass(NVPTXPeepholePass(), PMW);
252 }
253}
254
255Error NVPTXCodeGenPassBuilder::addFastRegAlloc(PassManagerWrapper &PMW) {
256 addMachineFunctionPass(PHIEliminationPass(), PMW);
257 addMachineFunctionPass(TwoAddressInstructionPass(), PMW);
258 return Error::success();
259}
260
261Error NVPTXCodeGenPassBuilder::addOptimizedRegAlloc(PassManagerWrapper &PMW) {
262 addMachineFunctionPass(ProcessImplicitDefsPass(), PMW);
263 addMachineFunctionPass(
264 RequireAnalysisPass<MachineLoopAnalysis, MachineFunction>(), PMW);
265 addMachineFunctionPass(PHIEliminationPass(), PMW);
266
267 addMachineFunctionPass(TwoAddressInstructionPass(), PMW);
268 addMachineFunctionPass(RegisterCoalescerPass(), PMW);
269
270 // PreRA instruction scheduling.
271 addMachineFunctionPass(MachineSchedulerPass(&TM), PMW);
272
273 addMachineFunctionPass(StackSlotColoringPass(), PMW);
274
275 // FIXME: Needs physical registers
276 // addMachineFunctionPass(MachineLICMPass(), PMW);
277
278 return Error::success();
279}
280
281void NVPTXCodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
282 addModulePass(NVPTXAsmPrinterBeginPass(), PMW, /*Force=*/true);
283}
284
285void NVPTXCodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
286 addMachineFunctionPass(NVPTXAsmPrinterPass(), PMW);
287}
288
289void NVPTXCodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
290 addModulePass(NVPTXAsmPrinterEndPass(), PMW);
291}
292
294#define GET_PASS_REGISTRY "NVPTXPassRegistry.def"
296
297 PB.registerPipelineStartEPCallback(
298 [this](ModulePassManager &PM, OptimizationLevel Level) {
299 // Before anything relies on alias analysis.
301
302 // We do not want to fold out calls to nvvm.reflect early if the user
303 // has not provided a target architecture just yet.
304 if (Subtarget.hasTargetName())
305 PM.addPass(NVVMReflectPass(Subtarget.getSmVersion()));
306
308 // Note: NVVMIntrRangePass was causing numerical discrepancies at one
309 // point, if issues crop up, consider disabling.
311 if (CLOpts.early_byval_copy)
313 PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
314 });
315
316 // Linking can bring in further declarations of the dynamic shared memory.
317 auto CanonicalizeAfterLink = [](ModulePassManager &PM, OptimizationLevel) {
319 };
320 PB.registerFullLinkTimeOptimizationEarlyEPCallback(CanonicalizeAfterLink);
321 PB.registerThinLinkTimeOptimizationEarlyEPCallback(CanonicalizeAfterLink);
322
323 if (!NoKernelInfoEndLTO) {
324 PB.registerFullLinkTimeOptimizationLastEPCallback(
325 [this](ModulePassManager &PM, OptimizationLevel Level) {
327 FPM.addPass(KernelInfoPrinter(this));
328 PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
329 });
330 }
331}
332
335 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
336 const CGPassBuilderOption &Opt, MCContext &Ctx,
338 auto CGPB = NVPTXCodeGenPassBuilder(*this, Opt, PIC);
339 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
340}
Interfaces for producing common pass manager configurations.
This file provides the interface for a simple, fast CSE pass.
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
This is the NVPTX address space based alias analysis pass.
ModuleAnalysisManager MAM
PassBuilder PB(Mod->getContext(), Machine, PassOpts->PTO, std::nullopt, &PIC)
PassInstrumentationCallbacks PIC
This file defines the Pass Instrumentation classes that provide instrumentation points into the pass ...
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
This file provides the interface for LLVM's Scalar Replacement of Aggregates pass.
This class provides access to building LLVM's passes.
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
static ErrorSuccess success()
Create a success value.
Definition Error.h:336
const TargetSubtargetInfo * getSubtargetImpl(const Function &) const override
Virtual method implemented by subclasses that returns a reference to that target's TargetSubtargetInf...
Context object for machine code objects.
Definition MCContext.h:83
void registerPassBuilderCallbacks(PassBuilder &PB) override
Allow the target to modify the pass pipeline.
Error buildCodeGenPipeline(ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opt, MCContext &Ctx, PassInstrumentationCallbacks *PIC) override
This class provides access to building LLVM's passes.
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same_v< PassT, PassManager > > addPass(PassT &&Pass)
An abstract base class for streams implementations that also support a pwrite operation.
Interfaces for registering analysis passes, producing common pass manager configurations,...
const AArch64MCOptions & getCLOpts(const MCTargetOptions &Options)
This is an optimization pass for GlobalISel generic memory operations.
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
CodeGenFileType
These enums are meant to be passed into addPassesToEmitFile to indicate what type of file to emit,...
Definition CodeGen.h:309
PassManager< Module > ModulePassManager
Convenience typedef for a pass manager over modules.
PassManager< Function > FunctionPassManager
Convenience typedef for a pass manager over functions.
LLVM_ABI llvm::cl::opt< bool > NoKernelInfoEndLTO
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39