LLVM 24.0.0git
X86CodeGenPassBuilder.cpp
Go to the documentation of this file.
1//===-- X86CodeGenPassBuilder.cpp ---------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file contains X86 CodeGen pipeline builder.
10///
11//===----------------------------------------------------------------------===//
12
13#include "X86.h"
14#include "X86AsmPrinter.h"
15#include "X86TargetMachine.h"
16
31#include "llvm/CodeGen/KCFI.h"
35#include "llvm/MC/MCAsmInfo.h"
36#include "llvm/MC/MCStreamer.h"
37#include "llvm/Pass.h"
43
44using namespace llvm;
45
46namespace {
47
48class X86CodeGenPassBuilder : public CodeGenPassBuilder {
49 using Base = CodeGenPassBuilder;
50
51 X86TargetMachine &getTM() const {
52 return static_cast<X86TargetMachine &>(TM);
53 }
54
55public:
56 explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
57 const CGPassBuilderOption &Opts,
58 PassInstrumentationCallbacks *PIC)
59 : CodeGenPassBuilder(TM, Opts, PIC) {}
60
61protected:
62 void addIRPasses(PassManagerWrapper &PMW) override;
63 void addPreISel(PassManagerWrapper &PMW) override;
64 Error addInstSelector(PassManagerWrapper &PMW) override;
65
66 Error addIRTranslator(PassManagerWrapper &PMW) override;
67 void addPreLegalizeMachineIR(PassManagerWrapper &PMW) override;
68 Error addLegalizeMachineIR(PassManagerWrapper &PMW) override;
69 void addPreRegBankSelect(PassManagerWrapper &PMW) override;
70 Error addRegBankSelect(PassManagerWrapper &PMW) override;
71 Error addGlobalInstructionSelect(PassManagerWrapper &PMW) override;
72
73 void addMachineSSAOptimization(PassManagerWrapper &PMW) override;
74 void addILPOpts(PassManagerWrapper &PMW) override;
75 void addPreRegAlloc(PassManagerWrapper &PMW) override;
76 Error addRegAssignAndRewriteFast(PassManagerWrapper &PMW) override;
77 Expected<bool>
78 addRegAssignAndRewriteOptimized(PassManagerWrapper &PMW) override;
79 void addPostRegAlloc(PassManagerWrapper &PMW) override;
80 void addPreSched2(PassManagerWrapper &PMW) override;
81 void addPreEmitPass(PassManagerWrapper &PMW) override;
82 void addPreEmitPass2(PassManagerWrapper &PMW) override;
83 void addAsmPrinterBegin(PassManagerWrapper &PMW) override;
84 void addAsmPrinter(PassManagerWrapper &PMW) override;
85 void addAsmPrinterEnd(PassManagerWrapper &PMW) override;
86};
87
88void X86CodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) {
89 addFunctionPass(AtomicExpandPass(TM), PMW);
90
91 // We add both pass anyway and when these two passes run, one will be a
92 // no-op based on the optimization level/attributes.
93 addFunctionPass(X86LowerAMXIntrinsicsPass(&TM), PMW);
94 addFunctionPass(X86LowerAMXTypePass(&TM), PMW);
95
96 Base::addIRPasses(PMW);
97
98 if (getOptLevel() != CodeGenOptLevel::None) {
99 addFunctionPass(InterleavedAccessPass(TM), PMW);
100 addFunctionPass(X86PartialReductionPass(&getTM()), PMW);
101 }
102
103 // Add passes that handle indirect branch removal and insertion of a retpoline
104 // thunk. These will be a no-op unless a function subtarget has the retpoline
105 // feature enabled.
106 addFunctionPass(IndirectBrExpandPass(TM), PMW);
107
108 // Add Control Flow Guard checks.
109 const Triple &TT = TM.getTargetTriple();
110 if (TT.isOSWindows())
111 addFunctionPass(CFGuardPass(), PMW);
112
113 if (TM.Options.JMCInstrument) {
114 flushFPMsToMPM(PMW);
115 addModulePass(JMCInstrumenterPass(), PMW);
116 }
117
118 addFunctionPass(X86FenceNonTemporalStoresPass(&getTM()), PMW);
119}
120
121void X86CodeGenPassBuilder::addPreISel(PassManagerWrapper &PMW) {
122 // Only add this pass for 32-bit x86 Windows.
123 const Triple &TT = TM.getTargetTriple();
124 if (TT.isOSWindows() && TT.isX86_32()) {
125 flushFPMsToMPM(PMW);
126 addModulePass(X86WinEHStatePass(), PMW);
127 }
128}
129
130Error X86CodeGenPassBuilder::addInstSelector(PassManagerWrapper &PMW) {
131 addMachineFunctionPass(X86ISelDAGToDAGPass(getTM()), PMW);
132
133 // For ELF, cleanup any local-dynamic TLS accesses
134 if (TM.getTargetTriple().isOSBinFormatELF() &&
135 getOptLevel() != CodeGenOptLevel::None) {
136 addMachineFunctionPass(X86CleanupLocalDynamicTLSPass(), PMW);
137 }
138
139 addMachineFunctionPass(X86GlobalBaseRegPass(), PMW);
140 addMachineFunctionPass(X86ArgumentStackSlotPass(), PMW);
141 return Error::success();
142}
143
144Error X86CodeGenPassBuilder::addIRTranslator(PassManagerWrapper &PMW) {
145 addMachineFunctionPass(IRTranslatorPass(getOptLevel()), PMW);
146 return Error::success();
147}
148
149void X86CodeGenPassBuilder::addPreLegalizeMachineIR(PassManagerWrapper &PMW) {
150 if (getOptLevel() != CodeGenOptLevel::None)
151 addMachineFunctionPass(X86PreLegalizerCombinerPass(), PMW);
152}
153
154Error X86CodeGenPassBuilder::addLegalizeMachineIR(PassManagerWrapper &PMW) {
155 addMachineFunctionPass(LegalizerPass(), PMW);
156 return Error::success();
157}
158
159void X86CodeGenPassBuilder::addPreRegBankSelect(PassManagerWrapper &PMW) {
160 if (getOptLevel() != CodeGenOptLevel::None)
161 addMachineFunctionPass(X86PostLegalizerCombinerPass(), PMW);
162}
163
164Error X86CodeGenPassBuilder::addRegBankSelect(PassManagerWrapper &PMW) {
165 addMachineFunctionPass(RegBankSelectPass(), PMW);
166 return Error::success();
167}
168
169Error X86CodeGenPassBuilder::addGlobalInstructionSelect(
170 PassManagerWrapper &PMW) {
171 addMachineFunctionPass(InstructionSelectPass(getOptLevel()), PMW);
172 // Add GlobalBaseReg in case there is no SelectionDAG passes afterwards
173 if (isGlobalISelAbortEnabled())
174 addMachineFunctionPass(X86GlobalBaseRegPass(), PMW);
175 return Error::success();
176}
177
178void X86CodeGenPassBuilder::addMachineSSAOptimization(PassManagerWrapper &PMW) {
179 addMachineFunctionPass(X86DomainReassignmentPass(), PMW);
180 Base::addMachineSSAOptimization(PMW);
181}
182
183void X86CodeGenPassBuilder::addILPOpts(PassManagerWrapper &PMW) {
184 addMachineFunctionPass(EarlyIfConverterPass(), PMW);
185 if (getTM().getCLOpts().machine_combiner)
186 addMachineFunctionPass(MachineCombinerPass(), PMW);
187 addMachineFunctionPass(X86CmovConversionPass(), PMW);
188}
189
190void X86CodeGenPassBuilder::addPreRegAlloc(PassManagerWrapper &PMW) {
191 if (getOptLevel() != CodeGenOptLevel::None) {
192 addMachineFunctionPass(LiveRangeShrinkPass(), PMW);
193 addMachineFunctionPass(X86FixupSetCCPass(), PMW);
194 addMachineFunctionPass(X86OptimizeLEAsPass(), PMW);
195 addMachineFunctionPass(X86CallFrameOptimizationPass(), PMW);
196 addMachineFunctionPass(X86AvoidStoreForwardingBlocksPass(), PMW);
197 }
198
199 addMachineFunctionPass(X86SuppressAPXForRelocationPass(), PMW);
200 addMachineFunctionPass(X86SpeculativeLoadHardeningPass(), PMW);
201 addMachineFunctionPass(X86FlagsCopyLoweringPass(), PMW);
202 addMachineFunctionPass(X86DynAllocaExpanderPass(), PMW);
203
204 if (getOptLevel() != CodeGenOptLevel::None)
205 addMachineFunctionPass(X86PreTileConfigPass(), PMW);
206 else
207 addMachineFunctionPass(X86FastPreTileConfigPass(), PMW);
208}
209
210static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI,
211 const MachineRegisterInfo &MRI,
212 const Register Reg) {
214 return static_cast<const X86RegisterInfo &>(TRI).isTileRegisterClass(RC);
215}
216
217Error X86CodeGenPassBuilder::addRegAssignAndRewriteFast(
218 PassManagerWrapper &PMW) {
219 Error PossibleError = Base::addRegAssignAndRewriteFast(PMW);
220 addMachineFunctionPass(X86FastTileConfigPass(), PMW);
221 return PossibleError;
222}
223
224Expected<bool> X86CodeGenPassBuilder::addRegAssignAndRewriteOptimized(
225 PassManagerWrapper &PMW) {
226 // Don't support tile RA when RA is specified by command line "-regalloc".
227 if (Opt.RegAlloc == RegAllocType::Default && getTM().getCLOpts().tile_ra) {
228 // Allocate tile register first.
229 addMachineFunctionPass(
230 RAGreedyPass({RAGreedyPass::Options(onlyAllocateTileRegisters)}), PMW);
231 addMachineFunctionPass(X86TileConfigPass(), PMW);
232 }
233 return Base::addRegAssignAndRewriteOptimized(PMW);
234}
235
236void X86CodeGenPassBuilder::addPostRegAlloc(PassManagerWrapper &PMW) {
237 addMachineFunctionPass(X86LowerTileCopyPass(), PMW);
238 addMachineFunctionPass(X86FPStackifierPass(), PMW);
239 // When -O0 is enabled, the Load Value Injection Hardening pass will fall back
240 // to using the Speculative Execution Side Effect Suppression pass for
241 // mitigation. This is to prevent slow downs due to
242 // analyses needed by the LVIHardening pass when compiling at -O0.
243 if (getOptLevel() != CodeGenOptLevel::None) {
244 addMachineFunctionPass(X86LoadValueInjectionLoadHardeningPass(), PMW);
245 }
246}
247
248void X86CodeGenPassBuilder::addPreSched2(PassManagerWrapper &PMW) {
249 addMachineFunctionPass(X86ExpandPseudoPass(), PMW);
250 addMachineFunctionPass(MachineKCFIPass(), PMW);
251}
252
253void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
254 if (getOptLevel() != CodeGenOptLevel::None) {
255 addMachineFunctionPass(ExecutionDomainFixPass(), PMW);
256 addMachineFunctionPass(BreakFalseDepsPass(), PMW);
257 }
258
259 addMachineFunctionPass(X86IndirectBranchTrackingPass(), PMW);
260 addMachineFunctionPass(X86InsertVZeroUpperPass(), PMW);
261
262 if (getOptLevel() != CodeGenOptLevel::None) {
263 addMachineFunctionPass(X86FixupBWInstsPass(), PMW);
264 addMachineFunctionPass(X86PadShortFunctionsPass(), PMW);
265 addMachineFunctionPass(X86FixupLEAsPass(), PMW);
266 addMachineFunctionPass(X86FixupInstTuningPass(), PMW);
267 addMachineFunctionPass(X86FixupVectorConstantsPass(), PMW);
268 }
269 addMachineFunctionPass(X86CompressEVEXPass(), PMW);
270 addMachineFunctionPass(X86InsertX87WaitPass(), PMW);
271
272 if (TM.getTargetTriple().isLFI())
273 addMachineFunctionPass(X86LFIRewritePass(), PMW);
274}
275
276void X86CodeGenPassBuilder::addPreEmitPass2(PassManagerWrapper &PMW) {
277 const Triple &TT = TM.getTargetTriple();
278
279 // The X86 Speculative Execution Pass must run after all control
280 // flow graph modifying passes. As a result it was listed to run right before
281 // the X86 Retpoline Thunks pass. The reason it must run after control flow
282 // graph modifications is that the model of LFENCE in LLVM has to be updated
283 // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the
284 // placement of this pass was hand checked to ensure that the subsequent
285 // passes don't move the code around the LFENCEs in a way that will hurt the
286 // correctness of this pass. This placement has been shown to work based on
287 // hand inspection of the codegen output.
288 addMachineFunctionPass(X86SpeculativeExecutionSideEffectSuppressionPass(),
289 PMW);
290 addMachineFunctionPass(X86IndirectThunksPass(), PMW);
291 addMachineFunctionPass(X86ReturnThunksPass(), PMW);
292
293 // Insert extra int3 instructions after trailing call instructions to avoid
294 // issues in the unwinder.
295 if (TT.isOSWindows() && TT.isX86_64())
296 addMachineFunctionPass(X86AvoidTrailingCallPass(), PMW);
297
298 // Verify basic block incoming and outgoing cfa offset and register values and
299 // correct CFA calculation rule where needed by inserting appropriate CFI
300 // instructions. CFIInstrInserter self-skips targets that do not use DWARF
301 // CFI.
302 if (!TT.isOSDarwin())
303 addMachineFunctionPass(CFIInstrInserterPass(), PMW);
304
305 if (TT.isOSWindows()) {
306 // Identify valid longjmp targets for Windows Control Flow Guard.
307 addMachineFunctionPass(CFGuardLongjmpPass(), PMW);
308 // Identify valid eh continuation targets for Windows EHCont Guard.
309 addMachineFunctionPass(EHContGuardTargetsPass(), PMW);
310 }
311
312 addMachineFunctionPass(X86LoadValueInjectionRetHardeningPass(), PMW);
313
314 // Insert pseudo probe annotation for callsite profiling
315 addMachineFunctionPass(PseudoProbeInserterPass(), PMW);
316
317 // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms,
318 // also CALL_RVMARKER.
319 addMachineFunctionPass(
320 UnpackMachineBundlesPass([&TT](const MachineFunction &MF) {
321 // Only run bundle expansion if the module uses kcfi, or there are
322 // relevant ObjC runtime functions present in the module.
323 const Function &F = MF.getFunction();
324 const Module *M = F.getParent();
325 return M->getModuleFlag("kcfi") ||
326 (TT.isOSDarwin() &&
327 (M->getFunction("objc_retainAutoreleasedReturnValue") ||
328 M->getFunction("objc_unsafeClaimAutoreleasedReturnValue")));
329 }),
330 PMW);
331
332 // Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
333 // after all real instructions have been added to the epilog.
334 if (TT.isOSWindows() && TT.isX86_64()) {
335 addMachineFunctionPass(X86WinEHUnwindV2Pass(), PMW);
336 }
337}
338
339void X86CodeGenPassBuilder::addAsmPrinterBegin(PassManagerWrapper &PMW) {
340 addModulePass(X86AsmPrinterBeginPass(), PMW, /*Force=*/true);
341}
342
343void X86CodeGenPassBuilder::addAsmPrinter(PassManagerWrapper &PMW) {
344 addMachineFunctionPass(X86AsmPrinterPass(), PMW);
345}
346
347void X86CodeGenPassBuilder::addAsmPrinterEnd(PassManagerWrapper &PMW) {
348 addModulePass(X86AsmPrinterEndPass(), PMW, /*Force=*/true);
349}
350
351} // namespace
352
354#define GET_PASS_REGISTRY "X86PassRegistry.def"
356}
357
360 raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
361 const CGPassBuilderOption &Opt, MCContext &Ctx,
363 auto CGPB = X86CodeGenPassBuilder(*this, Opt, PIC);
364 return CGPB.buildPipeline(MPM, MAM, Out, DwoOut, FileType, Ctx);
365}
This file contains the declaration of the BreakFalseDepsPass class, used to identify and avoid false ...
Interfaces for producing common pass manager configurations.
This file contains the declaration of the MachineKCFI class, which is a Machine Pass that implements ...
This file declares the IRTranslator pass.
This file contains the declaration of the InterleavedAccessPass class, its corresponding pass name is...
#define F(x, y, z)
Definition MD5.cpp:54
Machine Check Debug Module
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
ModuleAnalysisManager MAM
PassBuilder PB(Mod->getContext(), Machine, PassOpts->PTO, std::nullopt, &PIC)
PassInstrumentationCallbacks PIC
MachineRegisterInfo & MRI
This file describes the interface of the MachineFunctionPass responsible for assigning the generic vi...
const GCNTargetMachine & getTM(const GCNSubtarget *STI)
static bool onlyAllocateTileRegisters(const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, const Register Reg)
This class provides access to building LLVM's passes.
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
static ErrorSuccess success()
Create a success value.
Definition Error.h:336
Context object for machine code objects.
Definition MCContext.h:83
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
This class provides access to building LLVM's passes.
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
void registerPassBuilderCallbacks(PassBuilder &PB) override
Allow the target to modify the pass pipeline.
Error buildCodeGenPipeline(ModulePassManager &MPM, ModuleAnalysisManager &MAM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opt, MCContext &Ctx, PassInstrumentationCallbacks *PIC) override
An abstract base class for streams implementations that also support a pwrite operation.
Interfaces for registering analysis passes, producing common pass manager configurations,...
const AArch64MCOptions & getCLOpts(const MCTargetOptions &Options)
This is an optimization pass for GlobalISel generic memory operations.
CodeGenFileType
These enums are meant to be passed into addPassesToEmitFile to indicate what type of file to emit,...
Definition CodeGen.h:309
PassManager< Module > ModulePassManager
Convenience typedef for a pass manager over modules.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58