LLVM 20.0.0git
X86VZeroUpper.cpp
Go to the documentation of this file.
1//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the pass which inserts x86 AVX vzeroupper instructions
10// before calls to SSE encoded functions. This avoids transition latency
11// penalty when transferring control between AVX encoded instructions and old
12// SSE encoding mode.
13//
14//===----------------------------------------------------------------------===//
15
16#include "X86.h"
17#include "X86InstrInfo.h"
18#include "X86Subtarget.h"
20#include "llvm/ADT/Statistic.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/IR/Function.h"
33#include "llvm/Support/Debug.h"
36#include <cassert>
37
38using namespace llvm;
39
40#define DEBUG_TYPE "x86-vzeroupper"
41
42static cl::opt<bool>
43UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
44 cl::desc("Minimize AVX to SSE transition penalty"),
45 cl::init(true));
46
47STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
48
49namespace {
50
51 class VZeroUpperInserter : public MachineFunctionPass {
52 public:
53 VZeroUpperInserter() : MachineFunctionPass(ID) {}
54
55 bool runOnMachineFunction(MachineFunction &MF) override;
56
59 MachineFunctionProperties::Property::NoVRegs);
60 }
61
62 StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
63
64 private:
65 void processBasicBlock(MachineBasicBlock &MBB);
66 void insertVZeroUpper(MachineBasicBlock::iterator I,
68 void addDirtySuccessor(MachineBasicBlock &MBB);
69
70 using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
71
72 static const char* getBlockExitStateName(BlockExitState ST);
73
74 // Core algorithm state:
75 // BlockState - Each block is either:
76 // - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
77 // vzeroupper instructions in this block.
78 // - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
79 // block that will ensure that YMM/ZMM is clean on exit.
80 // - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
81 // subsequent vzeroupper in the block clears it.
82 //
83 // AddedToDirtySuccessors - This flag is raised when a block is added to the
84 // DirtySuccessors list to ensure that it's not
85 // added multiple times.
86 //
87 // FirstUnguardedCall - Records the location of the first unguarded call in
88 // each basic block that may need to be guarded by a
89 // vzeroupper. We won't know whether it actually needs
90 // to be guarded until we discover a predecessor that
91 // is DIRTY_OUT.
92 struct BlockState {
93 BlockExitState ExitState = PASS_THROUGH;
94 bool AddedToDirtySuccessors = false;
95 MachineBasicBlock::iterator FirstUnguardedCall;
96
97 BlockState() = default;
98 };
99
100 using BlockStateMap = SmallVector<BlockState, 8>;
101 using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
102
103 BlockStateMap BlockStates;
104 DirtySuccessorsWorkList DirtySuccessors;
105 bool EverMadeChange;
106 bool IsX86INTR;
107 const TargetInstrInfo *TII;
108
109 static char ID;
110 };
111
112} // end anonymous namespace
113
114char VZeroUpperInserter::ID = 0;
115
117 return new VZeroUpperInserter();
118}
119
120#ifndef NDEBUG
121const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
122 switch (ST) {
123 case PASS_THROUGH: return "Pass-through";
124 case EXITS_DIRTY: return "Exits-dirty";
125 case EXITS_CLEAN: return "Exits-clean";
126 }
127 llvm_unreachable("Invalid block exit state.");
128}
129#endif
130
131/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
132/// Thus, there is no need to check for Y/ZMM16 and above.
133static bool isYmmOrZmmReg(unsigned Reg) {
134 return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
135 (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
136}
137
139 for (std::pair<MCRegister, Register> LI : MRI.liveins())
140 if (isYmmOrZmmReg(LI.first))
141 return true;
142
143 return false;
144}
145
147 for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
148 if (!MO.clobbersPhysReg(reg))
149 return false;
150 }
151 for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
152 if (!MO.clobbersPhysReg(reg))
153 return false;
154 }
155 return true;
156}
157
159 for (const MachineOperand &MO : MI.operands()) {
160 if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
161 return true;
162 if (!MO.isReg())
163 continue;
164 if (MO.isDebug())
165 continue;
166 if (isYmmOrZmmReg(MO.getReg()))
167 return true;
168 }
169 return false;
170}
171
172/// Check if given call instruction has a RegMask operand.
174 assert(MI.isCall() && "Can only be called on call instructions.");
175 for (const MachineOperand &MO : MI.operands()) {
176 if (MO.isRegMask())
177 return true;
178 }
179 return false;
180}
181
182/// Insert a vzeroupper instruction before I.
183void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
185 BuildMI(MBB, I, I->getDebugLoc(), TII->get(X86::VZEROUPPER));
186 ++NumVZU;
187 EverMadeChange = true;
188}
189
190/// Add MBB to the DirtySuccessors list if it hasn't already been added.
191void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
192 if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
193 DirtySuccessors.push_back(&MBB);
194 BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
195 }
196}
197
198/// Loop over all of the instructions in the basic block, inserting vzeroupper
199/// instructions before function calls.
200void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
201 // Start by assuming that the block is PASS_THROUGH which implies no unguarded
202 // calls.
203 BlockExitState CurState = PASS_THROUGH;
204 BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
205
206 for (MachineInstr &MI : MBB) {
207 bool IsCall = MI.isCall();
208 bool IsReturn = MI.isReturn();
209 bool IsControlFlow = IsCall || IsReturn;
210
211 // No need for vzeroupper before iret in interrupt handler function,
212 // epilogue will restore YMM/ZMM registers if needed.
213 if (IsX86INTR && IsReturn)
214 continue;
215
216 // An existing VZERO* instruction resets the state.
217 if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
218 CurState = EXITS_CLEAN;
219 continue;
220 }
221
222 // Shortcut: don't need to check regular instructions in dirty state.
223 if (!IsControlFlow && CurState == EXITS_DIRTY)
224 continue;
225
226 if (hasYmmOrZmmReg(MI)) {
227 // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
228 // instruction, or it could be control flow.
229 CurState = EXITS_DIRTY;
230 continue;
231 }
232
233 // Check for control-flow out of the current function (which might
234 // indirectly execute SSE instructions).
235 if (!IsControlFlow)
236 continue;
237
238 // If the call has no RegMask, skip it as well. It usually happens on
239 // helper function calls (such as '_chkstk', '_ftol2') where standard
240 // calling convention is not used (RegMask is not used to mark register
241 // clobbered and register usage (def/implicit-def/use) is well-defined and
242 // explicitly specified.
243 if (IsCall && !callHasRegMask(MI))
244 continue;
245
246 // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
247 // registers. In addition, the processor changes back to Clean state, after
248 // which execution of SSE instructions or AVX instructions has no transition
249 // penalty. Add the VZEROUPPER instruction before any function call/return
250 // that might execute SSE code.
251 // FIXME: In some cases, we may want to move the VZEROUPPER into a
252 // predecessor block.
253 if (CurState == EXITS_DIRTY) {
254 // After the inserted VZEROUPPER the state becomes clean again, but
255 // other YMM/ZMM may appear before other subsequent calls or even before
256 // the end of the BB.
257 insertVZeroUpper(MI, MBB);
258 CurState = EXITS_CLEAN;
259 } else if (CurState == PASS_THROUGH) {
260 // If this block is currently in pass-through state and we encounter a
261 // call then whether we need a vzeroupper or not depends on whether this
262 // block has successors that exit dirty. Record the location of the call,
263 // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
264 // It will be inserted later if necessary.
265 BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
266 CurState = EXITS_CLEAN;
267 }
268 }
269
270 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
271 << getBlockExitStateName(CurState) << '\n');
272
273 if (CurState == EXITS_DIRTY)
274 for (MachineBasicBlock *Succ : MBB.successors())
275 addDirtySuccessor(*Succ);
276
277 BlockStates[MBB.getNumber()].ExitState = CurState;
278}
279
280/// Loop over all of the basic blocks, inserting vzeroupper instructions before
281/// function calls.
282bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
283 if (!UseVZeroUpper)
284 return false;
285
287 if (!ST.hasAVX() || !ST.insertVZEROUPPER())
288 return false;
289 TII = ST.getInstrInfo();
291 EverMadeChange = false;
293
294 bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
295
296 // Fast check: if the function doesn't use any ymm/zmm registers, we don't
297 // need to insert any VZEROUPPER instructions. This is constant-time, so it
298 // is cheap in the common case of no ymm/zmm use.
299 bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
300 for (const auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
301 if (!YmmOrZmmUsed) {
302 for (MCPhysReg R : *RC) {
303 if (!MRI.reg_nodbg_empty(R)) {
304 YmmOrZmmUsed = true;
305 break;
306 }
307 }
308 }
309 }
310 if (!YmmOrZmmUsed)
311 return false;
312
313 assert(BlockStates.empty() && DirtySuccessors.empty() &&
314 "X86VZeroUpper state should be clear");
315 BlockStates.resize(MF.getNumBlockIDs());
316
317 // Process all blocks. This will compute block exit states, record the first
318 // unguarded call in each block, and add successors of dirty blocks to the
319 // DirtySuccessors list.
320 for (MachineBasicBlock &MBB : MF)
321 processBasicBlock(MBB);
322
323 // If any YMM/ZMM regs are live-in to this function, add the entry block to
324 // the DirtySuccessors list
325 if (FnHasLiveInYmmOrZmm)
326 addDirtySuccessor(MF.front());
327
328 // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
329 // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
330 // through PASS_THROUGH blocks.
331 while (!DirtySuccessors.empty()) {
332 MachineBasicBlock &MBB = *DirtySuccessors.back();
333 DirtySuccessors.pop_back();
334 BlockState &BBState = BlockStates[MBB.getNumber()];
335
336 // MBB is a successor of a dirty block, so its first call needs to be
337 // guarded.
338 if (BBState.FirstUnguardedCall != MBB.end())
339 insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
340
341 // If this successor was a pass-through block, then it is now dirty. Its
342 // successors need to be added to the worklist (if they haven't been
343 // already).
344 if (BBState.ExitState == PASS_THROUGH) {
345 LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
346 << " was Pass-through, is now Dirty-out.\n");
347 for (MachineBasicBlock *Succ : MBB.successors())
348 addDirtySuccessor(*Succ);
349 }
350 }
351
352 BlockStates.clear();
353 return EverMadeChange;
354}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock & MBB
#define LLVM_DEBUG(...)
Definition: Debug.h:106
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
static bool callHasRegMask(MachineInstr &MI)
Check if given call instruction has a RegMask operand.
static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI)
static bool hasYmmOrZmmReg(MachineInstr &MI)
static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO)
static cl::opt< bool > UseVZeroUpper("x86-use-vzeroupper", cl::Hidden, cl::desc("Minimize AVX to SSE transition penalty"), cl::init(true))
static bool isYmmOrZmmReg(unsigned Reg)
VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:277
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
Representation of each machine instruction.
Definition: MachineInstr.h:69
MachineOperand class - Representation of each machine instruction operand.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
TargetInstrInfo - Interface to description of machine instruction set.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ X86_INTR
x86 hardware interrupt context.
Definition: CallingConv.h:173
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createX86IssueVZeroUpperPass()
This pass inserts AVX vzeroupper instructions before each call to avoid transition penalty between fu...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163