Go to the documentation of this file.
40 #define DEBUG_TYPE "x86-vzeroupper"
44 cl::desc(
"Minimize AVX to SSE transition penalty"),
47 STATISTIC(NumVZU,
"Number of vzeroupper instructions inserted");
62 StringRef getPassName()
const override {
return "X86 vzeroupper inserter"; }
70 using BlockExitState =
enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
72 static const char* getBlockExitStateName(BlockExitState
ST);
93 BlockExitState ExitState = PASS_THROUGH;
94 bool AddedToDirtySuccessors =
false;
97 BlockState() =
default;
103 BlockStateMap BlockStates;
104 DirtySuccessorsWorkList DirtySuccessors;
117 return new VZeroUpperInserter();
121 const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState
ST) {
123 case PASS_THROUGH:
return "Pass-through";
124 case EXITS_DIRTY:
return "Exits-dirty";
125 case EXITS_CLEAN:
return "Exits-clean";
134 return (
Reg >= X86::YMM0 &&
Reg <= X86::YMM15) ||
135 (
Reg >= X86::ZMM0 &&
Reg <= X86::ZMM15);
139 for (std::pair<unsigned, unsigned> LI :
MRI.
liveins())
147 for (
unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
151 for (
unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
174 assert(
MI.isCall() &&
"Can only be called on call instructions.");
187 EverMadeChange =
true;
192 if (!BlockStates[
MBB.
getNumber()].AddedToDirtySuccessors) {
193 DirtySuccessors.push_back(&
MBB);
194 BlockStates[
MBB.
getNumber()].AddedToDirtySuccessors =
true;
203 BlockExitState CurState = PASS_THROUGH;
207 bool IsCall =
MI.isCall();
208 bool IsReturn =
MI.isReturn();
209 bool IsControlFlow = IsCall || IsReturn;
213 if (IsX86INTR && IsReturn)
217 if (
MI.getOpcode() == X86::VZEROALL ||
MI.getOpcode() == X86::VZEROUPPER) {
218 CurState = EXITS_CLEAN;
223 if (!IsControlFlow && CurState == EXITS_DIRTY)
229 CurState = EXITS_DIRTY;
253 if (CurState == EXITS_DIRTY) {
257 insertVZeroUpper(
MI,
MBB);
258 CurState = EXITS_CLEAN;
259 }
else if (CurState == PASS_THROUGH) {
266 CurState = EXITS_CLEAN;
271 << getBlockExitStateName(CurState) <<
'\n');
273 if (CurState == EXITS_DIRTY)
275 addDirtySuccessor(*Succ);
287 if (!
ST.hasAVX() || !
ST.insertVZEROUPPER())
289 TII =
ST.getInstrInfo();
291 EverMadeChange =
false;
299 bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
300 for (
auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
314 assert(BlockStates.empty() && DirtySuccessors.empty() &&
315 "X86VZeroUpper state should be clear");
322 processBasicBlock(
MBB);
326 if (FnHasLiveInYmmOrZmm)
327 addDirtySuccessor(MF.front());
332 while (!DirtySuccessors.empty()) {
334 DirtySuccessors.pop_back();
339 if (BBState.FirstUnguardedCall !=
MBB.
end())
340 insertVZeroUpper(BBState.FirstUnguardedCall,
MBB);
345 if (BBState.ExitState == PASS_THROUGH) {
347 <<
" was Pass-through, is now Dirty-out.\n");
349 addDirtySuccessor(*Succ);
354 return EverMadeChange;
This is an optimization pass for GlobalISel generic memory operations.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Reg
All possible values of the reg field in the ModR/M byte.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
static bool hasYmmOrZmmReg(MachineInstr &MI)
static cl::opt< bool > UseVZeroUpper("x86-use-vzeroupper", cl::Hidden, cl::desc("Minimize AVX to SSE transition penalty"), cl::init(true))
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
Properties which a MachineFunction may have at a given point in time.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
TargetInstrInfo - Interface to description of machine instruction set.
const HexagonInstrInfo * TII
MachineOperand class - Representation of each machine instruction operand.
MachineFunctionProperties & set(Property P)
STATISTIC(NumFunctions, "Total number of functions")
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI)
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
Representation of each machine instruction.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
initializer< Ty > init(const Ty &Val)
static bool callHasRegMask(MachineInstr &MI)
Check if given call instruction has a RegMask operand.
ArrayRef< std::pair< MCRegister, Register > > liveins() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
iterator_range< succ_iterator > successors()
StringRef - Represent a constant reference to a string, i.e.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned const MachineRegisterInfo * MRI
Function & getFunction()
Return the LLVM function that this machine code represents.
static bool isYmmOrZmmReg(unsigned Reg)
VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
@ X86_INTR
X86_INTR - x86 hardware interrupt context.
static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass class - This class is used to implement most global optimizations.
FunctionPass * createX86IssueVZeroUpperPass()
This pass inserts AVX vzeroupper instructions before each call to avoid transition penalty between fu...
bool reg_nodbg_empty(Register RegNo) const
reg_nodbg_empty - Return true if the only instructions using or defining Reg are Debug instructions.