Go to the documentation of this file.
65 #define PASS_KEY "x86-flags-copy-lowering"
66 #define DEBUG_TYPE PASS_KEY
68 STATISTIC(NumCopiesEliminated,
"Number of copies of EFLAGS eliminated");
69 STATISTIC(NumSetCCsInserted,
"Number of setCC instructions inserted");
70 STATISTIC(NumTestsInserted,
"Number of test instructions inserted");
71 STATISTIC(NumAddsInserted,
"Number of adds instructions inserted");
76 using CondRegArray = std::array<unsigned, X86::LAST_VALID_COND + 1>;
82 StringRef getPassName()
const override {
return "X86 EFLAGS copy lowering"; }
103 std::pair<unsigned, bool> getCondOrInverseInReg(
116 CondRegArray &CondRegs);
124 CondRegArray &CondRegs);
136 "X86 EFLAGS copy lowering",
false,
false)
141 return new X86FlagsCopyLoweringPass();
146 void X86FlagsCopyLoweringPass::getAnalysisUsage(
AnalysisUsage &AU)
const {
157 enum class FlagArithMnemonic {
172 "by this instruction!");
174 #define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX) \
175 case X86::MNEMONIC##8##SUFFIX: \
176 case X86::MNEMONIC##16##SUFFIX: \
177 case X86::MNEMONIC##32##SUFFIX: \
178 case X86::MNEMONIC##64##SUFFIX:
180 #define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC) \
181 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr) \
182 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr_REV) \
183 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rm) \
184 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, mr) \
185 case X86::MNEMONIC##8ri: \
186 case X86::MNEMONIC##16ri8: \
187 case X86::MNEMONIC##32ri8: \
188 case X86::MNEMONIC##64ri8: \
189 case X86::MNEMONIC##16ri: \
190 case X86::MNEMONIC##32ri: \
191 case X86::MNEMONIC##64ri32: \
192 case X86::MNEMONIC##8mi: \
193 case X86::MNEMONIC##16mi8: \
194 case X86::MNEMONIC##32mi8: \
195 case X86::MNEMONIC##64mi8: \
196 case X86::MNEMONIC##16mi: \
197 case X86::MNEMONIC##32mi: \
198 case X86::MNEMONIC##64mi32: \
199 case X86::MNEMONIC##8i8: \
200 case X86::MNEMONIC##16i16: \
201 case X86::MNEMONIC##32i32: \
202 case X86::MNEMONIC##64i32:
210 #undef LLVM_EXPAND_ADC_SBB_INSTR
215 return FlagArithMnemonic::RCL;
220 return FlagArithMnemonic::RCR;
222 #undef LLVM_EXPAND_INSTR_SIZES
228 return FlagArithMnemonic::ADCX;
234 return FlagArithMnemonic::ADOX;
238 return FlagArithMnemonic::SETB;
248 "Split instruction must be in the split block!");
250 "Only designed to split a tail of branch instructions!");
252 "Must split on an actual jCC instruction!");
258 "Must split after an actual jCC instruction!");
260 "Must only have this one terminator prior to the split!");
271 assert(MI.isTerminator() &&
272 "Should only have spliced terminators!");
274 MI.operands(), [&](MachineOperand &MOp) {
275 return MOp.isMBB() && MOp.getMBB() == &UnsplitSucc;
292 if (IsEdgeSplit || *
SI != &UnsplitSucc)
301 if (Succ != &UnsplitSucc)
306 "Failed to make the new block a successor!");
314 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
318 assert(OpMBB.
isMBB() &&
"Block operand to a PHI is not a block!");
323 if (!IsEdgeSplit || Succ != &UnsplitSucc) {
332 MI.addOperand(MF, OpV);
345 case X86::CMOVBE_Fp32:
case X86::CMOVBE_Fp64:
case X86::CMOVBE_Fp80:
347 case X86::CMOVB_Fp32:
case X86::CMOVB_Fp64:
case X86::CMOVB_Fp80:
349 case X86::CMOVE_Fp32:
case X86::CMOVE_Fp64:
case X86::CMOVE_Fp80:
351 case X86::CMOVNBE_Fp32:
case X86::CMOVNBE_Fp64:
case X86::CMOVNBE_Fp80:
353 case X86::CMOVNB_Fp32:
case X86::CMOVNB_Fp64:
case X86::CMOVNB_Fp80:
355 case X86::CMOVNE_Fp32:
case X86::CMOVNE_Fp64:
case X86::CMOVNE_Fp80:
357 case X86::CMOVNP_Fp32:
case X86::CMOVNP_Fp64:
case X86::CMOVNP_Fp80:
359 case X86::CMOVP_Fp32:
case X86::CMOVP_Fp64:
case X86::CMOVP_Fp80:
364 bool X86FlagsCopyLoweringPass::runOnMachineFunction(
MachineFunction &MF) {
370 TII = Subtarget->getInstrInfo();
371 TRI = Subtarget->getRegisterInfo();
372 MDT = &getAnalysis<MachineDominatorTree>();
373 PromoteRC = &X86::GR8RegClass;
387 if (
MI.getOpcode() == TargetOpcode::COPY &&
388 MI.getOperand(0).getReg() == X86::EFLAGS)
396 "The input to the copy for EFLAGS should always be a register!");
398 if (CopyDefI.
getOpcode() != TargetOpcode::COPY) {
414 dbgs() <<
"ERROR: Encountered unexpected def of an eflags copy: ";
417 "Cannot lower EFLAGS copy unless it is defined in turn by a copy!");
423 CopyI->eraseFromParent();
426 ++NumCopiesEliminated;
430 assert(DOp.isDef() &&
"Expected register def!");
431 assert(DOp.getReg() == X86::EFLAGS &&
"Unexpected copy def register!");
459 return &
MI != CopyI &&
MI.findRegisterDefOperand(X86::EFLAGS);
464 assert(MDT->dominates(BeginMBB, EndMBB) &&
465 "Only support paths down the dominator tree!");
470 Worklist.push_back(EndMBB);
474 if (!Visited.
insert(PredMBB).second)
476 if (HasEFLAGSClobber(PredMBB->begin(), PredMBB->end()))
479 Worklist.push_back(PredMBB);
481 }
while (!Worklist.empty());
486 !HasEFLAGSClobber(TestMBB->
begin(), TestPos)) {
493 return MDT->findNearestCommonDominator(LHS, RHS);
499 if (HasEFLAGSClobberPath(HoistMBB, TestMBB))
517 return MI.findRegisterDefOperand(X86::EFLAGS);
520 dbgs() <<
" Using EFLAGS defined by: ";
523 dbgs() <<
" Using live-in flags for BB:\n";
538 CondRegArray CondRegs = collectCondsInRegs(*TestMBB, TestPos);
545 Blocks.push_back(&
MBB);
551 bool FlagsKilled =
false;
562 for (
auto MII = (&UseMBB == &
MBB && !VisitedBlocks.
count(&UseMBB))
563 ? std::next(CopyI->getIterator())
572 if (&
MI == CopyI || &
MI == &CopyDefI) {
574 "Should only encounter these on the second pass over the "
581 if (
MI.findRegisterDefOperand(X86::EFLAGS)) {
610 auto JmpIt =
MI.getIterator();
612 JmpIs.push_back(&*JmpIt);
622 rewriteCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
624 rewriteFCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
626 rewriteSetCC(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
627 }
else if (
MI.getOpcode() == TargetOpcode::COPY) {
628 rewriteCopy(
MI, *FlagUse, CopyDefI);
631 assert(
MI.findRegisterDefOperand(X86::EFLAGS) &&
632 "Expected a def of EFLAGS for this instruction!");
643 rewriteArithmetic(*TestMBB, TestPos, TestLoc,
MI, *FlagUse,
659 if (SuccMBB->isLiveIn(X86::EFLAGS) &&
660 VisitedBlocks.
insert(SuccMBB).second) {
674 if (SuccMBB == TestMBB || !MDT->dominates(TestMBB, SuccMBB)) {
677 <<
"ERROR: Encountered use that is not dominated by our test "
678 "basic block! Rewriting this would require inserting PHI "
679 "nodes to track the flag state across the CFG.\n\nTest "
682 dbgs() <<
"Use block:\n";
686 "Cannot lower EFLAGS copy when original copy def "
687 "does not dominate all uses.");
690 Blocks.push_back(SuccMBB);
693 SuccMBB->removeLiveIn(X86::EFLAGS);
695 }
while (!Blocks.empty());
704 if (JmpI->getParent() == LastJmpMBB)
709 rewriteCondJmp(*TestMBB, TestPos, TestLoc, *JmpI, CondRegs);
719 if (
MI.getOpcode() == TargetOpcode::COPY &&
720 (
MI.getOperand(0).getReg() == X86::EFLAGS ||
721 MI.getOperand(1).getReg() == X86::EFLAGS)) {
733 CondRegArray X86FlagsCopyLoweringPass::collectCondsInRegs(
735 CondRegArray CondRegs = {};
742 MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg().isVirtual()) {
744 "A non-storing SETcc should always define a register!");
745 CondRegs[
Cond] =
MI.getOperand(0).getReg();
750 if (
MI.findRegisterDefOperand(X86::EFLAGS))
756 Register X86FlagsCopyLoweringPass::promoteCondToReg(
760 auto SetI =
BuildMI(TestMBB, TestPos, TestLoc,
768 std::pair<unsigned, bool> X86FlagsCopyLoweringPass::getCondOrInverseInReg(
771 unsigned &CondReg = CondRegs[
Cond];
773 if (!CondReg && !InvCondReg)
774 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
777 return {CondReg,
false};
779 return {InvCondReg,
true};
792 void X86FlagsCopyLoweringPass::rewriteArithmetic(
795 CondRegArray &CondRegs) {
805 case FlagArithMnemonic::ADCX:
806 case FlagArithMnemonic::RCL:
807 case FlagArithMnemonic::RCR:
809 case FlagArithMnemonic::SETB:
816 case FlagArithMnemonic::ADOX:
827 unsigned &CondReg = CondRegs[
Cond];
829 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
851 CondRegArray &CondRegs) {
856 std::tie(CondReg, Inverted) =
857 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
877 CondRegArray &CondRegs) {
882 std::tie(CondReg, Inverted) =
883 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
890 auto getFCMOVOpcode = [](
unsigned Opcode,
bool Inverted) {
893 case X86::CMOVBE_Fp32:
case X86::CMOVNBE_Fp32:
894 case X86::CMOVB_Fp32:
case X86::CMOVNB_Fp32:
895 case X86::CMOVE_Fp32:
case X86::CMOVNE_Fp32:
896 case X86::CMOVP_Fp32:
case X86::CMOVNP_Fp32:
897 return Inverted ? X86::CMOVE_Fp32 : X86::CMOVNE_Fp32;
898 case X86::CMOVBE_Fp64:
case X86::CMOVNBE_Fp64:
899 case X86::CMOVB_Fp64:
case X86::CMOVNB_Fp64:
900 case X86::CMOVE_Fp64:
case X86::CMOVNE_Fp64:
901 case X86::CMOVP_Fp64:
case X86::CMOVNP_Fp64:
902 return Inverted ? X86::CMOVE_Fp64 : X86::CMOVNE_Fp64;
903 case X86::CMOVBE_Fp80:
case X86::CMOVNBE_Fp80:
904 case X86::CMOVB_Fp80:
case X86::CMOVNB_Fp80:
905 case X86::CMOVE_Fp80:
case X86::CMOVNE_Fp80:
906 case X86::CMOVP_Fp80:
case X86::CMOVNP_Fp80:
907 return Inverted ? X86::CMOVE_Fp80 : X86::CMOVNE_Fp80;
917 void X86FlagsCopyLoweringPass::rewriteCondJmp(
924 std::tie(CondReg, Inverted) =
925 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
945 MI.eraseFromParent();
953 CondRegArray &CondRegs) {
958 unsigned &CondReg = CondRegs[
Cond];
960 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
966 "Cannot have a non-register defined operand to SETcc!");
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
FunctionPass * createX86FlagsCopyLoweringPass()
Return a pass that lowers EFLAGS copy pseudo instructions.
pred_iterator pred_begin()
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
This is an optimization pass for GlobalISel generic memory operations.
static X86::CondCode getCondFromFCMOV(unsigned Opcode)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
static X86::CondCode getCondFromBranch(const MCInst &MI, const MCInstrInfo &MCII)
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
void setIsKill(bool Val=true)
Reg
All possible values of the reg field in the ModR/M byte.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void setImm(int64_t immVal)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
MachineOperand * findRegisterUseOperand(Register Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
OutputIt copy(R &&Range, OutputIt Out)
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
CondCode getCondFromBranch(const MachineInstr &MI)
void copySuccessor(MachineBasicBlock *Orig, succ_iterator I)
Copy a successor (and any probability info) from original block to this block's.
unsigned const TargetRegisterInfo * TRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
INITIALIZE_PASS_BEGIN(X86FlagsCopyLoweringPass, DEBUG_TYPE, "X86 EFLAGS copy lowering", false, false) INITIALIZE_PASS_END(X86FlagsCopyLoweringPass
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
CondCode getCondFromCMov(const MachineInstr &MI)
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineOperand & getOperand(unsigned i) const
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
CondCode getCondFromSETCC(const MachineInstr &MI)
Represent the analysis usage information of a pass.
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
const HexagonInstrInfo * TII
#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX)
MachineOperand class - Representation of each machine instruction operand.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
STATISTIC(NumFunctions, "Total number of functions")
static FlagArithMnemonic getMnemonicFromOpcode(unsigned Opcode)
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
bool use_empty(Register RegNo) const
use_empty - Return true if there are no instructions using the specified register.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static MachineBasicBlock & splitBlock(MachineBasicBlock &MBB, MachineInstr &SplitI, const X86InstrInfo &TII)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
void clearKillFlags(Register Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Representation of each machine instruction.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
succ_iterator succ_begin()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Register getReg() const
getReg - Returns the register number.
iterator_range< pred_iterator > predecessors()
instr_iterator instr_begin()
static const HTTPClientCleanup Cleanup
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
instr_iterator instr_end()
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
MachineBasicBlock * getMBB() const
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< MachineOperand, 4 > Cond
iterator_range< succ_iterator > successors()
StringRef - Represent a constant reference to a string, i.e.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
self_iterator getIterator()
const MachineBasicBlock * getParent() const
unsigned const MachineRegisterInfo * MRI
Wrapper class representing virtual and physical registers.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Iterator for intrusive lists based on ilist_node.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=false)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
void setMBB(MachineBasicBlock *MBB)
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
auto reverse(ContainerTy &&C)
@ AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
FunctionPass class - This class is used to implement most global optimizations.
AnalysisUsage & addRequired()
#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC)
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
static M68k::CondCode GetOppositeBranchCondition(M68k::CondCode CC)