65#define PASS_KEY "x86-flags-copy-lowering"
66#define DEBUG_TYPE PASS_KEY
68STATISTIC(NumCopiesEliminated,
"Number of copies of EFLAGS eliminated");
69STATISTIC(NumSetCCsInserted,
"Number of setCC instructions inserted");
70STATISTIC(NumTestsInserted,
"Number of test instructions inserted");
71STATISTIC(NumAddsInserted,
"Number of adds instructions inserted");
76using CondRegArray = std::array<unsigned, X86::LAST_VALID_COND + 1>;
103 std::pair<unsigned, bool> getCondOrInverseInReg(
116 CondRegArray &CondRegs);
124 CondRegArray &CondRegs);
136 "X86 EFLAGS copy lowering",
false,
false)
141 return new X86FlagsCopyLoweringPass();
144char X86FlagsCopyLoweringPass::ID = 0;
146void X86FlagsCopyLoweringPass::getAnalysisUsage(
AnalysisUsage &AU)
const {
157enum class FlagArithMnemonic {
172 "by this instruction!");
174#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX) \
175 case X86::MNEMONIC##8##SUFFIX: \
176 case X86::MNEMONIC##16##SUFFIX: \
177 case X86::MNEMONIC##32##SUFFIX: \
178 case X86::MNEMONIC##64##SUFFIX:
180#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC) \
181 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr) \
182 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr_REV) \
183 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rm) \
184 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, mr) \
185 case X86::MNEMONIC##8ri: \
186 case X86::MNEMONIC##16ri8: \
187 case X86::MNEMONIC##32ri8: \
188 case X86::MNEMONIC##64ri8: \
189 case X86::MNEMONIC##16ri: \
190 case X86::MNEMONIC##32ri: \
191 case X86::MNEMONIC##64ri32: \
192 case X86::MNEMONIC##8mi: \
193 case X86::MNEMONIC##16mi8: \
194 case X86::MNEMONIC##32mi8: \
195 case X86::MNEMONIC##64mi8: \
196 case X86::MNEMONIC##16mi: \
197 case X86::MNEMONIC##32mi: \
198 case X86::MNEMONIC##64mi32: \
199 case X86::MNEMONIC##8i8: \
200 case X86::MNEMONIC##16i16: \
201 case X86::MNEMONIC##32i32: \
202 case X86::MNEMONIC##64i32:
205 return FlagArithMnemonic::ADC;
208 return FlagArithMnemonic::SBB;
210#undef LLVM_EXPAND_ADC_SBB_INSTR
215 return FlagArithMnemonic::RCL;
220 return FlagArithMnemonic::RCR;
222#undef LLVM_EXPAND_INSTR_SIZES
228 return FlagArithMnemonic::ADCX;
234 return FlagArithMnemonic::ADOX;
238 return FlagArithMnemonic::SETB;
248 "Split instruction must be in the split block!");
250 "Only designed to split a tail of branch instructions!");
252 "Must split on an actual jCC instruction!");
258 "Must split after an actual jCC instruction!");
260 "Must only have this one terminator prior to the split!");
271 assert(MI.isTerminator() &&
272 "Should only have spliced terminators!");
274 MI.operands(), [&](MachineOperand &MOp) {
275 return MOp.isMBB() && MOp.getMBB() == &UnsplitSucc;
292 if (IsEdgeSplit || *SI != &UnsplitSucc)
301 if (Succ != &UnsplitSucc)
306 "Failed to make the new block a successor!");
314 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
318 assert(OpMBB.
isMBB() &&
"Block operand to a PHI is not a block!");
323 if (!IsEdgeSplit || Succ != &UnsplitSucc) {
332 MI.addOperand(MF, OpV);
345 case X86::CMOVBE_Fp32:
case X86::CMOVBE_Fp64:
case X86::CMOVBE_Fp80:
347 case X86::CMOVB_Fp32:
case X86::CMOVB_Fp64:
case X86::CMOVB_Fp80:
349 case X86::CMOVE_Fp32:
case X86::CMOVE_Fp64:
case X86::CMOVE_Fp80:
351 case X86::CMOVNBE_Fp32:
case X86::CMOVNBE_Fp64:
case X86::CMOVNBE_Fp80:
353 case X86::CMOVNB_Fp32:
case X86::CMOVNB_Fp64:
case X86::CMOVNB_Fp80:
355 case X86::CMOVNE_Fp32:
case X86::CMOVNE_Fp64:
case X86::CMOVNE_Fp80:
357 case X86::CMOVNP_Fp32:
case X86::CMOVNP_Fp64:
case X86::CMOVNP_Fp80:
359 case X86::CMOVP_Fp32:
case X86::CMOVP_Fp64:
case X86::CMOVP_Fp80:
364bool X86FlagsCopyLoweringPass::runOnMachineFunction(
MachineFunction &MF) {
370 TII = Subtarget->getInstrInfo();
371 TRI = Subtarget->getRegisterInfo();
372 MDT = &getAnalysis<MachineDominatorTree>();
373 PromoteRC = &X86::GR8RegClass;
387 if (
MI.getOpcode() == TargetOpcode::COPY &&
388 MI.getOperand(0).getReg() == X86::EFLAGS)
396 "The input to the copy for EFLAGS should always be a register!");
398 if (CopyDefI.
getOpcode() != TargetOpcode::COPY) {
414 dbgs() <<
"ERROR: Encountered unexpected def of an eflags copy: ";
417 "Cannot lower EFLAGS copy unless it is defined in turn by a copy!");
423 CopyI->eraseFromParent();
426 ++NumCopiesEliminated;
430 assert(DOp.isDef() &&
"Expected register def!");
431 assert(DOp.getReg() == X86::EFLAGS &&
"Unexpected copy def register!");
459 return &
MI != CopyI &&
MI.findRegisterDefOperand(X86::EFLAGS);
464 assert(MDT->dominates(BeginMBB, EndMBB) &&
465 "Only support paths down the dominator tree!");
474 if (!Visited.
insert(PredMBB).second)
476 if (HasEFLAGSClobber(PredMBB->begin(), PredMBB->end()))
481 }
while (!Worklist.
empty());
486 !HasEFLAGSClobber(TestMBB->
begin(), TestPos)) {
493 return MDT->findNearestCommonDominator(LHS, RHS);
499 if (HasEFLAGSClobberPath(HoistMBB, TestMBB))
517 return MI.findRegisterDefOperand(X86::EFLAGS);
520 dbgs() <<
" Using EFLAGS defined by: ";
523 dbgs() <<
" Using live-in flags for BB:\n";
538 CondRegArray CondRegs = collectCondsInRegs(*TestMBB, TestPos);
551 bool FlagsKilled =
false;
562 for (
auto MII = (&UseMBB == &
MBB && !VisitedBlocks.
count(&UseMBB))
563 ? std::next(CopyI->getIterator())
572 if (&
MI == CopyI || &
MI == &CopyDefI) {
574 "Should only encounter these on the second pass over the "
581 if (
MI.findRegisterDefOperand(X86::EFLAGS)) {
610 auto JmpIt =
MI.getIterator();
612 JmpIs.push_back(&*JmpIt);
622 rewriteCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
624 rewriteFCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
626 rewriteSetCC(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
627 }
else if (
MI.getOpcode() == TargetOpcode::COPY) {
628 rewriteCopy(
MI, *FlagUse, CopyDefI);
631 assert(
MI.findRegisterDefOperand(X86::EFLAGS) &&
632 "Expected a def of EFLAGS for this instruction!");
643 rewriteArithmetic(*TestMBB, TestPos, TestLoc,
MI, *FlagUse,
659 if (SuccMBB->isLiveIn(X86::EFLAGS) &&
660 VisitedBlocks.
insert(SuccMBB).second) {
674 if (SuccMBB == TestMBB || !MDT->dominates(TestMBB, SuccMBB)) {
677 <<
"ERROR: Encountered use that is not dominated by our test "
678 "basic block! Rewriting this would require inserting PHI "
679 "nodes to track the flag state across the CFG.\n\nTest "
682 dbgs() <<
"Use block:\n";
686 "Cannot lower EFLAGS copy when original copy def "
687 "does not dominate all uses.");
690 Blocks.push_back(SuccMBB);
693 SuccMBB->removeLiveIn(X86::EFLAGS);
695 }
while (!
Blocks.empty());
704 if (JmpI->getParent() == LastJmpMBB)
709 rewriteCondJmp(*TestMBB, TestPos, TestLoc, *JmpI, CondRegs);
719 if (
MI.getOpcode() == TargetOpcode::COPY &&
720 (
MI.getOperand(0).getReg() == X86::EFLAGS ||
721 MI.getOperand(1).getReg() == X86::EFLAGS)) {
733CondRegArray X86FlagsCopyLoweringPass::collectCondsInRegs(
735 CondRegArray CondRegs = {};
742 MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg().isVirtual()) {
744 "A non-storing SETcc should always define a register!");
745 CondRegs[
Cond] =
MI.getOperand(0).getReg();
750 if (
MI.findRegisterDefOperand(X86::EFLAGS))
756Register X86FlagsCopyLoweringPass::promoteCondToReg(
760 auto SetI =
BuildMI(TestMBB, TestPos, TestLoc,
768std::pair<unsigned, bool> X86FlagsCopyLoweringPass::getCondOrInverseInReg(
771 unsigned &CondReg = CondRegs[
Cond];
773 if (!CondReg && !InvCondReg)
774 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
777 return {CondReg,
false};
779 return {InvCondReg,
true};
784 const DebugLoc &Loc,
unsigned Reg) {
792void X86FlagsCopyLoweringPass::rewriteArithmetic(
795 CondRegArray &CondRegs) {
804 case FlagArithMnemonic::ADC:
805 case FlagArithMnemonic::ADCX:
806 case FlagArithMnemonic::RCL:
807 case FlagArithMnemonic::RCR:
808 case FlagArithMnemonic::SBB:
809 case FlagArithMnemonic::SETB:
816 case FlagArithMnemonic::ADOX:
827 unsigned &CondReg = CondRegs[
Cond];
829 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
834 Register TmpReg =
MRI->createVirtualRegister(PromoteRC);
851 CondRegArray &CondRegs) {
856 std::tie(CondReg, Inverted) =
857 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
877 CondRegArray &CondRegs) {
882 std::tie(CondReg, Inverted) =
883 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
890 auto getFCMOVOpcode = [](
unsigned Opcode,
bool Inverted) {
893 case X86::CMOVBE_Fp32:
case X86::CMOVNBE_Fp32:
894 case X86::CMOVB_Fp32:
case X86::CMOVNB_Fp32:
895 case X86::CMOVE_Fp32:
case X86::CMOVNE_Fp32:
896 case X86::CMOVP_Fp32:
case X86::CMOVNP_Fp32:
897 return Inverted ? X86::CMOVE_Fp32 : X86::CMOVNE_Fp32;
898 case X86::CMOVBE_Fp64:
case X86::CMOVNBE_Fp64:
899 case X86::CMOVB_Fp64:
case X86::CMOVNB_Fp64:
900 case X86::CMOVE_Fp64:
case X86::CMOVNE_Fp64:
901 case X86::CMOVP_Fp64:
case X86::CMOVNP_Fp64:
902 return Inverted ? X86::CMOVE_Fp64 : X86::CMOVNE_Fp64;
903 case X86::CMOVBE_Fp80:
case X86::CMOVNBE_Fp80:
904 case X86::CMOVB_Fp80:
case X86::CMOVNB_Fp80:
905 case X86::CMOVE_Fp80:
case X86::CMOVNE_Fp80:
906 case X86::CMOVP_Fp80:
case X86::CMOVNP_Fp80:
907 return Inverted ? X86::CMOVE_Fp80 : X86::CMOVNE_Fp80;
917void X86FlagsCopyLoweringPass::rewriteCondJmp(
924 std::tie(CondReg, Inverted) =
925 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
943 MRI->replaceRegWith(
MI.getOperand(0).getReg(),
945 MI.eraseFromParent();
953 CondRegArray &CondRegs) {
958 unsigned &CondReg = CondRegs[
Cond];
960 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
966 "Cannot have a non-register defined operand to SETcc!");
970 MRI->clearKillFlags(OldReg);
971 MRI->replaceRegWith(OldReg, CondReg);
unsigned const MachineRegisterInfo * MRI
SmallVector< MachineOperand, 4 > Cond
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
static const HTTPClientCleanup Cleanup
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static void r1(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the SparseBitVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static X86::CondCode getCondFromFCMOV(unsigned Opcode)
#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX)
static MachineBasicBlock & splitBlock(MachineBasicBlock &MBB, MachineInstr &SplitI, const X86InstrInfo &TII)
static FlagArithMnemonic getMnemonicFromOpcode(unsigned Opcode)
#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
FunctionPass class - This class is used to implement most global optimizations.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
instr_iterator instr_begin()
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
succ_iterator succ_begin()
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
pred_iterator pred_begin()
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void copySuccessor(MachineBasicBlock *Orig, succ_iterator I)
Copy a successor (and any probability info) from original block to this block's.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand * findRegisterUseOperand(Register Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsKill(bool Val=true)
void setMBB(MachineBasicBlock *MBB)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Reg
All possible values of the reg field in the ModR/M byte.
CondCode getCondFromBranch(const MachineInstr &MI)
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
@ AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
CondCode getCondFromSETCC(const MachineInstr &MI)
CondCode getCondFromCMov(const MachineInstr &MI)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
FunctionPass * createX86FlagsCopyLoweringPass()
Return a pass that lowers EFLAGS copy pseudo instructions.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
OutputIt copy(R &&Range, OutputIt Out)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.