61#define PASS_KEY "x86-flags-copy-lowering"
62#define DEBUG_TYPE PASS_KEY
64STATISTIC(NumCopiesEliminated,
"Number of copies of EFLAGS eliminated");
65STATISTIC(NumSetCCsInserted,
"Number of setCC instructions inserted");
66STATISTIC(NumTestsInserted,
"Number of test instructions inserted");
67STATISTIC(NumAddsInserted,
"Number of adds instructions inserted");
72using CondRegArray = std::array<unsigned, X86::LAST_VALID_COND + 1>;
99 std::pair<unsigned, bool> getCondOrInverseInReg(
112 CondRegArray &CondRegs);
120 CondRegArray &CondRegs);
132 "X86 EFLAGS copy lowering",
false,
false)
137 return new X86FlagsCopyLoweringPass();
140char X86FlagsCopyLoweringPass::ID = 0;
142void X86FlagsCopyLoweringPass::getAnalysisUsage(
AnalysisUsage &AU)
const {
153enum class FlagArithMnemonic {
166 "by this instruction!");
172#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX) \
173 CASE_ND(MNEMONIC##8##SUFFIX) \
174 CASE_ND(MNEMONIC##16##SUFFIX) \
175 CASE_ND(MNEMONIC##32##SUFFIX) \
176 CASE_ND(MNEMONIC##64##SUFFIX)
178#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC) \
179 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr) \
180 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rm) \
181 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, mr) \
182 CASE_ND(MNEMONIC##8ri) \
183 CASE_ND(MNEMONIC##16ri8) \
184 CASE_ND(MNEMONIC##32ri8) \
185 CASE_ND(MNEMONIC##64ri8) \
186 CASE_ND(MNEMONIC##16ri) \
187 CASE_ND(MNEMONIC##32ri) \
188 CASE_ND(MNEMONIC##64ri32) \
189 CASE_ND(MNEMONIC##8mi) \
190 CASE_ND(MNEMONIC##16mi8) \
191 CASE_ND(MNEMONIC##32mi8) \
192 CASE_ND(MNEMONIC##64mi8) \
193 CASE_ND(MNEMONIC##16mi) \
194 CASE_ND(MNEMONIC##32mi) \
195 CASE_ND(MNEMONIC##64mi32) \
196 case X86::MNEMONIC##8i8: \
197 case X86::MNEMONIC##16i16: \
198 case X86::MNEMONIC##32i32: \
199 case X86::MNEMONIC##64i32:
202 return FlagArithMnemonic::ADC;
205 return FlagArithMnemonic::SBB;
207#undef LLVM_EXPAND_ADC_SBB_INSTR
212 return FlagArithMnemonic::RCL;
217 return FlagArithMnemonic::RCR;
219#undef LLVM_EXPAND_INSTR_SIZES
224 return FlagArithMnemonic::SETB;
234 "Split instruction must be in the split block!");
236 "Only designed to split a tail of branch instructions!");
238 "Must split on an actual jCC instruction!");
244 "Must split after an actual jCC instruction!");
246 "Must only have this one terminator prior to the split!");
257 assert(MI.isTerminator() &&
258 "Should only have spliced terminators!");
260 MI.operands(), [&](MachineOperand &MOp) {
261 return MOp.isMBB() && MOp.getMBB() == &UnsplitSucc;
278 if (IsEdgeSplit || *SI != &UnsplitSucc)
287 if (Succ != &UnsplitSucc)
292 "Failed to make the new block a successor!");
300 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
304 assert(OpMBB.
isMBB() &&
"Block operand to a PHI is not a block!");
309 if (!IsEdgeSplit || Succ != &UnsplitSucc) {
318 MI.addOperand(MF, OpV);
331 case X86::CMOVBE_Fp32:
case X86::CMOVBE_Fp64:
case X86::CMOVBE_Fp80:
333 case X86::CMOVB_Fp32:
case X86::CMOVB_Fp64:
case X86::CMOVB_Fp80:
335 case X86::CMOVE_Fp32:
case X86::CMOVE_Fp64:
case X86::CMOVE_Fp80:
337 case X86::CMOVNBE_Fp32:
case X86::CMOVNBE_Fp64:
case X86::CMOVNBE_Fp80:
339 case X86::CMOVNB_Fp32:
case X86::CMOVNB_Fp64:
case X86::CMOVNB_Fp80:
341 case X86::CMOVNE_Fp32:
case X86::CMOVNE_Fp64:
case X86::CMOVNE_Fp80:
343 case X86::CMOVNP_Fp32:
case X86::CMOVNP_Fp64:
case X86::CMOVNP_Fp80:
345 case X86::CMOVP_Fp32:
case X86::CMOVP_Fp64:
case X86::CMOVP_Fp80:
350bool X86FlagsCopyLoweringPass::runOnMachineFunction(
MachineFunction &MF) {
356 TII = Subtarget->getInstrInfo();
357 TRI = Subtarget->getRegisterInfo();
358 MDT = &getAnalysis<MachineDominatorTree>();
359 PromoteRC = &X86::GR8RegClass;
373 if (
MI.getOpcode() == TargetOpcode::COPY &&
374 MI.getOperand(0).getReg() == X86::EFLAGS)
382 "The input to the copy for EFLAGS should always be a register!");
384 if (CopyDefI.
getOpcode() != TargetOpcode::COPY) {
400 dbgs() <<
"ERROR: Encountered unexpected def of an eflags copy: ";
403 "Cannot lower EFLAGS copy unless it is defined in turn by a copy!");
409 CopyI->eraseFromParent();
412 ++NumCopiesEliminated;
416 assert(DOp.isDef() &&
"Expected register def!");
417 assert(DOp.getReg() == X86::EFLAGS &&
"Unexpected copy def register!");
445 return &
MI != CopyI &&
MI.findRegisterDefOperand(X86::EFLAGS);
450 assert(MDT->dominates(BeginMBB, EndMBB) &&
451 "Only support paths down the dominator tree!");
460 if (!Visited.
insert(PredMBB).second)
462 if (HasEFLAGSClobber(PredMBB->begin(), PredMBB->end()))
467 }
while (!Worklist.
empty());
472 !HasEFLAGSClobber(TestMBB->
begin(), TestPos)) {
479 return MDT->findNearestCommonDominator(LHS, RHS);
485 if (HasEFLAGSClobberPath(HoistMBB, TestMBB))
503 return MI.findRegisterDefOperand(X86::EFLAGS);
506 dbgs() <<
" Using EFLAGS defined by: ";
509 dbgs() <<
" Using live-in flags for BB:\n";
524 CondRegArray CondRegs = collectCondsInRegs(*TestMBB, TestPos);
537 bool FlagsKilled =
false;
548 for (
auto MII = (&UseMBB == &
MBB && !VisitedBlocks.
count(&UseMBB))
549 ? std::next(CopyI->getIterator())
558 if (&
MI == CopyI || &
MI == &CopyDefI) {
560 "Should only encounter these on the second pass over the "
567 if (
MI.findRegisterDefOperand(X86::EFLAGS)) {
596 auto JmpIt =
MI.getIterator();
598 JmpIs.push_back(&*JmpIt);
609 rewriteCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
611 rewriteFCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
613 rewriteSetCC(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
614 }
else if (
MI.getOpcode() == TargetOpcode::COPY) {
615 rewriteCopy(
MI, *FlagUse, CopyDefI);
618 assert(
MI.findRegisterDefOperand(X86::EFLAGS) &&
619 "Expected a def of EFLAGS for this instruction!");
630 rewriteArithmetic(*TestMBB, TestPos, TestLoc,
MI, *FlagUse,
646 if (SuccMBB->isLiveIn(X86::EFLAGS) &&
647 VisitedBlocks.
insert(SuccMBB).second) {
661 if (SuccMBB == TestMBB || !MDT->dominates(TestMBB, SuccMBB)) {
664 <<
"ERROR: Encountered use that is not dominated by our test "
665 "basic block! Rewriting this would require inserting PHI "
666 "nodes to track the flag state across the CFG.\n\nTest "
669 dbgs() <<
"Use block:\n";
673 "Cannot lower EFLAGS copy when original copy def "
674 "does not dominate all uses.");
677 Blocks.push_back(SuccMBB);
680 SuccMBB->removeLiveIn(X86::EFLAGS);
682 }
while (!
Blocks.empty());
691 if (JmpI->getParent() == LastJmpMBB)
696 rewriteCondJmp(*TestMBB, TestPos, TestLoc, *JmpI, CondRegs);
706 if (
MI.getOpcode() == TargetOpcode::COPY &&
707 (
MI.getOperand(0).getReg() == X86::EFLAGS ||
708 MI.getOperand(1).getReg() == X86::EFLAGS)) {
720CondRegArray X86FlagsCopyLoweringPass::collectCondsInRegs(
722 CondRegArray CondRegs = {};
729 MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg().isVirtual()) {
731 "A non-storing SETcc should always define a register!");
732 CondRegs[
Cond] =
MI.getOperand(0).getReg();
737 if (
MI.findRegisterDefOperand(X86::EFLAGS))
743Register X86FlagsCopyLoweringPass::promoteCondToReg(
747 auto SetI =
BuildMI(TestMBB, TestPos, TestLoc,
755std::pair<unsigned, bool> X86FlagsCopyLoweringPass::getCondOrInverseInReg(
758 unsigned &CondReg = CondRegs[
Cond];
760 if (!CondReg && !InvCondReg)
761 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
764 return {CondReg,
false};
766 return {InvCondReg,
true};
771 const DebugLoc &Loc,
unsigned Reg) {
779void X86FlagsCopyLoweringPass::rewriteArithmetic(
782 CondRegArray &CondRegs) {
791 case FlagArithMnemonic::ADC:
792 case FlagArithMnemonic::RCL:
793 case FlagArithMnemonic::RCR:
794 case FlagArithMnemonic::SBB:
795 case FlagArithMnemonic::SETB:
806 unsigned &CondReg = CondRegs[
Cond];
808 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
813 Register TmpReg =
MRI->createVirtualRegister(PromoteRC);
816 TII->get(Subtarget->hasNDD() ? X86::ADD8ri_ND : X86::ADD8ri))
831 CondRegArray &CondRegs) {
838 std::tie(CondReg, Inverted) =
839 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
859 CondRegArray &CondRegs) {
864 std::tie(CondReg, Inverted) =
865 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
872 auto getFCMOVOpcode = [](
unsigned Opcode,
bool Inverted) {
875 case X86::CMOVBE_Fp32:
case X86::CMOVNBE_Fp32:
876 case X86::CMOVB_Fp32:
case X86::CMOVNB_Fp32:
877 case X86::CMOVE_Fp32:
case X86::CMOVNE_Fp32:
878 case X86::CMOVP_Fp32:
case X86::CMOVNP_Fp32:
879 return Inverted ? X86::CMOVE_Fp32 : X86::CMOVNE_Fp32;
880 case X86::CMOVBE_Fp64:
case X86::CMOVNBE_Fp64:
881 case X86::CMOVB_Fp64:
case X86::CMOVNB_Fp64:
882 case X86::CMOVE_Fp64:
case X86::CMOVNE_Fp64:
883 case X86::CMOVP_Fp64:
case X86::CMOVNP_Fp64:
884 return Inverted ? X86::CMOVE_Fp64 : X86::CMOVNE_Fp64;
885 case X86::CMOVBE_Fp80:
case X86::CMOVNBE_Fp80:
886 case X86::CMOVB_Fp80:
case X86::CMOVNB_Fp80:
887 case X86::CMOVE_Fp80:
case X86::CMOVNE_Fp80:
888 case X86::CMOVP_Fp80:
case X86::CMOVNP_Fp80:
889 return Inverted ? X86::CMOVE_Fp80 : X86::CMOVNE_Fp80;
899void X86FlagsCopyLoweringPass::rewriteCondJmp(
906 std::tie(CondReg, Inverted) =
907 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
925 MRI->replaceRegWith(
MI.getOperand(0).getReg(),
927 MI.eraseFromParent();
935 CondRegArray &CondRegs) {
940 unsigned &CondReg = CondRegs[
Cond];
942 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
948 "Cannot have a non-register defined operand to SETcc!");
952 MRI->clearKillFlags(OldReg);
953 MRI->replaceRegWith(OldReg, CondReg);
unsigned const MachineRegisterInfo * MRI
DenseMap< Block *, BlockRelaxAux > Blocks
static const HTTPClientCleanup Cleanup
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > & Cond
static void r1(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static X86::CondCode getCondFromFCMOV(unsigned Opcode)
#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX)
static MachineBasicBlock & splitBlock(MachineBasicBlock &MBB, MachineInstr &SplitI, const X86InstrInfo &TII)
static FlagArithMnemonic getMnemonicFromOpcode(unsigned Opcode)
#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
FunctionPass class - This class is used to implement most global optimizations.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
instr_iterator instr_begin()
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
succ_iterator succ_begin()
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
void copySuccessor(const MachineBasicBlock *Orig, succ_iterator I)
Copy a successor (and any probability info) from original block to this block's.
pred_iterator pred_begin()
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand * findRegisterUseOperand(Register Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsKill(bool Val=true)
void setMBB(MachineBasicBlock *MBB)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ X86
Windows x64, Windows Itanium (IA-64)
Reg
All possible values of the reg field in the ModR/M byte.
CondCode getCondFromBranch(const MachineInstr &MI)
CondCode getCondFromCFCMov(const MachineInstr &MI)
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
CondCode getCondFromSETCC(const MachineInstr &MI)
CondCode getCondFromCMov(const MachineInstr &MI)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
FunctionPass * createX86FlagsCopyLoweringPass()
Return a pass that lowers EFLAGS copy pseudo instructions.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
OutputIt copy(R &&Range, OutputIt Out)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.