Go to the documentation of this file.
30 #define DEBUG_TYPE "x86-pseudo"
31 #define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"
60 return "X86 pseudo instruction expansion pass";
79 void ExpandVastartSaveXmmRegs(
90 void X86ExpandPseudo::ExpandICallBranchFunnel(
99 std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;
104 auto CmpTarget = [&](
unsigned Target) {
105 if (Selector.
isReg())
119 auto CreateMBB = [&]() {
130 auto *ElseMBB = CreateMBB();
131 MF->
insert(InsPt, ElseMBB);
136 auto EmitCondJumpTarget = [&](
unsigned CC,
unsigned Target) {
137 auto *ThenMBB = CreateMBB();
138 TargetMBBs.push_back({ThenMBB,
Target});
139 EmitCondJump(
CC, ThenMBB);
142 auto EmitTailCall = [&](
unsigned Target) {
149 if (NumTargets == 1) {
154 if (NumTargets == 2) {
161 if (NumTargets < 6) {
169 auto *ThenMBB = CreateMBB();
173 EmitBranchFunnel(
FirstTarget + (NumTargets / 2) + 1,
174 NumTargets - (NumTargets / 2) - 1);
176 MF->
insert(InsPt, ThenMBB);
183 for (
auto P : TargetMBBs) {
188 JTMBB->
erase(JTInst);
198 assert((
MI.getOperand(1).isGlobal() ||
MI.getOperand(1).isReg()) &&
199 "invalid operand for regular call");
201 if (
MI.getOpcode() == X86::CALL64m_RVMARKER)
203 else if (
MI.getOpcode() == X86::CALL64r_RVMARKER)
205 else if (
MI.getOpcode() == X86::CALL64pcrel32_RVMARKER)
206 Opc = X86::CALL64pcrel32;
211 bool RAXImplicitDead =
false;
215 if (
Op.isReg() &&
Op.isImplicit() &&
Op.isDead() &&
219 RAXImplicitDead =
true;
229 auto TargetReg = STI->getTargetTriple().isOSWindows() ? X86::RCX : X86::RDI;
234 if (
MI.shouldUpdateCallSiteInfo())
249 MI.eraseFromParent();
254 if (
TM.getTargetTriple().isOSDarwin())
265 unsigned Opcode =
MI.getOpcode();
270 case X86::TCRETURNdi:
271 case X86::TCRETURNdicc:
272 case X86::TCRETURNri:
273 case X86::TCRETURNmi:
274 case X86::TCRETURNdi64:
275 case X86::TCRETURNdi64cc:
276 case X86::TCRETURNri64:
277 case X86::TCRETURNmi64: {
278 bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
282 assert(StackAdjust.
isImm() &&
"Expecting immediate value.");
285 int StackAdj = StackAdjust.
getImm();
286 int MaxTCDelta = X86FI->getTCReturnAddrDelta();
288 assert(MaxTCDelta <= 0 &&
"MaxTCDelta should never be positive");
291 Offset = StackAdj - MaxTCDelta;
292 assert(
Offset >= 0 &&
"Offset should never be negative");
294 if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {
295 assert(
Offset == 0 &&
"Conditional tail call cannot adjust the stack.");
305 bool IsWin64 = STI->isTargetWin64();
306 if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
307 Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
310 case X86::TCRETURNdi:
313 case X86::TCRETURNdicc:
314 Op = X86::TAILJMPd_CC;
316 case X86::TCRETURNdi64cc:
318 "Conditional tail calls confuse "
319 "the Win64 unwinder.");
320 Op = X86::TAILJMPd64_CC;
325 Op = X86::TAILJMPd64;
337 if (
Op == X86::TAILJMPd_CC ||
Op == X86::TAILJMPd64_CC) {
341 }
else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
342 unsigned Op = (Opcode == X86::TCRETURNmi)
344 : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
348 }
else if (Opcode == X86::TCRETURNri64) {
351 TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
364 if (
MBBI->isCandidateForCallSiteEntry())
373 case X86::EH_RETURN64: {
375 assert(DestAddr.
isReg() &&
"Offset should be in register!");
376 const bool Uses64BitFramePtr =
377 STI->isTarget64BitLP64() || STI->isTargetNaCl64();
380 TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr)
387 int64_t StackAdj =
MBBI->getOperand(0).getImm();
388 X86FL->emitSPUpdate(
MBB,
MBBI,
DL, StackAdj,
true);
390 unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32;
392 if (STI->is64Bit() && STI->hasUINTR() &&
401 int64_t StackAdj =
MBBI->getOperand(0).getImm();
405 TII->get(STI->is64Bit() ? X86::RET64 : X86::RET32));
406 }
else if (isUInt<16>(StackAdj)) {
408 TII->get(STI->is64Bit() ? X86::RETI64 : X86::RETI32))
412 "shouldn't need to do this for x86_64 targets!");
416 X86FL->emitSPUpdate(
MBB,
MBBI,
DL, StackAdj,
true);
420 for (
unsigned I = 1,
E =
MBBI->getNumOperands();
I !=
E; ++
I)
425 case X86::LCMPXCHG16B_SAVE_RBX: {
443 for (
unsigned Idx = 1; Idx < 6; ++Idx)
461 case X86::MASKPAIR16LOAD: {
463 assert(Disp >= 0 && Disp <= INT32_MAX - 2 &&
"Unexpected displacement");
465 bool DstIsDead =
MBBI->getOperand(0).isDead();
488 MIBLo.setMemRefs(MMOLo);
489 MIBHi.setMemRefs(MMOHi);
495 case X86::MASKPAIR16STORE: {
497 assert(Disp >= 0 && Disp <= INT32_MAX - 2 &&
"Unexpected displacement");
507 MIBLo.add(
MBBI->getOperand(
i));
509 MIBHi.addImm(Disp + 2);
511 MIBHi.add(
MBBI->getOperand(
i));
522 MIBLo.setMemRefs(MMOLo);
523 MIBHi.setMemRefs(MMOHi);
529 case X86::MWAITX_SAVE_RBX: {
549 case TargetOpcode::ICALL_BRANCH_FUNNEL:
550 ExpandICallBranchFunnel(&
MBB,
MBBI);
552 case X86::PLDTILECFGV: {
553 MI.setDesc(
TII->get(X86::LDTILECFG));
556 case X86::PTILELOADDV:
557 case X86::PTILELOADDT1V: {
558 for (
unsigned i = 2;
i > 0; --
i)
561 Opcode == X86::PTILELOADDV ? X86::TILELOADD : X86::TILELOADDT1;
562 MI.setDesc(
TII->get(Opc));
569 case X86::PTDPBF16PSV:
570 case X86::PTDPFP16PSV: {
571 MI.untieRegOperand(4);
572 for (
unsigned i = 3;
i > 0; --
i)
576 case X86::PTDPBSSDV: Opc = X86::TDPBSSD;
break;
577 case X86::PTDPBSUDV: Opc = X86::TDPBSUD;
break;
578 case X86::PTDPBUSDV: Opc = X86::TDPBUSD;
break;
579 case X86::PTDPBUUDV: Opc = X86::TDPBUUD;
break;
580 case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS;
break;
581 case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS;
break;
584 MI.setDesc(
TII->get(Opc));
585 MI.tieOperands(0, 1);
588 case X86::PTILESTOREDV: {
589 for (
int i = 1;
i >= 0; --
i)
591 MI.setDesc(
TII->get(X86::TILESTORED));
594 case X86::PTILEZEROV: {
595 for (
int i = 2;
i > 0; --
i)
597 MI.setDesc(
TII->get(X86::TILEZERO));
600 case X86::CALL64pcrel32_RVMARKER:
601 case X86::CALL64r_RVMARKER:
602 case X86::CALL64m_RVMARKER:
603 expandCALL_RVMARKER(
MBB,
MBBI);
623 void X86ExpandPseudo::ExpandVastartSaveXmmRegs(
630 const DebugLoc &
DL = VAStartPseudoInstr->getDebugLoc();
631 Register CountReg = VAStartPseudoInstr->getOperand(0).getReg();
637 LiveRegs.addLiveIns(*EntryBlk);
639 if (
MI.getOpcode() == VAStartPseudoInstr->getOpcode())
642 LiveRegs.stepForward(
MI, Clobbers);
652 Func->insert(EntryBlkIter, GuardedRegsBlk);
653 Func->insert(EntryBlkIter, TailBlk);
661 uint64_t FrameOffset = VAStartPseudoInstr->getOperand(4).getImm();
662 uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(6).getImm();
665 unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
668 for (int64_t OpndIdx = 7, RegIdx = 0;
669 OpndIdx < VAStartPseudoInstr->getNumOperands() - 1;
670 OpndIdx++, RegIdx++) {
671 auto NewMI =
BuildMI(GuardedRegsBlk,
DL,
TII->get(MOVOpc));
674 NewMI.addImm(FrameOffset + VarArgsRegsOffset + RegIdx * 16);
676 NewMI.add(VAStartPseudoInstr->getOperand(
i + 1));
678 NewMI.addReg(VAStartPseudoInstr->getOperand(OpndIdx).getReg());
679 assert(VAStartPseudoInstr->getOperand(OpndIdx).getReg().isPhysical());
687 if (!STI->isCallingConvWin64(
Func->getFunction().getCallingConv())) {
703 VAStartPseudoInstr->eraseFromParent();
722 bool X86ExpandPseudo::ExpandPseudosWhichAffectControlFlow(
MachineFunction &MF) {
728 ExpandVastartSaveXmmRegs(&(MF.
front()), Instr);
738 TII = STI->getInstrInfo();
739 TRI = STI->getRegisterInfo();
741 X86FL = STI->getFrameLowering();
743 bool Modified = ExpandPseudosWhichAffectControlFlow(MF);
752 return new X86ExpandPseudo();
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
const GlobalValue * getGlobal() const
const MachineInstrBuilder & add(const MachineOperand &MO) const
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Target - Wrapper for Target specific information.
void setIsKill(bool Val=true)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Reg
All possible values of the reg field in the ModR/M byte.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
A set of physical registers with utility functions to track liveness when walking backward/forward th...
void moveCallSiteInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
A description of a memory reference used in the backend.
void insert(iterator MBBI, MachineBasicBlock *MBB)
unsigned getDeadRegState(bool B)
@ C
The default llvm calling convention, compatible with C.
Properties which a MachineFunction may have at a given point in time.
void addLiveIns(MachineBasicBlock &MBB, const LivePhysRegs &LiveRegs)
Adds registers contained in LiveRegs to the block live-in list of MBB.
int64_t getOffset() const
Return the offset from the symbol in this operand.
unsigned const TargetRegisterInfo * TRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
BasicBlockListType::iterator iterator
LLVM Basic Block Representation.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
const MachineBasicBlock & front() const
TargetInstrInfo - Interface to description of machine instruction set.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI)
Copy implicit register operands from specified instruction to this instruction.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineOperand & getOperand(unsigned i) const
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
Represent the analysis usage information of a pass.
INITIALIZE_PASS(X86ExpandPseudo, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME, false, false) void X86ExpandPseudo
const HexagonInstrInfo * TII
MachineOperand class - Representation of each machine instruction operand.
MachineFunctionProperties & set(Property P)
@ IRET
Return from interrupt. Operand 0 is the number of bytes to pop.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
char & MachineLoopInfoID
MachineLoopInfo - This pass is a loop analysis pass.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
unsigned getTargetFlags() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Representation of each machine instruction.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
print Print MemDeps of function
FunctionPass * createX86ExpandPseudoPass()
Return a Machine IR pass that expands X86-specific pseudo instructions into a sequence of actual inst...
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Register getReg() const
getReg - Returns the register number.
static Target * FirstTarget
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
StringRef - Represent a constant reference to a string, i.e.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
AnalysisUsage & addPreservedID(const void *ID)
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
self_iterator getIterator()
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Wrapper class representing virtual and physical registers.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Iterator for intrusive lists based on ilist_node.
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
unsigned getKillRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
#define X86_EXPAND_PSEUDO_NAME
const char * getSymbolName() const
unsigned getNumOperands() const
Retuns the total number of operands.
char & MachineDominatorsID
MachineDominators - This pass is a machine dominators analysis pass.
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
static bool isMem(const MachineInstr &MI, unsigned Op)
void setCFIType(MachineFunction &MF, uint32_t Type)
Set the CFI type for the instruction.
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
@ AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
const char LLVMTargetMachineRef TM
FunctionPass class - This class is used to implement most global optimizations.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
CodeModel::Model getCodeModel() const
Returns the code model.
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
@ Define
Register definition.
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.