26#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME \
27 "RISC-V atomic pseudo instruction expansion pass"
53 bool IsMasked,
int Width,
73char RISCVExpandAtomicPseudo::ID = 0;
75bool RISCVExpandAtomicPseudo::runOnMachineFunction(
MachineFunction &MF) {
77 TII = STI->getInstrInfo();
89 assert(OldSize >= NewSize);
113 switch (
MBBI->getOpcode()) {
114 case RISCV::PseudoAtomicLoadNand32:
117 case RISCV::PseudoAtomicLoadNand64:
120 case RISCV::PseudoMaskedAtomicSwap32:
123 case RISCV::PseudoMaskedAtomicLoadAdd32:
125 case RISCV::PseudoMaskedAtomicLoadSub32:
127 case RISCV::PseudoMaskedAtomicLoadNand32:
130 case RISCV::PseudoMaskedAtomicLoadMax32:
133 case RISCV::PseudoMaskedAtomicLoadMin32:
136 case RISCV::PseudoMaskedAtomicLoadUMax32:
139 case RISCV::PseudoMaskedAtomicLoadUMin32:
142 case RISCV::PseudoCmpXchg32:
143 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 32, NextMBBI);
144 case RISCV::PseudoCmpXchg64:
145 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 64, NextMBBI);
146 case RISCV::PseudoMaskedCmpXchg32:
147 return expandAtomicCmpXchg(
MBB,
MBBI,
true, 32, NextMBBI);
158 case AtomicOrdering::Monotonic:
160 case AtomicOrdering::Acquire:
161 if (Subtarget->hasStdExtZtso())
163 return RISCV::LR_W_AQ;
164 case AtomicOrdering::Release:
166 case AtomicOrdering::AcquireRelease:
167 if (Subtarget->hasStdExtZtso())
169 return RISCV::LR_W_AQ;
170 case AtomicOrdering::SequentiallyConsistent:
171 return RISCV::LR_W_AQ_RL;
180 case AtomicOrdering::Monotonic:
182 case AtomicOrdering::Acquire:
184 case AtomicOrdering::Release:
185 if (Subtarget->hasStdExtZtso())
187 return RISCV::SC_W_RL;
188 case AtomicOrdering::AcquireRelease:
189 if (Subtarget->hasStdExtZtso())
191 return RISCV::SC_W_RL;
192 case AtomicOrdering::SequentiallyConsistent:
193 return RISCV::SC_W_RL;
202 case AtomicOrdering::Monotonic:
204 case AtomicOrdering::Acquire:
205 if (Subtarget->hasStdExtZtso())
207 return RISCV::LR_D_AQ;
208 case AtomicOrdering::Release:
210 case AtomicOrdering::AcquireRelease:
211 if (Subtarget->hasStdExtZtso())
213 return RISCV::LR_D_AQ;
214 case AtomicOrdering::SequentiallyConsistent:
215 return RISCV::LR_D_AQ_RL;
224 case AtomicOrdering::Monotonic:
226 case AtomicOrdering::Acquire:
228 case AtomicOrdering::Release:
229 if (Subtarget->hasStdExtZtso())
231 return RISCV::SC_D_RL;
232 case AtomicOrdering::AcquireRelease:
233 if (Subtarget->hasStdExtZtso())
235 return RISCV::SC_D_RL;
236 case AtomicOrdering::SequentiallyConsistent:
237 return RISCV::SC_D_RL;
244 return getLRForRMW32(Ordering, Subtarget);
246 return getLRForRMW64(Ordering, Subtarget);
253 return getSCForRMW32(Ordering, Subtarget);
255 return getSCForRMW64(Ordering, Subtarget);
266 Register ScratchReg =
MI.getOperand(1).getReg();
277 BuildMI(LoopMBB,
DL,
TII->get(getLRForRMW(Ordering, Width, STI)), DestReg)
291 BuildMI(LoopMBB,
DL,
TII->get(getSCForRMW(Ordering, Width, STI)), ScratchReg)
304 assert(OldValReg != ScratchReg &&
"OldValReg and ScratchReg must be unique");
305 assert(OldValReg != MaskReg &&
"OldValReg and MaskReg must be unique");
306 assert(ScratchReg != MaskReg &&
"ScratchReg and MaskReg must be unique");
329 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
331 Register ScratchReg =
MI.getOperand(1).getReg();
346 BuildMI(LoopMBB,
DL,
TII->get(getLRForRMW32(Ordering, STI)), DestReg)
376 insertMaskedMerge(
TII,
DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
379 BuildMI(LoopMBB,
DL,
TII->get(getSCForRMW32(Ordering, STI)), ScratchReg)
388bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
411 doAtomicBinOpExpansion(
TII,
MI,
DL, &
MBB, LoopMBB, DoneMBB, BinOp, Width,
414 doMaskedAtomicBinOpExpansion(
TII,
MI,
DL, &
MBB, LoopMBB, DoneMBB, BinOp,
418 MI.eraseFromParent();
438bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
442 assert(IsMasked ==
true &&
443 "Should only need to expand masked atomic max/min");
444 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
456 MF->
insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
457 MF->
insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
458 MF->
insert(++LoopTailMBB->getIterator(), DoneMBB);
461 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
462 LoopHeadMBB->addSuccessor(LoopTailMBB);
463 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
464 LoopTailMBB->addSuccessor(LoopHeadMBB);
465 LoopTailMBB->addSuccessor(DoneMBB);
471 Register Scratch1Reg =
MI.getOperand(1).getReg();
472 Register Scratch2Reg =
MI.getOperand(2).getReg();
487 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW32(Ordering, STI)), DestReg)
489 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::AND), Scratch2Reg)
492 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::ADDI), Scratch1Reg)
500 insertSext(
TII,
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
508 insertSext(
TII,
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
533 insertMaskedMerge(
TII,
DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
534 MaskReg, Scratch1Reg);
539 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW32(Ordering, STI)), Scratch1Reg)
548 MI.eraseFromParent();
583 if (
MBBI == E ||
MBBI->getOpcode() != RISCV::AND)
587 if (!(ANDOp1 == DestReg && ANDOp2 == MaskReg) &&
588 !(ANDOp1 == MaskReg && ANDOp2 == DestReg))
591 DestReg =
MBBI->getOperand(0).getReg();
597 if (
MBBI == E ||
MBBI->getOpcode() != RISCV::BNE)
601 if (!(BNEOp0 == DestReg && BNEOp1 == CmpValReg) &&
602 !(BNEOp0 == CmpValReg && BNEOp1 == DestReg))
607 if (BNEOp0 == DestReg && !
MBBI->getOperand(0).isKill())
609 if (BNEOp1 == DestReg && !
MBBI->getOperand(1).isKill())
614 LoopHeadBNETarget =
MBBI->getOperand(2).getMBB();
620 for (
auto *
MI : ToErase)
621 MI->eraseFromParent();
625bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
636 Register ScratchReg =
MI.getOperand(1).getReg();
638 Register CmpValReg =
MI.getOperand(3).getReg();
639 Register NewValReg =
MI.getOperand(4).getReg();
643 tryToFoldBNEOnCmpXchgResult(
MBB, std::next(
MBBI), DestReg, CmpValReg, MaskReg,
648 MF->
insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
649 MF->
insert(++LoopTailMBB->getIterator(), DoneMBB);
652 LoopHeadMBB->addSuccessor(LoopTailMBB);
653 LoopHeadMBB->addSuccessor(LoopHeadBNETarget);
654 LoopTailMBB->addSuccessor(DoneMBB);
655 LoopTailMBB->addSuccessor(LoopHeadMBB);
667 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width, STI)),
673 .
addMBB(LoopHeadBNETarget);
677 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width, STI)),
691 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width, STI)),
694 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::AND), ScratchReg)
700 .
addMBB(LoopHeadBNETarget);
708 insertMaskedMerge(
TII,
DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
709 MaskReg, ScratchReg);
710 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width, STI)),
721 MI.eraseFromParent();
739 return new RISCVExpandAtomicPseudo();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static unsigned getInstSizeInBytes(const MachineInstr &MI, const SystemZInstrInfo *TII)
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
FunctionPass class - This class is used to implement most global optimizations.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createRISCVExpandAtomicPseudoPass()
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
void initializeRISCVExpandAtomicPseudoPass(PassRegistry &)
AtomicOrdering
Atomic ordering for LLVM's memory model.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().