26#define RISCV_EXPAND_PSEUDO_ATOMICS_NAME \
27 "RISC-V Pseudo Instruction Expansion - Atomics"
37 bool IsMasked,
int Width,
93 return RISCVExpandPseudoAtomicsImpl().run(MF);
103bool RISCVExpandPseudoAtomicsImpl::expandMI(
109 switch (
MBBI->getOpcode()) {
110 case RISCV::PseudoAtomicSwap32:
113 case RISCV::PseudoAtomicSwap64:
116 case RISCV::PseudoAtomicLoadAdd32:
119 case RISCV::PseudoAtomicLoadAdd64:
122 case RISCV::PseudoAtomicLoadSub32:
125 case RISCV::PseudoAtomicLoadSub64:
128 case RISCV::PseudoAtomicLoadAnd32:
131 case RISCV::PseudoAtomicLoadAnd64:
134 case RISCV::PseudoAtomicLoadOr32:
136 case RISCV::PseudoAtomicLoadOr64:
138 case RISCV::PseudoAtomicLoadXor32:
141 case RISCV::PseudoAtomicLoadXor64:
144 case RISCV::PseudoAtomicLoadNand32:
147 case RISCV::PseudoAtomicLoadNand64:
150 case RISCV::PseudoAtomicLoadMin32:
153 case RISCV::PseudoAtomicLoadMin64:
156 case RISCV::PseudoAtomicLoadMax32:
159 case RISCV::PseudoAtomicLoadMax64:
162 case RISCV::PseudoAtomicLoadUMin32:
165 case RISCV::PseudoAtomicLoadUMin64:
168 case RISCV::PseudoAtomicLoadUMax32:
171 case RISCV::PseudoAtomicLoadUMax64:
174 case RISCV::PseudoMaskedAtomicSwap32:
177 case RISCV::PseudoMaskedAtomicLoadAdd32:
179 case RISCV::PseudoMaskedAtomicLoadSub32:
181 case RISCV::PseudoMaskedAtomicLoadNand32:
184 case RISCV::PseudoMaskedAtomicLoadMax32:
187 case RISCV::PseudoMaskedAtomicLoadMin32:
190 case RISCV::PseudoMaskedAtomicLoadUMax32:
193 case RISCV::PseudoMaskedAtomicLoadUMin32:
196 case RISCV::PseudoCmpXchg32:
197 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 32, NextMBBI);
198 case RISCV::PseudoCmpXchg64:
199 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 64, NextMBBI);
200 case RISCV::PseudoMaskedCmpXchg32:
201 return expandAtomicCmpXchg(
MBB,
MBBI,
true, 32, NextMBBI);
208RISCVExpandPseudoAtomicsImpl::getLRForRMW32(
AtomicOrdering Ordering)
const {
212 case AtomicOrdering::Monotonic:
214 case AtomicOrdering::Acquire:
215 if (STI->hasStdExtZtso())
217 return RISCV::LR_W_AQ;
218 case AtomicOrdering::Release:
220 case AtomicOrdering::AcquireRelease:
221 if (STI->hasStdExtZtso())
223 return RISCV::LR_W_AQ;
224 case AtomicOrdering::SequentiallyConsistent:
225 return RISCV::LR_W_AQRL;
230RISCVExpandPseudoAtomicsImpl::getSCForRMW32(
AtomicOrdering Ordering)
const {
234 case AtomicOrdering::Monotonic:
236 case AtomicOrdering::Acquire:
238 case AtomicOrdering::Release:
239 if (STI->hasStdExtZtso())
241 return RISCV::SC_W_RL;
242 case AtomicOrdering::AcquireRelease:
243 if (STI->hasStdExtZtso())
245 return RISCV::SC_W_RL;
246 case AtomicOrdering::SequentiallyConsistent:
247 return RISCV::SC_W_RL;
252RISCVExpandPseudoAtomicsImpl::getLRForRMW64(
AtomicOrdering Ordering)
const {
256 case AtomicOrdering::Monotonic:
258 case AtomicOrdering::Acquire:
259 if (STI->hasStdExtZtso())
261 return RISCV::LR_D_AQ;
262 case AtomicOrdering::Release:
264 case AtomicOrdering::AcquireRelease:
265 if (STI->hasStdExtZtso())
267 return RISCV::LR_D_AQ;
268 case AtomicOrdering::SequentiallyConsistent:
269 return RISCV::LR_D_AQRL;
274RISCVExpandPseudoAtomicsImpl::getSCForRMW64(
AtomicOrdering Ordering)
const {
278 case AtomicOrdering::Monotonic:
280 case AtomicOrdering::Acquire:
282 case AtomicOrdering::Release:
283 if (STI->hasStdExtZtso())
285 return RISCV::SC_D_RL;
286 case AtomicOrdering::AcquireRelease:
287 if (STI->hasStdExtZtso())
289 return RISCV::SC_D_RL;
290 case AtomicOrdering::SequentiallyConsistent:
291 return RISCV::SC_D_RL;
295unsigned RISCVExpandPseudoAtomicsImpl::getLRForRMW(
AtomicOrdering Ordering,
298 return getLRForRMW32(Ordering);
300 return getLRForRMW64(Ordering);
304unsigned RISCVExpandPseudoAtomicsImpl::getSCForRMW(
AtomicOrdering Ordering,
307 return getSCForRMW32(Ordering);
309 return getSCForRMW64(Ordering);
313void RISCVExpandPseudoAtomicsImpl::doAtomicBinOpExpansion(
318 Register ScratchReg =
MI.getOperand(1).getReg();
329 BuildMI(LoopMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
393 BuildMI(LoopMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
402void RISCVExpandPseudoAtomicsImpl::insertMaskedMerge(
405 assert(OldValReg != ScratchReg &&
"OldValReg and ScratchReg must be unique");
406 assert(OldValReg != MaskReg &&
"OldValReg and MaskReg must be unique");
407 assert(ScratchReg != MaskReg &&
"ScratchReg and MaskReg must be unique");
413 if (NewValReg == RISCV::X0) {
430void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicBinOpExpansion(
434 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
436 Register ScratchReg =
MI.getOperand(1).getReg();
451 BuildMI(LoopMBB,
DL,
TII->get(getLRForRMW32(Ordering)), DestReg)
481 insertMaskedMerge(
DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
484 BuildMI(LoopMBB,
DL,
TII->get(getSCForRMW32(Ordering)), ScratchReg)
493bool RISCVExpandPseudoAtomicsImpl::expandAtomicBinOp(
509 DoneMBB->splice(DoneMBB->end(), &
MBB,
MI,
MBB.
end());
510 DoneMBB->transferSuccessors(&
MBB);
514 doAtomicBinOpExpansion(
MI, LoopMBB, BinOp, Width);
516 doMaskedAtomicBinOpExpansion(
MI, LoopMBB, BinOp, Width);
519 MI.eraseFromParent();
521 LivePhysRegs LiveRegs;
528void RISCVExpandPseudoAtomicsImpl::insertSext(
DebugLoc DL,
529 MachineBasicBlock *
MBB,
540void RISCVExpandPseudoAtomicsImpl::doAtomicMinMaxOpExpansion(
541 MachineInstr &
MI, MachineBasicBlock *LoopHeadMBB,
542 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
546 Register ScratchReg =
MI.getOperand(1).getReg();
556 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
558 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::ADDI), ScratchReg)
594 BuildMI(LoopIfBodyMBB,
DL,
TII->get(RISCV::ADDI), ScratchReg)
601 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
610void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicMinMaxOpExpansion(
611 MachineInstr &
MI, MachineBasicBlock *LoopHeadMBB,
612 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
615 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
617 Register Scratch1Reg =
MI.getOperand(1).getReg();
618 Register Scratch2Reg =
MI.getOperand(2).getReg();
633 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW32(Ordering)), DestReg)
635 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::AND), Scratch2Reg)
638 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::ADDI), Scratch1Reg)
646 insertSext(
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
654 insertSext(
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
679 insertMaskedMerge(
DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg, MaskReg,
685 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
694bool RISCVExpandPseudoAtomicsImpl::expandAtomicMinMaxOp(
699 if (STI->hasPermissiveZalrsc() && STI->hasStdExtZbb() && !IsMasked)
721 DoneMBB->splice(DoneMBB->end(), &
MBB,
MI,
MBB.
end());
722 DoneMBB->transferSuccessors(&
MBB);
726 doAtomicMinMaxOpExpansion(
MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
729 doMaskedAtomicMinMaxOpExpansion(
MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
733 MI.eraseFromParent();
735 LivePhysRegs LiveRegs;
755bool RISCVExpandPseudoAtomicsImpl::tryToFoldBNEOnCmpXchgResult(
758 MachineBasicBlock *&LoopHeadBNETarget)
const {
767 if (
MBBI ==
E ||
MBBI->getOpcode() != RISCV::AND)
771 if (!(ANDOp1 == DestReg && ANDOp2 == MaskReg) &&
772 !(ANDOp1 == MaskReg && ANDOp2 == DestReg))
775 DestReg =
MBBI->getOperand(0).getReg();
781 if (
MBBI ==
E ||
MBBI->getOpcode() != RISCV::BNE)
785 if (!(BNEOp0 == DestReg && BNEOp1 == CmpValReg) &&
786 !(BNEOp0 == CmpValReg && BNEOp1 == DestReg))
791 if (BNEOp0 == DestReg && !
MBBI->getOperand(0).isKill())
793 if (BNEOp1 == DestReg && !
MBBI->getOperand(1).isKill())
798 LoopHeadBNETarget =
MBBI->getOperand(2).getMBB();
804 for (
auto *
MI : ToErase)
805 MI->eraseFromParent();
809bool RISCVExpandPseudoAtomicsImpl::expandAtomicCmpXchg(
820 Register ScratchReg =
MI.getOperand(1).getReg();
822 Register CmpValReg =
MI.getOperand(3).getReg();
823 Register NewValReg =
MI.getOperand(4).getReg();
826 MachineBasicBlock *LoopHeadBNETarget = DoneMBB;
827 tryToFoldBNEOnCmpXchgResult(
MBB, std::next(
MBBI), DestReg, CmpValReg, MaskReg,
840 DoneMBB->splice(DoneMBB->end(), &
MBB,
MI,
MBB.
end());
841 DoneMBB->transferSuccessors(&
MBB);
851 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
856 .
addMBB(LoopHeadBNETarget);
860 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
873 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
875 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::AND), ScratchReg)
881 .
addMBB(LoopHeadBNETarget);
891 if (NewValReg == RISCV::X0)
892 BuildMI(LoopTailMBB,
DL,
TII->get(RISCV::XOR), ScratchReg)
896 insertMaskedMerge(
DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
897 MaskReg, ScratchReg);
898 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
908 MI.eraseFromParent();
910 LivePhysRegs LiveRegs;
918char RISCVExpandPseudoAtomicsLegacy::ID = 0;
924 return new RISCVExpandPseudoAtomicsLegacy();
930 bool Changed = RISCVExpandPseudoAtomicsImpl().run(MF);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
Promote Memory to Register
static bool expandAtomicBinOp(MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI, unsigned Size, const MipsSubtarget *STI, const MipsInstrInfo *TII)
static bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NMBB, const MipsSubtarget *STI, const MipsInstrInfo *TII)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
#define RISCV_EXPAND_PSEUDO_ATOMICS_NAME
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
FunctionPass class - This class is used to implement most global optimizations.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
void push_back(const T &Elt)
Represent a constant reference to a string, i.e.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createRISCVExpandPseudoAtomicsLegacyPass()
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
AtomicOrdering
Atomic ordering for LLVM's memory model.
LLVM_ABI void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().