26#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME \
27 "LoongArch atomic pseudo instruction expansion pass"
52 bool IsMasked,
int Width,
66char LoongArchExpandAtomicPseudo::ID = 0;
68bool LoongArchExpandAtomicPseudo::runOnMachineFunction(
MachineFunction &MF) {
90bool LoongArchExpandAtomicPseudo::expandMI(
93 switch (
MBBI->getOpcode()) {
94 case LoongArch::PseudoMaskedAtomicSwap32:
97 case LoongArch::PseudoAtomicSwap32:
100 case LoongArch::PseudoMaskedAtomicLoadAdd32:
102 case LoongArch::PseudoMaskedAtomicLoadSub32:
104 case LoongArch::PseudoAtomicLoadNand32:
107 case LoongArch::PseudoAtomicLoadNand64:
110 case LoongArch::PseudoMaskedAtomicLoadNand32:
113 case LoongArch::PseudoAtomicLoadAdd32:
116 case LoongArch::PseudoAtomicLoadSub32:
119 case LoongArch::PseudoAtomicLoadAnd32:
122 case LoongArch::PseudoAtomicLoadOr32:
124 case LoongArch::PseudoAtomicLoadXor32:
127 case LoongArch::PseudoMaskedAtomicLoadUMax32:
130 case LoongArch::PseudoMaskedAtomicLoadUMin32:
133 case LoongArch::PseudoCmpXchg32:
134 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 32, NextMBBI);
135 case LoongArch::PseudoCmpXchg64:
136 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 64, NextMBBI);
137 case LoongArch::PseudoCmpXchg128:
138 case LoongArch::PseudoCmpXchg128Acquire:
139 return expandAtomicCmpXchg128(
MBB,
MBBI, NextMBBI);
140 case LoongArch::PseudoMaskedCmpXchg32:
141 return expandAtomicCmpXchg(
MBB,
MBBI,
true, 32, NextMBBI);
142 case LoongArch::PseudoMaskedAtomicLoadMax32:
145 case LoongArch::PseudoMaskedAtomicLoadMin32:
159 Register ScratchReg =
MI.getOperand(1).getReg();
169 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
176 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::OR), ScratchReg)
181 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::AND), ScratchReg)
184 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::NOR), ScratchReg)
189 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::ADD_W), ScratchReg)
194 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::SUB_W), ScratchReg)
199 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::AND), ScratchReg)
204 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::OR), ScratchReg)
209 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::XOR), ScratchReg)
215 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D), ScratchReg)
228 assert(OldValReg != ScratchReg &&
"OldValReg and ScratchReg must be unique");
229 assert(OldValReg != MaskReg &&
"OldValReg and MaskReg must be unique");
230 assert(ScratchReg != MaskReg &&
"ScratchReg and MaskReg must be unique");
244static void doMaskedAtomicBinOpExpansion(
248 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
250 Register ScratchReg =
MI.getOperand(1).getReg();
263 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::LL_W), DestReg)
270 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::ADDI_W), ScratchReg)
275 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::ADD_W), ScratchReg)
280 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::SUB_W), ScratchReg)
285 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::AND), ScratchReg)
288 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::NOR), ScratchReg)
294 insertMaskedMerge(
TII,
DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
297 BuildMI(LoopMBB,
DL,
TII->get(LoongArch::SC_W), ScratchReg)
306bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
329 doMaskedAtomicBinOpExpansion(
TII,
MI,
DL, &
MBB, LoopMBB, DoneMBB, BinOp,
332 doAtomicBinOpExpansion(
TII,
MI,
DL, &
MBB, LoopMBB, DoneMBB, BinOp, Width);
335 MI.eraseFromParent();
355bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
359 assert(IsMasked ==
true &&
360 "Should only need to expand masked atomic max/min");
361 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
373 MF->
insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
374 MF->
insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
375 MF->
insert(++LoopTailMBB->getIterator(), DoneMBB);
378 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
379 LoopHeadMBB->addSuccessor(LoopTailMBB);
380 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
381 LoopTailMBB->addSuccessor(LoopHeadMBB);
382 LoopTailMBB->addSuccessor(DoneMBB);
388 Register Scratch1Reg =
MI.getOperand(1).getReg();
389 Register Scratch2Reg =
MI.getOperand(2).getReg();
399 BuildMI(LoopHeadMBB,
DL,
TII->get(LoongArch::LL_W), DestReg)
402 BuildMI(LoopHeadMBB,
DL,
TII->get(LoongArch::AND), Scratch2Reg)
405 BuildMI(LoopHeadMBB,
DL,
TII->get(LoongArch::OR), Scratch1Reg)
427 insertSext(
TII,
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
435 insertSext(
TII,
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
449 insertMaskedMerge(
TII,
DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
450 MaskReg, Scratch1Reg);
455 BuildMI(LoopTailMBB,
DL,
TII->get(LoongArch::SC_W), Scratch1Reg)
464 MI.eraseFromParent();
475bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
488 MF->
insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
489 MF->
insert(++LoopTailMBB->getIterator(), TailMBB);
490 MF->
insert(++TailMBB->getIterator(), DoneMBB);
493 LoopHeadMBB->addSuccessor(LoopTailMBB);
494 LoopHeadMBB->addSuccessor(TailMBB);
495 LoopTailMBB->addSuccessor(DoneMBB);
496 LoopTailMBB->addSuccessor(LoopHeadMBB);
497 TailMBB->addSuccessor(DoneMBB);
503 Register ScratchReg =
MI.getOperand(1).getReg();
505 Register CmpValReg =
MI.getOperand(3).getReg();
506 Register NewValReg =
MI.getOperand(4).getReg();
513 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
525 BuildMI(LoopTailMBB,
DL,
TII->get(LoongArch::OR), ScratchReg)
529 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
545 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
548 BuildMI(LoopHeadMBB,
DL,
TII->get(LoongArch::AND), ScratchReg)
562 BuildMI(LoopTailMBB,
DL,
TII->get(LoongArch::ANDN), ScratchReg)
565 BuildMI(LoopTailMBB,
DL,
TII->get(LoongArch::OR), ScratchReg)
569 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
584 switch (FailureOrdering) {
585 case AtomicOrdering::Acquire:
586 case AtomicOrdering::AcquireRelease:
587 case AtomicOrdering::SequentiallyConsistent:
599 BuildMI(TailMBB,
DL,
TII->get(LoongArch::DBAR)).addImm(hint);
602 MI.eraseFromParent();
613bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg128(
626 MF->
insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
627 MF->
insert(++LoopTailMBB->getIterator(), TailMBB);
628 MF->
insert(++TailMBB->getIterator(), DoneMBB);
631 LoopHeadMBB->addSuccessor(LoopTailMBB);
632 LoopHeadMBB->addSuccessor(TailMBB);
633 LoopTailMBB->addSuccessor(DoneMBB);
634 LoopTailMBB->addSuccessor(LoopHeadMBB);
635 TailMBB->addSuccessor(DoneMBB);
640 Register DestLoReg =
MI.getOperand(0).getReg();
641 Register DestHiReg =
MI.getOperand(1).getReg();
642 Register ScratchReg =
MI.getOperand(2).getReg();
644 Register CmpValLoReg =
MI.getOperand(4).getReg();
645 Register CmpValHiReg =
MI.getOperand(5).getReg();
646 Register NewValLoReg =
MI.getOperand(6).getReg();
647 Register NewValHiReg =
MI.getOperand(7).getReg();
655 BuildMI(LoopHeadMBB,
DL,
TII->get(LoongArch::LL_D), DestLoReg)
659 BuildMI(LoopHeadMBB,
DL,
TII->get(LoongArch::LD_D), DestHiReg)
675 BuildMI(LoopTailMBB,
DL,
TII->get(LoongArch::OR), ScratchReg)
678 BuildMI(LoopTailMBB,
DL,
TII->get(LoongArch::SC_Q), ScratchReg)
688 switch (
MI.getOpcode()) {
689 case LoongArch::PseudoCmpXchg128Acquire:
693 case LoongArch::PseudoCmpXchg128:
706 MI.eraseFromParent();
725 return new LoongArchExpandAtomicPseudo();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
FunctionPass class - This class is used to implement most global optimizations.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
StringRef - Represent a constant reference to a string, i.e.
virtual const TargetInstrInfo * getInstrInfo() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
This is an optimization pass for GlobalISel generic memory operations.
FunctionPass * createLoongArchExpandAtomicPseudoPass()
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &)
AtomicOrdering
Atomic ordering for LLVM's memory model.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().