LLVM 20.0.0git
PPCExpandAtomicPseudoInsts.cpp
Go to the documentation of this file.
1//===-- PPCExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands atomic pseudo instructions into
10// target instructions post RA. With such method, LL/SC loop is considered as
11// a whole blob and make spilling unlikely happens in the LL/SC loop.
12//
13//===----------------------------------------------------------------------===//
14
16#include "PPC.h"
17#include "PPCInstrInfo.h"
18
22
23using namespace llvm;
24
25#define DEBUG_TYPE "ppc-atomic-expand"
26
27namespace {
28
29class PPCExpandAtomicPseudo : public MachineFunctionPass {
30public:
31 const PPCInstrInfo *TII;
32 const PPCRegisterInfo *TRI;
33 static char ID;
34
35 PPCExpandAtomicPseudo() : MachineFunctionPass(ID) {
37 }
38
39 bool runOnMachineFunction(MachineFunction &MF) override;
40
41private:
42 bool expandMI(MachineBasicBlock &MBB, MachineInstr &MI,
44 bool expandAtomicRMW128(MachineBasicBlock &MBB, MachineInstr &MI,
46 bool expandAtomicCmpSwap128(MachineBasicBlock &MBB, MachineInstr &MI,
48};
49
50static void PairedCopy(const PPCInstrInfo *TII, MachineBasicBlock &MBB,
52 Register Dest0, Register Dest1, Register Src0,
53 Register Src1) {
54 const MCInstrDesc &OR = TII->get(PPC::OR8);
55 const MCInstrDesc &XOR = TII->get(PPC::XOR8);
56 if (Dest0 == Src1 && Dest1 == Src0) {
57 // The most tricky case, swapping values.
58 BuildMI(MBB, MBBI, DL, XOR, Dest0).addReg(Dest0).addReg(Dest1);
59 BuildMI(MBB, MBBI, DL, XOR, Dest1).addReg(Dest0).addReg(Dest1);
60 BuildMI(MBB, MBBI, DL, XOR, Dest0).addReg(Dest0).addReg(Dest1);
61 } else if (Dest0 != Src0 || Dest1 != Src1) {
62 if (Dest0 == Src1 || Dest1 != Src0) {
63 BuildMI(MBB, MBBI, DL, OR, Dest1).addReg(Src1).addReg(Src1);
64 BuildMI(MBB, MBBI, DL, OR, Dest0).addReg(Src0).addReg(Src0);
65 } else {
66 BuildMI(MBB, MBBI, DL, OR, Dest0).addReg(Src0).addReg(Src0);
67 BuildMI(MBB, MBBI, DL, OR, Dest1).addReg(Src1).addReg(Src1);
68 }
69 }
70}
71
72bool PPCExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
73 bool Changed = false;
74 TII = static_cast<const PPCInstrInfo *>(MF.getSubtarget().getInstrInfo());
75 TRI = &TII->getRegisterInfo();
76 for (MachineBasicBlock &MBB : MF) {
78 MBBI != MBBE;) {
80 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
81 Changed |= expandMI(MBB, MI, NMBBI);
82 MBBI = NMBBI;
83 }
84 }
85 if (Changed)
86 MF.RenumberBlocks();
87 return Changed;
88}
89
90bool PPCExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB, MachineInstr &MI,
92 switch (MI.getOpcode()) {
93 case PPC::ATOMIC_SWAP_I128:
94 case PPC::ATOMIC_LOAD_ADD_I128:
95 case PPC::ATOMIC_LOAD_SUB_I128:
96 case PPC::ATOMIC_LOAD_XOR_I128:
97 case PPC::ATOMIC_LOAD_NAND_I128:
98 case PPC::ATOMIC_LOAD_AND_I128:
99 case PPC::ATOMIC_LOAD_OR_I128:
100 return expandAtomicRMW128(MBB, MI, NMBBI);
101 case PPC::ATOMIC_CMP_SWAP_I128:
102 return expandAtomicCmpSwap128(MBB, MI, NMBBI);
103 case PPC::BUILD_QUADWORD: {
104 Register Dst = MI.getOperand(0).getReg();
105 Register DstHi = TRI->getSubReg(Dst, PPC::sub_gp8_x0);
106 Register DstLo = TRI->getSubReg(Dst, PPC::sub_gp8_x1);
107 Register Lo = MI.getOperand(1).getReg();
108 Register Hi = MI.getOperand(2).getReg();
109 PairedCopy(TII, MBB, MI, MI.getDebugLoc(), DstHi, DstLo, Hi, Lo);
110 MI.eraseFromParent();
111 return true;
112 }
113 default:
114 return false;
115 }
116}
117
118bool PPCExpandAtomicPseudo::expandAtomicRMW128(
121 const MCInstrDesc &LL = TII->get(PPC::LQARX);
122 const MCInstrDesc &SC = TII->get(PPC::STQCX);
123 DebugLoc DL = MI.getDebugLoc();
125 const BasicBlock *BB = MBB.getBasicBlock();
126 // Create layout of control flow.
130 MF->insert(MFI, LoopMBB);
131 MF->insert(MFI, ExitMBB);
132 ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()),
133 MBB.end());
135 MBB.addSuccessor(LoopMBB);
136
137 // For non-min/max operations, control flow is kinda like:
138 // MBB:
139 // ...
140 // LoopMBB:
141 // lqarx in, ptr
142 // addc out.sub_x1, in.sub_x1, op.sub_x1
143 // adde out.sub_x0, in.sub_x0, op.sub_x0
144 // stqcx out, ptr
145 // bne- LoopMBB
146 // ExitMBB:
147 // ...
148 Register Old = MI.getOperand(0).getReg();
149 Register OldHi = TRI->getSubReg(Old, PPC::sub_gp8_x0);
150 Register OldLo = TRI->getSubReg(Old, PPC::sub_gp8_x1);
151 Register Scratch = MI.getOperand(1).getReg();
152 Register ScratchHi = TRI->getSubReg(Scratch, PPC::sub_gp8_x0);
153 Register ScratchLo = TRI->getSubReg(Scratch, PPC::sub_gp8_x1);
154 Register RA = MI.getOperand(2).getReg();
155 Register RB = MI.getOperand(3).getReg();
156 Register IncrLo = MI.getOperand(4).getReg();
157 Register IncrHi = MI.getOperand(5).getReg();
158 unsigned RMWOpcode = MI.getOpcode();
159
160 MachineBasicBlock *CurrentMBB = LoopMBB;
161 BuildMI(CurrentMBB, DL, LL, Old).addReg(RA).addReg(RB);
162
163 switch (RMWOpcode) {
164 case PPC::ATOMIC_SWAP_I128:
165 PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo,
166 IncrHi, IncrLo);
167 break;
168 case PPC::ATOMIC_LOAD_ADD_I128:
169 BuildMI(CurrentMBB, DL, TII->get(PPC::ADDC8), ScratchLo)
170 .addReg(IncrLo)
171 .addReg(OldLo);
172 BuildMI(CurrentMBB, DL, TII->get(PPC::ADDE8), ScratchHi)
173 .addReg(IncrHi)
174 .addReg(OldHi);
175 break;
176 case PPC::ATOMIC_LOAD_SUB_I128:
177 BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFC8), ScratchLo)
178 .addReg(IncrLo)
179 .addReg(OldLo);
180 BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFE8), ScratchHi)
181 .addReg(IncrHi)
182 .addReg(OldHi);
183 break;
184
185#define TRIVIAL_ATOMICRMW(Opcode, Instr) \
186 case Opcode: \
187 BuildMI(CurrentMBB, DL, TII->get((Instr)), ScratchLo) \
188 .addReg(IncrLo) \
189 .addReg(OldLo); \
190 BuildMI(CurrentMBB, DL, TII->get((Instr)), ScratchHi) \
191 .addReg(IncrHi) \
192 .addReg(OldHi); \
193 break
194
195 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_OR_I128, PPC::OR8);
196 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_XOR_I128, PPC::XOR8);
197 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_AND_I128, PPC::AND8);
198 TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_NAND_I128, PPC::NAND8);
199#undef TRIVIAL_ATOMICRMW
200 default:
201 llvm_unreachable("Unhandled atomic RMW operation");
202 }
203 BuildMI(CurrentMBB, DL, SC).addReg(Scratch).addReg(RA).addReg(RB);
204 BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
206 .addReg(PPC::CR0)
207 .addMBB(LoopMBB);
208 CurrentMBB->addSuccessor(LoopMBB);
209 CurrentMBB->addSuccessor(ExitMBB);
210 fullyRecomputeLiveIns({ExitMBB, LoopMBB});
211 NMBBI = MBB.end();
212 MI.eraseFromParent();
213 return true;
214}
215
216bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128(
219 const MCInstrDesc &LL = TII->get(PPC::LQARX);
220 const MCInstrDesc &SC = TII->get(PPC::STQCX);
221 DebugLoc DL = MI.getDebugLoc();
223 const BasicBlock *BB = MBB.getBasicBlock();
224 Register Old = MI.getOperand(0).getReg();
225 Register OldHi = TRI->getSubReg(Old, PPC::sub_gp8_x0);
226 Register OldLo = TRI->getSubReg(Old, PPC::sub_gp8_x1);
227 Register Scratch = MI.getOperand(1).getReg();
228 Register ScratchHi = TRI->getSubReg(Scratch, PPC::sub_gp8_x0);
229 Register ScratchLo = TRI->getSubReg(Scratch, PPC::sub_gp8_x1);
230 Register RA = MI.getOperand(2).getReg();
231 Register RB = MI.getOperand(3).getReg();
232 Register CmpLo = MI.getOperand(4).getReg();
233 Register CmpHi = MI.getOperand(5).getReg();
234 Register NewLo = MI.getOperand(6).getReg();
235 Register NewHi = MI.getOperand(7).getReg();
236 // Create layout of control flow.
237 // loop:
238 // old = lqarx ptr
239 // <compare old, cmp>
240 // bne 0, exit
241 // succ:
242 // stqcx new ptr
243 // bne 0, loop
244 // exit:
245 // ....
247 MachineBasicBlock *LoopCmpMBB = MF->CreateMachineBasicBlock(BB);
248 MachineBasicBlock *CmpSuccMBB = MF->CreateMachineBasicBlock(BB);
250 MF->insert(MFI, LoopCmpMBB);
251 MF->insert(MFI, CmpSuccMBB);
252 MF->insert(MFI, ExitMBB);
253 ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()),
254 MBB.end());
256 MBB.addSuccessor(LoopCmpMBB);
257 // Build loop.
258 MachineBasicBlock *CurrentMBB = LoopCmpMBB;
259 BuildMI(CurrentMBB, DL, LL, Old).addReg(RA).addReg(RB);
260 BuildMI(CurrentMBB, DL, TII->get(PPC::XOR8), ScratchLo)
261 .addReg(OldLo)
262 .addReg(CmpLo);
263 BuildMI(CurrentMBB, DL, TII->get(PPC::XOR8), ScratchHi)
264 .addReg(OldHi)
265 .addReg(CmpHi);
266 BuildMI(CurrentMBB, DL, TII->get(PPC::OR8_rec), ScratchLo)
267 .addReg(ScratchLo)
268 .addReg(ScratchHi);
269 BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
271 .addReg(PPC::CR0)
272 .addMBB(ExitMBB);
273 CurrentMBB->addSuccessor(CmpSuccMBB);
274 CurrentMBB->addSuccessor(ExitMBB);
275 // Build succ.
276 CurrentMBB = CmpSuccMBB;
277 PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo,
278 NewHi, NewLo);
279 BuildMI(CurrentMBB, DL, SC).addReg(Scratch).addReg(RA).addReg(RB);
280 BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
282 .addReg(PPC::CR0)
283 .addMBB(LoopCmpMBB);
284 CurrentMBB->addSuccessor(LoopCmpMBB);
285 CurrentMBB->addSuccessor(ExitMBB);
286
287 fullyRecomputeLiveIns({ExitMBB, CmpSuccMBB, LoopCmpMBB});
288 NMBBI = MBB.end();
289 MI.eraseFromParent();
290 return true;
291}
292
293} // namespace
294
295INITIALIZE_PASS(PPCExpandAtomicPseudo, DEBUG_TYPE, "PowerPC Expand Atomic",
296 false, false)
297
298char PPCExpandAtomicPseudo::ID = 0;
300 return new PPCExpandAtomicPseudo();
301}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
unsigned const TargetRegisterInfo * TRI
#define TRIVIAL_ATOMICRMW(Opcode, Instr)
#define DEBUG_TYPE
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
SI optimize exec mask operations pre RA
support::ulittle16_t & Lo
Definition: aarch32.cpp:204
support::ulittle16_t & Hi
Definition: aarch32.cpp:203
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
A debug info location.
Definition: DebugLoc.h:33
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
virtual const TargetInstrInfo * getInstrInfo() const
self_iterator getIterator()
Definition: ilist_node.h:132
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createPPCExpandAtomicPseudoPass()
void initializePPCExpandAtomicPseudoPass(PassRegistry &)
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Definition: LivePhysRegs.h:215