LLVM 20.0.0git
LoongArchExpandAtomicPseudoInsts.cpp
Go to the documentation of this file.
1//==- LoongArchExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands atomic pseudo instructions into
10// target instructions. This pass should be run at the last possible moment,
11// avoiding the possibility for other passes to break the requirements for
12// forward progress in the LL/SC block.
13//
14//===----------------------------------------------------------------------===//
15
16#include "LoongArch.h"
17#include "LoongArchInstrInfo.h"
18#include "LoongArchTargetMachine.h"
19
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23
24using namespace llvm;
25
26#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME \
27 "LoongArch atomic pseudo instruction expansion pass"
28
29namespace {
30
31class LoongArchExpandAtomicPseudo : public MachineFunctionPass {
32public:
33 const LoongArchInstrInfo *TII;
34 static char ID;
35
36 LoongArchExpandAtomicPseudo() : MachineFunctionPass(ID) {
37 initializeLoongArchExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
38 }
39
40 bool runOnMachineFunction(MachineFunction &MF) override;
41
42 StringRef getPassName() const override {
43 return LoongArch_EXPAND_ATOMIC_PSEUDO_NAME;
44 }
45
46private:
47 bool expandMBB(MachineBasicBlock &MBB);
48 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
49 MachineBasicBlock::iterator &NextMBBI);
50 bool expandAtomicBinOp(MachineBasicBlock &MBB,
51 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
52 bool IsMasked, int Width,
53 MachineBasicBlock::iterator &NextMBBI);
54 bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
55 MachineBasicBlock::iterator MBBI,
56 AtomicRMWInst::BinOp, bool IsMasked, int Width,
57 MachineBasicBlock::iterator &NextMBBI);
58 bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
59 MachineBasicBlock::iterator MBBI, bool IsMasked,
60 int Width, MachineBasicBlock::iterator &NextMBBI);
61 bool expandAtomicCmpXchg128(MachineBasicBlock &MBB,
62 MachineBasicBlock::iterator,
63 MachineBasicBlock::iterator &NextMBBI);
64};
65
66char LoongArchExpandAtomicPseudo::ID = 0;
67
68bool LoongArchExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
69 TII =
70 static_cast<const LoongArchInstrInfo *>(MF.getSubtarget().getInstrInfo());
71 bool Modified = false;
72 for (auto &MBB : MF)
73 Modified |= expandMBB(MBB);
74 return Modified;
75}
76
77bool LoongArchExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
78 bool Modified = false;
79
80 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
81 while (MBBI != E) {
82 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
83 Modified |= expandMI(MBB, MBBI, NMBBI);
84 MBBI = NMBBI;
85 }
86
87 return Modified;
88}
89
90bool LoongArchExpandAtomicPseudo::expandMI(
91 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
92 MachineBasicBlock::iterator &NextMBBI) {
93 switch (MBBI->getOpcode()) {
94 case LoongArch::PseudoMaskedAtomicSwap32:
95 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
96 NextMBBI);
97 case LoongArch::PseudoAtomicSwap32:
98 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
99 NextMBBI);
100 case LoongArch::PseudoMaskedAtomicLoadAdd32:
101 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
102 case LoongArch::PseudoMaskedAtomicLoadSub32:
103 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
104 case LoongArch::PseudoAtomicLoadNand32:
105 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
106 NextMBBI);
107 case LoongArch::PseudoAtomicLoadNand64:
108 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
109 NextMBBI);
110 case LoongArch::PseudoMaskedAtomicLoadNand32:
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
112 NextMBBI);
113 case LoongArch::PseudoAtomicLoadAdd32:
114 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
115 NextMBBI);
116 case LoongArch::PseudoAtomicLoadSub32:
117 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
118 NextMBBI);
119 case LoongArch::PseudoAtomicLoadAnd32:
120 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
121 NextMBBI);
122 case LoongArch::PseudoAtomicLoadOr32:
123 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
124 case LoongArch::PseudoAtomicLoadXor32:
125 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
126 NextMBBI);
127 case LoongArch::PseudoMaskedAtomicLoadUMax32:
128 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
129 NextMBBI);
130 case LoongArch::PseudoMaskedAtomicLoadUMin32:
131 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
132 NextMBBI);
133 case LoongArch::PseudoCmpXchg32:
134 return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
135 case LoongArch::PseudoCmpXchg64:
136 return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
137 case LoongArch::PseudoCmpXchg128:
138 case LoongArch::PseudoCmpXchg128Acquire:
139 return expandAtomicCmpXchg128(MBB, MBBI, NextMBBI);
140 case LoongArch::PseudoMaskedCmpXchg32:
141 return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
142 case LoongArch::PseudoMaskedAtomicLoadMax32:
143 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
144 NextMBBI);
145 case LoongArch::PseudoMaskedAtomicLoadMin32:
146 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
147 NextMBBI);
148 }
149 return false;
150}
151
152static void doAtomicBinOpExpansion(const LoongArchInstrInfo *TII,
153 MachineInstr &MI, DebugLoc DL,
154 MachineBasicBlock *ThisMBB,
155 MachineBasicBlock *LoopMBB,
156 MachineBasicBlock *DoneMBB,
157 AtomicRMWInst::BinOp BinOp, int Width) {
158 Register DestReg = MI.getOperand(0).getReg();
159 Register ScratchReg = MI.getOperand(1).getReg();
160 Register AddrReg = MI.getOperand(2).getReg();
161 Register IncrReg = MI.getOperand(3).getReg();
162
163 // .loop:
164 // ll.[w|d] dest, (addr)
165 // binop scratch, dest, val
166 // sc.[w|d] scratch, scratch, (addr)
167 // beqz scratch, loop
168 BuildMI(LoopMBB, DL,
169 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
170 .addReg(AddrReg)
171 .addImm(0);
172 switch (BinOp) {
173 default:
174 llvm_unreachable("Unexpected AtomicRMW BinOp");
175 case AtomicRMWInst::Xchg:
176 BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
177 .addReg(IncrReg)
178 .addReg(LoongArch::R0);
179 break;
180 case AtomicRMWInst::Nand:
181 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
182 .addReg(DestReg)
183 .addReg(IncrReg);
184 BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
185 .addReg(ScratchReg)
186 .addReg(LoongArch::R0);
187 break;
188 case AtomicRMWInst::Add:
189 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
190 .addReg(DestReg)
191 .addReg(IncrReg);
192 break;
193 case AtomicRMWInst::Sub:
194 BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
195 .addReg(DestReg)
196 .addReg(IncrReg);
197 break;
198 case AtomicRMWInst::And:
199 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
200 .addReg(DestReg)
201 .addReg(IncrReg);
202 break;
203 case AtomicRMWInst::Or:
204 BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
205 .addReg(DestReg)
206 .addReg(IncrReg);
207 break;
208 case AtomicRMWInst::Xor:
209 BuildMI(LoopMBB, DL, TII->get(LoongArch::XOR), ScratchReg)
210 .addReg(DestReg)
211 .addReg(IncrReg);
212 break;
213 }
214 BuildMI(LoopMBB, DL,
215 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D), ScratchReg)
216 .addReg(ScratchReg)
217 .addReg(AddrReg)
218 .addImm(0);
219 BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
220 .addReg(ScratchReg)
221 .addMBB(LoopMBB);
222}
223
224static void insertMaskedMerge(const LoongArchInstrInfo *TII, DebugLoc DL,
225 MachineBasicBlock *MBB, Register DestReg,
226 Register OldValReg, Register NewValReg,
227 Register MaskReg, Register ScratchReg) {
228 assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
229 assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
230 assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
231
232 // res = oldval ^ ((oldval ^ newval) & masktargetdata);
233 BuildMI(MBB, DL, TII->get(LoongArch::XOR), ScratchReg)
234 .addReg(OldValReg)
235 .addReg(NewValReg);
236 BuildMI(MBB, DL, TII->get(LoongArch::AND), ScratchReg)
237 .addReg(ScratchReg)
238 .addReg(MaskReg);
239 BuildMI(MBB, DL, TII->get(LoongArch::XOR), DestReg)
240 .addReg(OldValReg)
241 .addReg(ScratchReg);
242}
243
244static void doMaskedAtomicBinOpExpansion(
245 const LoongArchInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
246 MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
247 MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
248 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
249 Register DestReg = MI.getOperand(0).getReg();
250 Register ScratchReg = MI.getOperand(1).getReg();
251 Register AddrReg = MI.getOperand(2).getReg();
252 Register IncrReg = MI.getOperand(3).getReg();
253 Register MaskReg = MI.getOperand(4).getReg();
254
255 // .loop:
256 // ll.w destreg, (alignedaddr)
257 // binop scratch, destreg, incr
258 // xor scratch, destreg, scratch
259 // and scratch, scratch, masktargetdata
260 // xor scratch, destreg, scratch
261 // sc.w scratch, scratch, (alignedaddr)
262 // beqz scratch, loop
263 BuildMI(LoopMBB, DL, TII->get(LoongArch::LL_W), DestReg)
264 .addReg(AddrReg)
265 .addImm(0);
266 switch (BinOp) {
267 default:
268 llvm_unreachable("Unexpected AtomicRMW BinOp");
269 case AtomicRMWInst::Xchg:
270 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADDI_W), ScratchReg)
271 .addReg(IncrReg)
272 .addImm(0);
273 break;
274 case AtomicRMWInst::Add:
275 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
276 .addReg(DestReg)
277 .addReg(IncrReg);
278 break;
279 case AtomicRMWInst::Sub:
280 BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
281 .addReg(DestReg)
282 .addReg(IncrReg);
283 break;
284 case AtomicRMWInst::Nand:
285 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
286 .addReg(DestReg)
287 .addReg(IncrReg);
288 BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
289 .addReg(ScratchReg)
290 .addReg(LoongArch::R0);
291 // TODO: support other AtomicRMWInst.
292 }
293
294 insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
295 ScratchReg);
296
297 BuildMI(LoopMBB, DL, TII->get(LoongArch::SC_W), ScratchReg)
298 .addReg(ScratchReg)
299 .addReg(AddrReg)
300 .addImm(0);
301 BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
302 .addReg(ScratchReg)
303 .addMBB(LoopMBB);
304}
305
306bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
307 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
308 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
309 MachineBasicBlock::iterator &NextMBBI) {
310 MachineInstr &MI = *MBBI;
311 DebugLoc DL = MI.getDebugLoc();
312
313 MachineFunction *MF = MBB.getParent();
314 auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
315 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
316
317 // Insert new MBBs.
318 MF->insert(++MBB.getIterator(), LoopMBB);
319 MF->insert(++LoopMBB->getIterator(), DoneMBB);
320
321 // Set up successors and transfer remaining instructions to DoneMBB.
322 LoopMBB->addSuccessor(LoopMBB);
323 LoopMBB->addSuccessor(DoneMBB);
324 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
325 DoneMBB->transferSuccessors(&MBB);
326 MBB.addSuccessor(LoopMBB);
327
328 if (IsMasked)
329 doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
330 Width);
331 else
332 doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
333
334 NextMBBI = MBB.end();
335 MI.eraseFromParent();
336
337 LivePhysRegs LiveRegs;
338 computeAndAddLiveIns(LiveRegs, *LoopMBB);
339 computeAndAddLiveIns(LiveRegs, *DoneMBB);
340
341 return true;
342}
343
344static void insertSext(const LoongArchInstrInfo *TII, DebugLoc DL,
345 MachineBasicBlock *MBB, Register ValReg,
346 Register ShamtReg) {
347 BuildMI(MBB, DL, TII->get(LoongArch::SLL_W), ValReg)
348 .addReg(ValReg)
349 .addReg(ShamtReg);
350 BuildMI(MBB, DL, TII->get(LoongArch::SRA_W), ValReg)
351 .addReg(ValReg)
352 .addReg(ShamtReg);
353}
354
355bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
356 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
357 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
358 MachineBasicBlock::iterator &NextMBBI) {
359 assert(IsMasked == true &&
360 "Should only need to expand masked atomic max/min");
361 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
362
363 MachineInstr &MI = *MBBI;
364 DebugLoc DL = MI.getDebugLoc();
365 MachineFunction *MF = MBB.getParent();
366 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
367 auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
368 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
369 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
370
371 // Insert new MBBs.
372 MF->insert(++MBB.getIterator(), LoopHeadMBB);
373 MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
374 MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
375 MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
376
377 // Set up successors and transfer remaining instructions to DoneMBB.
378 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
379 LoopHeadMBB->addSuccessor(LoopTailMBB);
380 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
381 LoopTailMBB->addSuccessor(LoopHeadMBB);
382 LoopTailMBB->addSuccessor(DoneMBB);
383 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
384 DoneMBB->transferSuccessors(&MBB);
385 MBB.addSuccessor(LoopHeadMBB);
386
387 Register DestReg = MI.getOperand(0).getReg();
388 Register Scratch1Reg = MI.getOperand(1).getReg();
389 Register Scratch2Reg = MI.getOperand(2).getReg();
390 Register AddrReg = MI.getOperand(3).getReg();
391 Register IncrReg = MI.getOperand(4).getReg();
392 Register MaskReg = MI.getOperand(5).getReg();
393
394 //
395 // .loophead:
396 // ll.w destreg, (alignedaddr)
397 // and scratch2, destreg, mask
398 // move scratch1, destreg
399 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
400 .addReg(AddrReg)
401 .addImm(0);
402 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), Scratch2Reg)
403 .addReg(DestReg)
404 .addReg(MaskReg);
405 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), Scratch1Reg)
406 .addReg(DestReg)
407 .addReg(LoongArch::R0);
408
409 switch (BinOp) {
410 default:
411 llvm_unreachable("Unexpected AtomicRMW BinOp");
412 // bgeu scratch2, incr, .looptail
413 case AtomicRMWInst::UMax:
414 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
415 .addReg(Scratch2Reg)
416 .addReg(IncrReg)
417 .addMBB(LoopTailMBB);
418 break;
419 // bgeu incr, scratch2, .looptail
420 case AtomicRMWInst::UMin:
421 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
422 .addReg(IncrReg)
423 .addReg(Scratch2Reg)
424 .addMBB(LoopTailMBB);
425 break;
426 case AtomicRMWInst::Max:
427 insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
428 // bge scratch2, incr, .looptail
429 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
430 .addReg(Scratch2Reg)
431 .addReg(IncrReg)
432 .addMBB(LoopTailMBB);
433 break;
434 case AtomicRMWInst::Min:
435 insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
436 // bge incr, scratch2, .looptail
437 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
438 .addReg(IncrReg)
439 .addReg(Scratch2Reg)
440 .addMBB(LoopTailMBB);
441 break;
442 // TODO: support other AtomicRMWInst.
443 }
444
445 // .loopifbody:
446 // xor scratch1, destreg, incr
447 // and scratch1, scratch1, mask
448 // xor scratch1, destreg, scratch1
449 insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
450 MaskReg, Scratch1Reg);
451
452 // .looptail:
453 // sc.w scratch1, scratch1, (addr)
454 // beqz scratch1, loop
455 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
456 .addReg(Scratch1Reg)
457 .addReg(AddrReg)
458 .addImm(0);
459 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
460 .addReg(Scratch1Reg)
461 .addMBB(LoopHeadMBB);
462
463 NextMBBI = MBB.end();
464 MI.eraseFromParent();
465
466 LivePhysRegs LiveRegs;
467 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
468 computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
469 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
470 computeAndAddLiveIns(LiveRegs, *DoneMBB);
471
472 return true;
473}
474
475bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
476 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
477 int Width, MachineBasicBlock::iterator &NextMBBI) {
478 MachineInstr &MI = *MBBI;
479 DebugLoc DL = MI.getDebugLoc();
480 MachineFunction *MF = MBB.getParent();
481 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
482 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
483 auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
484 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
485
486 // Insert new MBBs.
487 MF->insert(++MBB.getIterator(), LoopHeadMBB);
488 MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
489 MF->insert(++LoopTailMBB->getIterator(), TailMBB);
490 MF->insert(++TailMBB->getIterator(), DoneMBB);
491
492 // Set up successors and transfer remaining instructions to DoneMBB.
493 LoopHeadMBB->addSuccessor(LoopTailMBB);
494 LoopHeadMBB->addSuccessor(TailMBB);
495 LoopTailMBB->addSuccessor(DoneMBB);
496 LoopTailMBB->addSuccessor(LoopHeadMBB);
497 TailMBB->addSuccessor(DoneMBB);
498 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
499 DoneMBB->transferSuccessors(&MBB);
500 MBB.addSuccessor(LoopHeadMBB);
501
502 Register DestReg = MI.getOperand(0).getReg();
503 Register ScratchReg = MI.getOperand(1).getReg();
504 Register AddrReg = MI.getOperand(2).getReg();
505 Register CmpValReg = MI.getOperand(3).getReg();
506 Register NewValReg = MI.getOperand(4).getReg();
507
508 if (!IsMasked) {
509 // .loophead:
510 // ll.[w|d] dest, (addr)
511 // bne dest, cmpval, tail
512 BuildMI(LoopHeadMBB, DL,
513 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
514 .addReg(AddrReg)
515 .addImm(0);
516 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
517 .addReg(DestReg)
518 .addReg(CmpValReg)
519 .addMBB(TailMBB);
520 // .looptail:
521 // move scratch, newval
522 // sc.[w|d] scratch, scratch, (addr)
523 // beqz scratch, loophead
524 // b done
525 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
526 .addReg(NewValReg)
527 .addReg(LoongArch::R0);
528 BuildMI(LoopTailMBB, DL,
529 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
530 ScratchReg)
531 .addReg(ScratchReg)
532 .addReg(AddrReg)
533 .addImm(0);
534 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
535 .addReg(ScratchReg)
536 .addMBB(LoopHeadMBB);
537 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
538 } else {
539 // .loophead:
540 // ll.[w|d] dest, (addr)
541 // and scratch, dest, mask
542 // bne scratch, cmpval, tail
543 Register MaskReg = MI.getOperand(5).getReg();
544 BuildMI(LoopHeadMBB, DL,
545 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
546 .addReg(AddrReg)
547 .addImm(0);
548 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), ScratchReg)
549 .addReg(DestReg)
550 .addReg(MaskReg);
551 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
552 .addReg(ScratchReg)
553 .addReg(CmpValReg)
554 .addMBB(TailMBB);
555
556 // .looptail:
557 // andn scratch, dest, mask
558 // or scratch, scratch, newval
559 // sc.[w|d] scratch, scratch, (addr)
560 // beqz scratch, loophead
561 // b done
562 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::ANDN), ScratchReg)
563 .addReg(DestReg)
564 .addReg(MaskReg);
565 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
566 .addReg(ScratchReg)
567 .addReg(NewValReg);
568 BuildMI(LoopTailMBB, DL,
569 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
570 ScratchReg)
571 .addReg(ScratchReg)
572 .addReg(AddrReg)
573 .addImm(0);
574 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
575 .addReg(ScratchReg)
576 .addMBB(LoopHeadMBB);
577 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
578 }
579
580 AtomicOrdering FailureOrdering =
581 static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
582 int hint;
583
584 switch (FailureOrdering) {
585 case AtomicOrdering::Acquire:
586 case AtomicOrdering::AcquireRelease:
587 case AtomicOrdering::SequentiallyConsistent:
588 // acquire
589 hint = 0b10100;
590 break;
591 default:
592 hint = 0x700;
593 }
594
595 // .tail:
596 // dbar 0x700 | acquire
597
598 if (!(hint == 0x700 && MF->getSubtarget<LoongArchSubtarget>().hasLD_SEQ_SA()))
599 BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(hint);
600
601 NextMBBI = MBB.end();
602 MI.eraseFromParent();
603
604 LivePhysRegs LiveRegs;
605 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
606 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
607 computeAndAddLiveIns(LiveRegs, *TailMBB);
608 computeAndAddLiveIns(LiveRegs, *DoneMBB);
609
610 return true;
611}
612
613bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg128(
614 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
615 MachineBasicBlock::iterator &NextMBBI) {
616 MachineInstr &MI = *MBBI;
617 DebugLoc DL = MI.getDebugLoc();
618 MachineFunction *MF = MBB.getParent();
619 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
620 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
621 auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
622 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
623
624 // Insert new MBBs
625 MF->insert(++MBB.getIterator(), LoopHeadMBB);
626 MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
627 MF->insert(++LoopTailMBB->getIterator(), TailMBB);
628 MF->insert(++TailMBB->getIterator(), DoneMBB);
629
630 // Set up successors and transfer remaining instructions to DoneMBB.
631 LoopHeadMBB->addSuccessor(LoopTailMBB);
632 LoopHeadMBB->addSuccessor(TailMBB);
633 LoopTailMBB->addSuccessor(DoneMBB);
634 LoopTailMBB->addSuccessor(LoopHeadMBB);
635 TailMBB->addSuccessor(DoneMBB);
636 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
637 DoneMBB->transferSuccessors(&MBB);
638 MBB.addSuccessor(LoopHeadMBB);
639
640 Register DestLoReg = MI.getOperand(0).getReg();
641 Register DestHiReg = MI.getOperand(1).getReg();
642 Register ScratchReg = MI.getOperand(2).getReg();
643 Register AddrReg = MI.getOperand(3).getReg();
644 Register CmpValLoReg = MI.getOperand(4).getReg();
645 Register CmpValHiReg = MI.getOperand(5).getReg();
646 Register NewValLoReg = MI.getOperand(6).getReg();
647 Register NewValHiReg = MI.getOperand(7).getReg();
648
649 // .loophead:
650 // ll.d res_lo, (addr)
651 // dbar acquire
652 // ld.d res_hi, (addr), 8
653 // bne dest_lo, cmpval_lo, tail
654 // bne dest_hi, cmpval_hi, tail
655 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_D), DestLoReg)
656 .addReg(AddrReg)
657 .addImm(0);
658 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::DBAR)).addImm(0b10100);
659 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LD_D), DestHiReg)
660 .addReg(AddrReg)
661 .addImm(8);
662 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
663 .addReg(DestLoReg)
664 .addReg(CmpValLoReg)
665 .addMBB(TailMBB);
666 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
667 .addReg(DestHiReg)
668 .addReg(CmpValHiReg)
669 .addMBB(TailMBB);
670 // .looptail:
671 // move scratch, newval_lo
672 // sc.q scratch, newval_hi, (addr)
673 // beqz scratch, loophead
674 // b done
675 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
676 .addReg(NewValLoReg)
677 .addReg(LoongArch::R0);
678 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_Q), ScratchReg)
679 .addReg(ScratchReg)
680 .addReg(NewValHiReg)
681 .addReg(AddrReg);
682 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
683 .addReg(ScratchReg)
684 .addMBB(LoopHeadMBB);
685 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
686 int hint;
687
688 switch (MI.getOpcode()) {
689 case LoongArch::PseudoCmpXchg128Acquire:
690 // acquire acqrel seqcst
691 hint = 0b10100;
692 break;
693 case LoongArch::PseudoCmpXchg128:
694 hint = 0x700;
695 break;
696 default:
697 llvm_unreachable("Unexpected opcode");
698 }
699
700 // .tail:
701 // dbar 0x700 | acquire
702 if (!(hint == 0x700 && MF->getSubtarget<LoongArchSubtarget>().hasLD_SEQ_SA()))
703 BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(hint);
704
705 NextMBBI = MBB.end();
706 MI.eraseFromParent();
707
708 LivePhysRegs LiveRegs;
709 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
710 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
711 computeAndAddLiveIns(LiveRegs, *TailMBB);
712 computeAndAddLiveIns(LiveRegs, *DoneMBB);
713
714 return true;
715}
716
717} // end namespace
718
719INITIALIZE_PASS(LoongArchExpandAtomicPseudo, "loongarch-expand-atomic-pseudo",
720 LoongArch_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
721
722namespace llvm {
723
724FunctionPass *createLoongArchExpandAtomicPseudoPass() {
725 return new LoongArchExpandAtomicPseudo();
726}
727
728} // end namespace llvm
MBB
MachineBasicBlock & MBB
Definition: ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: ARMSLSHardening.cpp:73
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: ARMSLSHardening.cpp:72
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:112
LivePhysRegs.h
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
Definition: LoongArchExpandAtomicPseudoInsts.cpp:26
INITIALIZE_PASS
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:716
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:720
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:734
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:728
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:722
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:724
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:730
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:732
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:738
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:736
llvm::AtomicRMWInst::Nand
@ Nand
*p = ~(old & v)
Definition: Instructions.h:726
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:52
llvm::MachineBasicBlock::transferSuccessors
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
Definition: MachineBasicBlock.cpp:917
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:256
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:798
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:311
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1109
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MachineFunctionPass::runOnMachineFunction
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:733
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:499
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:966
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:133
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:99
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:148
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:71
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
llvm::Pass::getPassName
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:97
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:132
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::createLoongArchExpandAtomicPseudoPass
FunctionPass * createLoongArchExpandAtomicPseudoPass()
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:373
llvm::initializeLoongArchExpandAtomicPseudoPass
void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &)
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition: AtomicOrdering.h:56
llvm::computeAndAddLiveIns
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
Definition: LivePhysRegs.cpp:336
Generated on Fri Jun 20 2025 09:07:26 for LLVM by doxygen 1.9.6