LLVM 20.0.0git
LoongArchExpandAtomicPseudoInsts.cpp
Go to the documentation of this file.
1//==- LoongArchExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands atomic pseudo instructions into
10// target instructions. This pass should be run at the last possible moment,
11// avoiding the possibility for other passes to break the requirements for
12// forward progress in the LL/SC block.
13//
14//===----------------------------------------------------------------------===//
15
16#include "LoongArch.h"
17#include "LoongArchInstrInfo.h"
18#include "LoongArchTargetMachine.h"
19
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23
24using namespace llvm;
25
26#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME \
27 "LoongArch atomic pseudo instruction expansion pass"
28
29namespace {
30
31class LoongArchExpandAtomicPseudo : public MachineFunctionPass {
32public:
33 const LoongArchInstrInfo *TII;
34 static char ID;
35
36 LoongArchExpandAtomicPseudo() : MachineFunctionPass(ID) {
37 initializeLoongArchExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
38 }
39
40 bool runOnMachineFunction(MachineFunction &MF) override;
41
42 StringRef getPassName() const override {
43 return LoongArch_EXPAND_ATOMIC_PSEUDO_NAME;
44 }
45
46private:
47 bool expandMBB(MachineBasicBlock &MBB);
48 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
49 MachineBasicBlock::iterator &NextMBBI);
50 bool expandAtomicBinOp(MachineBasicBlock &MBB,
51 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
52 bool IsMasked, int Width,
53 MachineBasicBlock::iterator &NextMBBI);
54 bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
55 MachineBasicBlock::iterator MBBI,
56 AtomicRMWInst::BinOp, bool IsMasked, int Width,
57 MachineBasicBlock::iterator &NextMBBI);
58 bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
59 MachineBasicBlock::iterator MBBI, bool IsMasked,
60 int Width, MachineBasicBlock::iterator &NextMBBI);
61};
62
63char LoongArchExpandAtomicPseudo::ID = 0;
64
65bool LoongArchExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
66 TII =
67 static_cast<const LoongArchInstrInfo *>(MF.getSubtarget().getInstrInfo());
68 bool Modified = false;
69 for (auto &MBB : MF)
70 Modified |= expandMBB(MBB);
71 return Modified;
72}
73
74bool LoongArchExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
75 bool Modified = false;
76
77 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
78 while (MBBI != E) {
79 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
80 Modified |= expandMI(MBB, MBBI, NMBBI);
81 MBBI = NMBBI;
82 }
83
84 return Modified;
85}
86
87bool LoongArchExpandAtomicPseudo::expandMI(
88 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
89 MachineBasicBlock::iterator &NextMBBI) {
90 switch (MBBI->getOpcode()) {
91 case LoongArch::PseudoMaskedAtomicSwap32:
92 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
93 NextMBBI);
94 case LoongArch::PseudoAtomicSwap32:
95 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
96 NextMBBI);
97 case LoongArch::PseudoMaskedAtomicLoadAdd32:
98 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
99 case LoongArch::PseudoMaskedAtomicLoadSub32:
100 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
101 case LoongArch::PseudoAtomicLoadNand32:
102 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
103 NextMBBI);
104 case LoongArch::PseudoAtomicLoadNand64:
105 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
106 NextMBBI);
107 case LoongArch::PseudoMaskedAtomicLoadNand32:
108 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
109 NextMBBI);
110 case LoongArch::PseudoAtomicLoadAdd32:
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
112 NextMBBI);
113 case LoongArch::PseudoAtomicLoadSub32:
114 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
115 NextMBBI);
116 case LoongArch::PseudoAtomicLoadAnd32:
117 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
118 NextMBBI);
119 case LoongArch::PseudoAtomicLoadOr32:
120 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
121 case LoongArch::PseudoAtomicLoadXor32:
122 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
123 NextMBBI);
124 case LoongArch::PseudoMaskedAtomicLoadUMax32:
125 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
126 NextMBBI);
127 case LoongArch::PseudoMaskedAtomicLoadUMin32:
128 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
129 NextMBBI);
130 case LoongArch::PseudoCmpXchg32:
131 return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
132 case LoongArch::PseudoCmpXchg64:
133 return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
134 case LoongArch::PseudoMaskedCmpXchg32:
135 return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
136 case LoongArch::PseudoMaskedAtomicLoadMax32:
137 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
138 NextMBBI);
139 case LoongArch::PseudoMaskedAtomicLoadMin32:
140 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
141 NextMBBI);
142 }
143 return false;
144}
145
146static void doAtomicBinOpExpansion(const LoongArchInstrInfo *TII,
147 MachineInstr &MI, DebugLoc DL,
148 MachineBasicBlock *ThisMBB,
149 MachineBasicBlock *LoopMBB,
150 MachineBasicBlock *DoneMBB,
151 AtomicRMWInst::BinOp BinOp, int Width) {
152 Register DestReg = MI.getOperand(0).getReg();
153 Register ScratchReg = MI.getOperand(1).getReg();
154 Register AddrReg = MI.getOperand(2).getReg();
155 Register IncrReg = MI.getOperand(3).getReg();
156
157 // .loop:
158 // ll.[w|d] dest, (addr)
159 // binop scratch, dest, val
160 // sc.[w|d] scratch, scratch, (addr)
161 // beqz scratch, loop
162 BuildMI(LoopMBB, DL,
163 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
164 .addReg(AddrReg)
165 .addImm(0);
166 switch (BinOp) {
167 default:
168 llvm_unreachable("Unexpected AtomicRMW BinOp");
169 case AtomicRMWInst::Xchg:
170 BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
171 .addReg(IncrReg)
172 .addReg(LoongArch::R0);
173 break;
174 case AtomicRMWInst::Nand:
175 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
176 .addReg(DestReg)
177 .addReg(IncrReg);
178 BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
179 .addReg(ScratchReg)
180 .addReg(LoongArch::R0);
181 break;
182 case AtomicRMWInst::Add:
183 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
184 .addReg(DestReg)
185 .addReg(IncrReg);
186 break;
187 case AtomicRMWInst::Sub:
188 BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
189 .addReg(DestReg)
190 .addReg(IncrReg);
191 break;
192 case AtomicRMWInst::And:
193 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
194 .addReg(DestReg)
195 .addReg(IncrReg);
196 break;
197 case AtomicRMWInst::Or:
198 BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
199 .addReg(DestReg)
200 .addReg(IncrReg);
201 break;
202 case AtomicRMWInst::Xor:
203 BuildMI(LoopMBB, DL, TII->get(LoongArch::XOR), ScratchReg)
204 .addReg(DestReg)
205 .addReg(IncrReg);
206 break;
207 }
208 BuildMI(LoopMBB, DL,
209 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D), ScratchReg)
210 .addReg(ScratchReg)
211 .addReg(AddrReg)
212 .addImm(0);
213 BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
214 .addReg(ScratchReg)
215 .addMBB(LoopMBB);
216}
217
218static void insertMaskedMerge(const LoongArchInstrInfo *TII, DebugLoc DL,
219 MachineBasicBlock *MBB, Register DestReg,
220 Register OldValReg, Register NewValReg,
221 Register MaskReg, Register ScratchReg) {
222 assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
223 assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
224 assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
225
226 // res = oldval ^ ((oldval ^ newval) & masktargetdata);
227 BuildMI(MBB, DL, TII->get(LoongArch::XOR), ScratchReg)
228 .addReg(OldValReg)
229 .addReg(NewValReg);
230 BuildMI(MBB, DL, TII->get(LoongArch::AND), ScratchReg)
231 .addReg(ScratchReg)
232 .addReg(MaskReg);
233 BuildMI(MBB, DL, TII->get(LoongArch::XOR), DestReg)
234 .addReg(OldValReg)
235 .addReg(ScratchReg);
236}
237
238static void doMaskedAtomicBinOpExpansion(
239 const LoongArchInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
240 MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
241 MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
242 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
243 Register DestReg = MI.getOperand(0).getReg();
244 Register ScratchReg = MI.getOperand(1).getReg();
245 Register AddrReg = MI.getOperand(2).getReg();
246 Register IncrReg = MI.getOperand(3).getReg();
247 Register MaskReg = MI.getOperand(4).getReg();
248
249 // .loop:
250 // ll.w destreg, (alignedaddr)
251 // binop scratch, destreg, incr
252 // xor scratch, destreg, scratch
253 // and scratch, scratch, masktargetdata
254 // xor scratch, destreg, scratch
255 // sc.w scratch, scratch, (alignedaddr)
256 // beqz scratch, loop
257 BuildMI(LoopMBB, DL, TII->get(LoongArch::LL_W), DestReg)
258 .addReg(AddrReg)
259 .addImm(0);
260 switch (BinOp) {
261 default:
262 llvm_unreachable("Unexpected AtomicRMW BinOp");
263 case AtomicRMWInst::Xchg:
264 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADDI_W), ScratchReg)
265 .addReg(IncrReg)
266 .addImm(0);
267 break;
268 case AtomicRMWInst::Add:
269 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
270 .addReg(DestReg)
271 .addReg(IncrReg);
272 break;
273 case AtomicRMWInst::Sub:
274 BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
275 .addReg(DestReg)
276 .addReg(IncrReg);
277 break;
278 case AtomicRMWInst::Nand:
279 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
280 .addReg(DestReg)
281 .addReg(IncrReg);
282 BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
283 .addReg(ScratchReg)
284 .addReg(LoongArch::R0);
285 // TODO: support other AtomicRMWInst.
286 }
287
288 insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
289 ScratchReg);
290
291 BuildMI(LoopMBB, DL, TII->get(LoongArch::SC_W), ScratchReg)
292 .addReg(ScratchReg)
293 .addReg(AddrReg)
294 .addImm(0);
295 BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
296 .addReg(ScratchReg)
297 .addMBB(LoopMBB);
298}
299
300bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
301 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
302 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
303 MachineBasicBlock::iterator &NextMBBI) {
304 MachineInstr &MI = *MBBI;
305 DebugLoc DL = MI.getDebugLoc();
306
307 MachineFunction *MF = MBB.getParent();
308 auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
309 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
310
311 // Insert new MBBs.
312 MF->insert(++MBB.getIterator(), LoopMBB);
313 MF->insert(++LoopMBB->getIterator(), DoneMBB);
314
315 // Set up successors and transfer remaining instructions to DoneMBB.
316 LoopMBB->addSuccessor(LoopMBB);
317 LoopMBB->addSuccessor(DoneMBB);
318 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
319 DoneMBB->transferSuccessors(&MBB);
320 MBB.addSuccessor(LoopMBB);
321
322 if (IsMasked)
323 doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
324 Width);
325 else
326 doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
327
328 NextMBBI = MBB.end();
329 MI.eraseFromParent();
330
331 LivePhysRegs LiveRegs;
332 computeAndAddLiveIns(LiveRegs, *LoopMBB);
333 computeAndAddLiveIns(LiveRegs, *DoneMBB);
334
335 return true;
336}
337
338static void insertSext(const LoongArchInstrInfo *TII, DebugLoc DL,
339 MachineBasicBlock *MBB, Register ValReg,
340 Register ShamtReg) {
341 BuildMI(MBB, DL, TII->get(LoongArch::SLL_W), ValReg)
342 .addReg(ValReg)
343 .addReg(ShamtReg);
344 BuildMI(MBB, DL, TII->get(LoongArch::SRA_W), ValReg)
345 .addReg(ValReg)
346 .addReg(ShamtReg);
347}
348
349bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
350 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
351 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
352 MachineBasicBlock::iterator &NextMBBI) {
353 assert(IsMasked == true &&
354 "Should only need to expand masked atomic max/min");
355 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
356
357 MachineInstr &MI = *MBBI;
358 DebugLoc DL = MI.getDebugLoc();
359 MachineFunction *MF = MBB.getParent();
360 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
361 auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
362 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
363 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
364
365 // Insert new MBBs.
366 MF->insert(++MBB.getIterator(), LoopHeadMBB);
367 MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
368 MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
369 MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
370
371 // Set up successors and transfer remaining instructions to DoneMBB.
372 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
373 LoopHeadMBB->addSuccessor(LoopTailMBB);
374 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
375 LoopTailMBB->addSuccessor(LoopHeadMBB);
376 LoopTailMBB->addSuccessor(DoneMBB);
377 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
378 DoneMBB->transferSuccessors(&MBB);
379 MBB.addSuccessor(LoopHeadMBB);
380
381 Register DestReg = MI.getOperand(0).getReg();
382 Register Scratch1Reg = MI.getOperand(1).getReg();
383 Register Scratch2Reg = MI.getOperand(2).getReg();
384 Register AddrReg = MI.getOperand(3).getReg();
385 Register IncrReg = MI.getOperand(4).getReg();
386 Register MaskReg = MI.getOperand(5).getReg();
387
388 //
389 // .loophead:
390 // ll.w destreg, (alignedaddr)
391 // and scratch2, destreg, mask
392 // move scratch1, destreg
393 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
394 .addReg(AddrReg)
395 .addImm(0);
396 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), Scratch2Reg)
397 .addReg(DestReg)
398 .addReg(MaskReg);
399 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), Scratch1Reg)
400 .addReg(DestReg)
401 .addReg(LoongArch::R0);
402
403 switch (BinOp) {
404 default:
405 llvm_unreachable("Unexpected AtomicRMW BinOp");
406 // bgeu scratch2, incr, .looptail
407 case AtomicRMWInst::UMax:
408 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
409 .addReg(Scratch2Reg)
410 .addReg(IncrReg)
411 .addMBB(LoopTailMBB);
412 break;
413 // bgeu incr, scratch2, .looptail
414 case AtomicRMWInst::UMin:
415 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
416 .addReg(IncrReg)
417 .addReg(Scratch2Reg)
418 .addMBB(LoopTailMBB);
419 break;
420 case AtomicRMWInst::Max:
421 insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
422 // bge scratch2, incr, .looptail
423 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
424 .addReg(Scratch2Reg)
425 .addReg(IncrReg)
426 .addMBB(LoopTailMBB);
427 break;
428 case AtomicRMWInst::Min:
429 insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
430 // bge incr, scratch2, .looptail
431 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
432 .addReg(IncrReg)
433 .addReg(Scratch2Reg)
434 .addMBB(LoopTailMBB);
435 break;
436 // TODO: support other AtomicRMWInst.
437 }
438
439 // .loopifbody:
440 // xor scratch1, destreg, incr
441 // and scratch1, scratch1, mask
442 // xor scratch1, destreg, scratch1
443 insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
444 MaskReg, Scratch1Reg);
445
446 // .looptail:
447 // sc.w scratch1, scratch1, (addr)
448 // beqz scratch1, loop
449 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
450 .addReg(Scratch1Reg)
451 .addReg(AddrReg)
452 .addImm(0);
453 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
454 .addReg(Scratch1Reg)
455 .addMBB(LoopHeadMBB);
456
457 NextMBBI = MBB.end();
458 MI.eraseFromParent();
459
460 LivePhysRegs LiveRegs;
461 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
462 computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
463 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
464 computeAndAddLiveIns(LiveRegs, *DoneMBB);
465
466 return true;
467}
468
469bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
470 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
471 int Width, MachineBasicBlock::iterator &NextMBBI) {
472 MachineInstr &MI = *MBBI;
473 DebugLoc DL = MI.getDebugLoc();
474 MachineFunction *MF = MBB.getParent();
475 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
476 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
477 auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
478 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
479
480 // Insert new MBBs.
481 MF->insert(++MBB.getIterator(), LoopHeadMBB);
482 MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
483 MF->insert(++LoopTailMBB->getIterator(), TailMBB);
484 MF->insert(++TailMBB->getIterator(), DoneMBB);
485
486 // Set up successors and transfer remaining instructions to DoneMBB.
487 LoopHeadMBB->addSuccessor(LoopTailMBB);
488 LoopHeadMBB->addSuccessor(TailMBB);
489 LoopTailMBB->addSuccessor(DoneMBB);
490 LoopTailMBB->addSuccessor(LoopHeadMBB);
491 TailMBB->addSuccessor(DoneMBB);
492 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
493 DoneMBB->transferSuccessors(&MBB);
494 MBB.addSuccessor(LoopHeadMBB);
495
496 Register DestReg = MI.getOperand(0).getReg();
497 Register ScratchReg = MI.getOperand(1).getReg();
498 Register AddrReg = MI.getOperand(2).getReg();
499 Register CmpValReg = MI.getOperand(3).getReg();
500 Register NewValReg = MI.getOperand(4).getReg();
501
502 if (!IsMasked) {
503 // .loophead:
504 // ll.[w|d] dest, (addr)
505 // bne dest, cmpval, tail
506 BuildMI(LoopHeadMBB, DL,
507 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
508 .addReg(AddrReg)
509 .addImm(0);
510 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
511 .addReg(DestReg)
512 .addReg(CmpValReg)
513 .addMBB(TailMBB);
514 // .looptail:
515 // move scratch, newval
516 // sc.[w|d] scratch, scratch, (addr)
517 // beqz scratch, loophead
518 // b done
519 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
520 .addReg(NewValReg)
521 .addReg(LoongArch::R0);
522 BuildMI(LoopTailMBB, DL,
523 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
524 ScratchReg)
525 .addReg(ScratchReg)
526 .addReg(AddrReg)
527 .addImm(0);
528 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
529 .addReg(ScratchReg)
530 .addMBB(LoopHeadMBB);
531 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
532 } else {
533 // .loophead:
534 // ll.[w|d] dest, (addr)
535 // and scratch, dest, mask
536 // bne scratch, cmpval, tail
537 Register MaskReg = MI.getOperand(5).getReg();
538 BuildMI(LoopHeadMBB, DL,
539 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
540 .addReg(AddrReg)
541 .addImm(0);
542 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), ScratchReg)
543 .addReg(DestReg)
544 .addReg(MaskReg);
545 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
546 .addReg(ScratchReg)
547 .addReg(CmpValReg)
548 .addMBB(TailMBB);
549
550 // .looptail:
551 // andn scratch, dest, mask
552 // or scratch, scratch, newval
553 // sc.[w|d] scratch, scratch, (addr)
554 // beqz scratch, loophead
555 // b done
556 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::ANDN), ScratchReg)
557 .addReg(DestReg)
558 .addReg(MaskReg);
559 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
560 .addReg(ScratchReg)
561 .addReg(NewValReg);
562 BuildMI(LoopTailMBB, DL,
563 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
564 ScratchReg)
565 .addReg(ScratchReg)
566 .addReg(AddrReg)
567 .addImm(0);
568 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
569 .addReg(ScratchReg)
570 .addMBB(LoopHeadMBB);
571 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
572 }
573
574 AtomicOrdering FailureOrdering =
575 static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
576 int hint;
577
578 switch (FailureOrdering) {
579 case AtomicOrdering::Acquire:
580 case AtomicOrdering::AcquireRelease:
581 case AtomicOrdering::SequentiallyConsistent:
582 // acquire
583 hint = 0b10100;
584 break;
585 default:
586 hint = 0x700;
587 }
588
589 // .tail:
590 // dbar 0x700 | acquire
591
592 if (!(hint == 0x700 && MF->getSubtarget<LoongArchSubtarget>().hasLD_SEQ_SA()))
593 BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(hint);
594
595 NextMBBI = MBB.end();
596 MI.eraseFromParent();
597
598 LivePhysRegs LiveRegs;
599 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
600 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
601 computeAndAddLiveIns(LiveRegs, *TailMBB);
602 computeAndAddLiveIns(LiveRegs, *DoneMBB);
603
604 return true;
605}
606
607} // end namespace
608
609INITIALIZE_PASS(LoongArchExpandAtomicPseudo, "loongarch-expand-atomic-pseudo",
610 LoongArch_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
611
612namespace llvm {
613
614FunctionPass *createLoongArchExpandAtomicPseudoPass() {
615 return new LoongArchExpandAtomicPseudo();
616}
617
618} // end namespace llvm
MBB
MachineBasicBlock & MBB
Definition: ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: ARMSLSHardening.cpp:73
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: ARMSLSHardening.cpp:72
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:112
LivePhysRegs.h
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
Definition: LoongArchExpandAtomicPseudoInsts.cpp:26
INITIALIZE_PASS
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:716
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:720
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:734
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:728
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:722
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:724
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:730
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:732
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:738
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:736
llvm::AtomicRMWInst::Nand
@ Nand
*p = ~(old & v)
Definition: Instructions.h:726
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:52
llvm::MachineBasicBlock::transferSuccessors
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
Definition: MachineBasicBlock.cpp:917
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:256
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:798
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:311
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1109
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MachineFunctionPass::runOnMachineFunction
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:714
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:494
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:947
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:133
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:99
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:148
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:69
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
llvm::Pass::getPassName
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:97
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:132
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::createLoongArchExpandAtomicPseudoPass
FunctionPass * createLoongArchExpandAtomicPseudoPass()
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:373
llvm::initializeLoongArchExpandAtomicPseudoPass
void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &)
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition: AtomicOrdering.h:56
llvm::computeAndAddLiveIns
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
Definition: LivePhysRegs.cpp:336
Generated on Fri Dec 20 2024 05:52:29 for LLVM by doxygen 1.9.6