LLVM 18.0.0git
LoongArchExpandAtomicPseudoInsts.cpp
Go to the documentation of this file.
1//==- LoongArchExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands atomic pseudo instructions into
10// target instructions. This pass should be run at the last possible moment,
11// avoiding the possibility for other passes to break the requirements for
12// forward progress in the LL/SC block.
13//
14//===----------------------------------------------------------------------===//
15
16#include "LoongArch.h"
17#include "LoongArchInstrInfo.h"
18#include "LoongArchTargetMachine.h"
19
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23
24using namespace llvm;
25
26#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME \
27 "LoongArch atomic pseudo instruction expansion pass"
28
29namespace {
30
31class LoongArchExpandAtomicPseudo : public MachineFunctionPass {
32public:
33 const LoongArchInstrInfo *TII;
34 static char ID;
35
36 LoongArchExpandAtomicPseudo() : MachineFunctionPass(ID) {
37 initializeLoongArchExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
38 }
39
40 bool runOnMachineFunction(MachineFunction &MF) override;
41
42 StringRef getPassName() const override {
43 return LoongArch_EXPAND_ATOMIC_PSEUDO_NAME;
44 }
45
46private:
47 bool expandMBB(MachineBasicBlock &MBB);
48 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
49 MachineBasicBlock::iterator &NextMBBI);
50 bool expandAtomicBinOp(MachineBasicBlock &MBB,
51 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
52 bool IsMasked, int Width,
53 MachineBasicBlock::iterator &NextMBBI);
54 bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
55 MachineBasicBlock::iterator MBBI,
56 AtomicRMWInst::BinOp, bool IsMasked, int Width,
57 MachineBasicBlock::iterator &NextMBBI);
58 bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
59 MachineBasicBlock::iterator MBBI, bool IsMasked,
60 int Width, MachineBasicBlock::iterator &NextMBBI);
61};
62
63char LoongArchExpandAtomicPseudo::ID = 0;
64
65bool LoongArchExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
66 TII =
67 static_cast<const LoongArchInstrInfo *>(MF.getSubtarget().getInstrInfo());
68 bool Modified = false;
69 for (auto &MBB : MF)
70 Modified |= expandMBB(MBB);
71 return Modified;
72}
73
74bool LoongArchExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
75 bool Modified = false;
76
77 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
78 while (MBBI != E) {
79 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
80 Modified |= expandMI(MBB, MBBI, NMBBI);
81 MBBI = NMBBI;
82 }
83
84 return Modified;
85}
86
87bool LoongArchExpandAtomicPseudo::expandMI(
88 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
89 MachineBasicBlock::iterator &NextMBBI) {
90 switch (MBBI->getOpcode()) {
91 case LoongArch::PseudoMaskedAtomicSwap32:
92 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
93 NextMBBI);
94 case LoongArch::PseudoAtomicSwap32:
95 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
96 NextMBBI);
97 case LoongArch::PseudoMaskedAtomicLoadAdd32:
98 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
99 case LoongArch::PseudoMaskedAtomicLoadSub32:
100 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
101 case LoongArch::PseudoAtomicLoadNand32:
102 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
103 NextMBBI);
104 case LoongArch::PseudoAtomicLoadNand64:
105 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
106 NextMBBI);
107 case LoongArch::PseudoMaskedAtomicLoadNand32:
108 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
109 NextMBBI);
110 case LoongArch::PseudoAtomicLoadAdd32:
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
112 NextMBBI);
113 case LoongArch::PseudoAtomicLoadSub32:
114 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
115 NextMBBI);
116 case LoongArch::PseudoAtomicLoadAnd32:
117 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
118 NextMBBI);
119 case LoongArch::PseudoAtomicLoadOr32:
120 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
121 case LoongArch::PseudoAtomicLoadXor32:
122 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
123 NextMBBI);
124 case LoongArch::PseudoMaskedAtomicLoadUMax32:
125 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
126 NextMBBI);
127 case LoongArch::PseudoMaskedAtomicLoadUMin32:
128 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
129 NextMBBI);
130 case LoongArch::PseudoCmpXchg32:
131 return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
132 case LoongArch::PseudoCmpXchg64:
133 return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
134 case LoongArch::PseudoMaskedCmpXchg32:
135 return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
136 case LoongArch::PseudoMaskedAtomicLoadMax32:
137 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
138 NextMBBI);
139 case LoongArch::PseudoMaskedAtomicLoadMin32:
140 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
141 NextMBBI);
142 }
143 return false;
144}
145
146static void doAtomicBinOpExpansion(const LoongArchInstrInfo *TII,
147 MachineInstr &MI, DebugLoc DL,
148 MachineBasicBlock *ThisMBB,
149 MachineBasicBlock *LoopMBB,
150 MachineBasicBlock *DoneMBB,
151 AtomicRMWInst::BinOp BinOp, int Width) {
152 Register DestReg = MI.getOperand(0).getReg();
153 Register ScratchReg = MI.getOperand(1).getReg();
154 Register AddrReg = MI.getOperand(2).getReg();
155 Register IncrReg = MI.getOperand(3).getReg();
156 AtomicOrdering Ordering =
157 static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
158
159 // .loop:
160 // if(Ordering != AtomicOrdering::Monotonic)
161 // dbar 0
162 // ll.[w|d] dest, (addr)
163 // binop scratch, dest, val
164 // sc.[w|d] scratch, scratch, (addr)
165 // beqz scratch, loop
166 if (Ordering != AtomicOrdering::Monotonic)
167 BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
168 BuildMI(LoopMBB, DL,
169 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
170 .addReg(AddrReg)
171 .addImm(0);
172 switch (BinOp) {
173 default:
174 llvm_unreachable("Unexpected AtomicRMW BinOp");
175 case AtomicRMWInst::Xchg:
176 BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
177 .addReg(IncrReg)
178 .addReg(LoongArch::R0);
179 break;
180 case AtomicRMWInst::Nand:
181 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
182 .addReg(DestReg)
183 .addReg(IncrReg);
184 BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
185 .addReg(ScratchReg)
186 .addReg(LoongArch::R0);
187 break;
188 case AtomicRMWInst::Add:
189 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
190 .addReg(DestReg)
191 .addReg(IncrReg);
192 break;
193 case AtomicRMWInst::Sub:
194 BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
195 .addReg(DestReg)
196 .addReg(IncrReg);
197 break;
198 case AtomicRMWInst::And:
199 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
200 .addReg(DestReg)
201 .addReg(IncrReg);
202 break;
203 case AtomicRMWInst::Or:
204 BuildMI(LoopMBB, DL, TII->get(LoongArch::OR), ScratchReg)
205 .addReg(DestReg)
206 .addReg(IncrReg);
207 break;
208 case AtomicRMWInst::Xor:
209 BuildMI(LoopMBB, DL, TII->get(LoongArch::XOR), ScratchReg)
210 .addReg(DestReg)
211 .addReg(IncrReg);
212 break;
213 }
214 BuildMI(LoopMBB, DL,
215 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D), ScratchReg)
216 .addReg(ScratchReg)
217 .addReg(AddrReg)
218 .addImm(0);
219 BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
220 .addReg(ScratchReg)
221 .addMBB(LoopMBB);
222}
223
224static void insertMaskedMerge(const LoongArchInstrInfo *TII, DebugLoc DL,
225 MachineBasicBlock *MBB, Register DestReg,
226 Register OldValReg, Register NewValReg,
227 Register MaskReg, Register ScratchReg) {
228 assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
229 assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
230 assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
231
232 // res = oldval ^ ((oldval ^ newval) & masktargetdata);
233 BuildMI(MBB, DL, TII->get(LoongArch::XOR), ScratchReg)
234 .addReg(OldValReg)
235 .addReg(NewValReg);
236 BuildMI(MBB, DL, TII->get(LoongArch::AND), ScratchReg)
237 .addReg(ScratchReg)
238 .addReg(MaskReg);
239 BuildMI(MBB, DL, TII->get(LoongArch::XOR), DestReg)
240 .addReg(OldValReg)
241 .addReg(ScratchReg);
242}
243
244static void doMaskedAtomicBinOpExpansion(
245 const LoongArchInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
246 MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
247 MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
248 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
249 Register DestReg = MI.getOperand(0).getReg();
250 Register ScratchReg = MI.getOperand(1).getReg();
251 Register AddrReg = MI.getOperand(2).getReg();
252 Register IncrReg = MI.getOperand(3).getReg();
253 Register MaskReg = MI.getOperand(4).getReg();
254 AtomicOrdering Ordering =
255 static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
256
257 // .loop:
258 // if(Ordering != AtomicOrdering::Monotonic)
259 // dbar 0
260 // ll.w destreg, (alignedaddr)
261 // binop scratch, destreg, incr
262 // xor scratch, destreg, scratch
263 // and scratch, scratch, masktargetdata
264 // xor scratch, destreg, scratch
265 // sc.w scratch, scratch, (alignedaddr)
266 // beqz scratch, loop
267 if (Ordering != AtomicOrdering::Monotonic)
268 BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
269 BuildMI(LoopMBB, DL, TII->get(LoongArch::LL_W), DestReg)
270 .addReg(AddrReg)
271 .addImm(0);
272 switch (BinOp) {
273 default:
274 llvm_unreachable("Unexpected AtomicRMW BinOp");
275 case AtomicRMWInst::Xchg:
276 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADDI_W), ScratchReg)
277 .addReg(IncrReg)
278 .addImm(0);
279 break;
280 case AtomicRMWInst::Add:
281 BuildMI(LoopMBB, DL, TII->get(LoongArch::ADD_W), ScratchReg)
282 .addReg(DestReg)
283 .addReg(IncrReg);
284 break;
285 case AtomicRMWInst::Sub:
286 BuildMI(LoopMBB, DL, TII->get(LoongArch::SUB_W), ScratchReg)
287 .addReg(DestReg)
288 .addReg(IncrReg);
289 break;
290 case AtomicRMWInst::Nand:
291 BuildMI(LoopMBB, DL, TII->get(LoongArch::AND), ScratchReg)
292 .addReg(DestReg)
293 .addReg(IncrReg);
294 BuildMI(LoopMBB, DL, TII->get(LoongArch::NOR), ScratchReg)
295 .addReg(ScratchReg)
296 .addReg(LoongArch::R0);
297 // TODO: support other AtomicRMWInst.
298 }
299
300 insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
301 ScratchReg);
302
303 BuildMI(LoopMBB, DL, TII->get(LoongArch::SC_W), ScratchReg)
304 .addReg(ScratchReg)
305 .addReg(AddrReg)
306 .addImm(0);
307 BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
308 .addReg(ScratchReg)
309 .addMBB(LoopMBB);
310}
311
312bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
313 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
314 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
315 MachineBasicBlock::iterator &NextMBBI) {
316 MachineInstr &MI = *MBBI;
317 DebugLoc DL = MI.getDebugLoc();
318
319 MachineFunction *MF = MBB.getParent();
320 auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
321 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
322
323 // Insert new MBBs.
324 MF->insert(++MBB.getIterator(), LoopMBB);
325 MF->insert(++LoopMBB->getIterator(), DoneMBB);
326
327 // Set up successors and transfer remaining instructions to DoneMBB.
328 LoopMBB->addSuccessor(LoopMBB);
329 LoopMBB->addSuccessor(DoneMBB);
330 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
331 DoneMBB->transferSuccessors(&MBB);
332 MBB.addSuccessor(LoopMBB);
333
334 if (IsMasked)
335 doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
336 Width);
337 else
338 doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
339
340 NextMBBI = MBB.end();
341 MI.eraseFromParent();
342
343 LivePhysRegs LiveRegs;
344 computeAndAddLiveIns(LiveRegs, *LoopMBB);
345 computeAndAddLiveIns(LiveRegs, *DoneMBB);
346
347 return true;
348}
349
350static void insertSext(const LoongArchInstrInfo *TII, DebugLoc DL,
351 MachineBasicBlock *MBB, Register ValReg,
352 Register ShamtReg) {
353 BuildMI(MBB, DL, TII->get(LoongArch::SLL_W), ValReg)
354 .addReg(ValReg)
355 .addReg(ShamtReg);
356 BuildMI(MBB, DL, TII->get(LoongArch::SRA_W), ValReg)
357 .addReg(ValReg)
358 .addReg(ShamtReg);
359}
360
361bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
362 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
363 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
364 MachineBasicBlock::iterator &NextMBBI) {
365 assert(IsMasked == true &&
366 "Should only need to expand masked atomic max/min");
367 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
368
369 MachineInstr &MI = *MBBI;
370 DebugLoc DL = MI.getDebugLoc();
371 MachineFunction *MF = MBB.getParent();
372 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
373 auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
374 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
375 auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
376 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
377
378 // Insert new MBBs.
379 MF->insert(++MBB.getIterator(), LoopHeadMBB);
380 MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
381 MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
382 MF->insert(++LoopTailMBB->getIterator(), TailMBB);
383 MF->insert(++TailMBB->getIterator(), DoneMBB);
384
385 // Set up successors and transfer remaining instructions to DoneMBB.
386 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
387 LoopHeadMBB->addSuccessor(LoopTailMBB);
388 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
389 LoopTailMBB->addSuccessor(LoopHeadMBB);
390 LoopTailMBB->addSuccessor(TailMBB);
391 TailMBB->addSuccessor(DoneMBB);
392 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
393 DoneMBB->transferSuccessors(&MBB);
394 MBB.addSuccessor(LoopHeadMBB);
395
396 Register DestReg = MI.getOperand(0).getReg();
397 Register Scratch1Reg = MI.getOperand(1).getReg();
398 Register Scratch2Reg = MI.getOperand(2).getReg();
399 Register AddrReg = MI.getOperand(3).getReg();
400 Register IncrReg = MI.getOperand(4).getReg();
401 Register MaskReg = MI.getOperand(5).getReg();
402
403 //
404 // .loophead:
405 // dbar 0
406 // ll.w destreg, (alignedaddr)
407 // and scratch2, destreg, mask
408 // move scratch1, destreg
409 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
410 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
411 .addReg(AddrReg)
412 .addImm(0);
413 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), Scratch2Reg)
414 .addReg(DestReg)
415 .addReg(MaskReg);
416 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), Scratch1Reg)
417 .addReg(DestReg)
418 .addReg(LoongArch::R0);
419
420 switch (BinOp) {
421 default:
422 llvm_unreachable("Unexpected AtomicRMW BinOp");
423 // bgeu scratch2, incr, .looptail
424 case AtomicRMWInst::UMax:
425 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
426 .addReg(Scratch2Reg)
427 .addReg(IncrReg)
428 .addMBB(LoopTailMBB);
429 break;
430 // bgeu incr, scratch2, .looptail
431 case AtomicRMWInst::UMin:
432 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
433 .addReg(IncrReg)
434 .addReg(Scratch2Reg)
435 .addMBB(LoopTailMBB);
436 break;
437 case AtomicRMWInst::Max:
438 insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
439 // bge scratch2, incr, .looptail
440 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
441 .addReg(Scratch2Reg)
442 .addReg(IncrReg)
443 .addMBB(LoopTailMBB);
444 break;
445 case AtomicRMWInst::Min:
446 insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
447 // bge incr, scratch2, .looptail
448 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGE))
449 .addReg(IncrReg)
450 .addReg(Scratch2Reg)
451 .addMBB(LoopTailMBB);
452 break;
453 // TODO: support other AtomicRMWInst.
454 }
455
456 // .loopifbody:
457 // xor scratch1, destreg, incr
458 // and scratch1, scratch1, mask
459 // xor scratch1, destreg, scratch1
460 insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
461 MaskReg, Scratch1Reg);
462
463 // .looptail:
464 // sc.w scratch1, scratch1, (addr)
465 // beqz scratch1, loop
466 // dbar 0x700
467 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
468 .addReg(Scratch1Reg)
469 .addReg(AddrReg)
470 .addImm(0);
471 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
472 .addReg(Scratch1Reg)
473 .addMBB(LoopHeadMBB);
474
475 // .tail:
476 // dbar 0x700
477 BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
478
479 NextMBBI = MBB.end();
480 MI.eraseFromParent();
481
482 LivePhysRegs LiveRegs;
483 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
484 computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
485 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
486 computeAndAddLiveIns(LiveRegs, *TailMBB);
487 computeAndAddLiveIns(LiveRegs, *DoneMBB);
488
489 return true;
490}
491
492bool LoongArchExpandAtomicPseudo::expandAtomicCmpXchg(
493 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
494 int Width, MachineBasicBlock::iterator &NextMBBI) {
495 MachineInstr &MI = *MBBI;
496 DebugLoc DL = MI.getDebugLoc();
497 MachineFunction *MF = MBB.getParent();
498 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
499 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
500 auto TailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
501 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
502
503 // Insert new MBBs.
504 MF->insert(++MBB.getIterator(), LoopHeadMBB);
505 MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
506 MF->insert(++LoopTailMBB->getIterator(), TailMBB);
507 MF->insert(++TailMBB->getIterator(), DoneMBB);
508
509 // Set up successors and transfer remaining instructions to DoneMBB.
510 LoopHeadMBB->addSuccessor(LoopTailMBB);
511 LoopHeadMBB->addSuccessor(TailMBB);
512 LoopTailMBB->addSuccessor(DoneMBB);
513 LoopTailMBB->addSuccessor(LoopHeadMBB);
514 TailMBB->addSuccessor(DoneMBB);
515 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
516 DoneMBB->transferSuccessors(&MBB);
517 MBB.addSuccessor(LoopHeadMBB);
518
519 Register DestReg = MI.getOperand(0).getReg();
520 Register ScratchReg = MI.getOperand(1).getReg();
521 Register AddrReg = MI.getOperand(2).getReg();
522 Register CmpValReg = MI.getOperand(3).getReg();
523 Register NewValReg = MI.getOperand(4).getReg();
524
525 if (!IsMasked) {
526 // .loophead:
527 // ll.[w|d] dest, (addr)
528 // bne dest, cmpval, tail
529 BuildMI(LoopHeadMBB, DL,
530 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
531 .addReg(AddrReg)
532 .addImm(0);
533 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
534 .addReg(DestReg)
535 .addReg(CmpValReg)
536 .addMBB(TailMBB);
537 // .looptail:
538 // dbar 0
539 // move scratch, newval
540 // sc.[w|d] scratch, scratch, (addr)
541 // beqz scratch, loophead
542 // b done
543 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
544 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
545 .addReg(NewValReg)
546 .addReg(LoongArch::R0);
547 BuildMI(LoopTailMBB, DL,
548 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
549 ScratchReg)
550 .addReg(ScratchReg)
551 .addReg(AddrReg)
552 .addImm(0);
553 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
554 .addReg(ScratchReg)
555 .addMBB(LoopHeadMBB);
556 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
557 } else {
558 // .loophead:
559 // ll.[w|d] dest, (addr)
560 // and scratch, dest, mask
561 // bne scratch, cmpval, tail
562 Register MaskReg = MI.getOperand(5).getReg();
563 BuildMI(LoopHeadMBB, DL,
564 TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
565 .addReg(AddrReg)
566 .addImm(0);
567 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), ScratchReg)
568 .addReg(DestReg)
569 .addReg(MaskReg);
570 BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BNE))
571 .addReg(ScratchReg)
572 .addReg(CmpValReg)
573 .addMBB(TailMBB);
574
575 // .looptail:
576 // dbar 0
577 // andn scratch, dest, mask
578 // or scratch, scratch, newval
579 // sc.[w|d] scratch, scratch, (addr)
580 // beqz scratch, loophead
581 // b done
582 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
583 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::ANDN), ScratchReg)
584 .addReg(DestReg)
585 .addReg(MaskReg);
586 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::OR), ScratchReg)
587 .addReg(ScratchReg)
588 .addReg(NewValReg);
589 BuildMI(LoopTailMBB, DL,
590 TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D),
591 ScratchReg)
592 .addReg(ScratchReg)
593 .addReg(AddrReg)
594 .addImm(0);
595 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
596 .addReg(ScratchReg)
597 .addMBB(LoopHeadMBB);
598 BuildMI(LoopTailMBB, DL, TII->get(LoongArch::B)).addMBB(DoneMBB);
599 }
600
601 // .tail:
602 // dbar 0x700
603 BuildMI(TailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
604
605 NextMBBI = MBB.end();
606 MI.eraseFromParent();
607
608 LivePhysRegs LiveRegs;
609 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
610 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
611 computeAndAddLiveIns(LiveRegs, *TailMBB);
612 computeAndAddLiveIns(LiveRegs, *DoneMBB);
613
614 return true;
615}
616
617} // end namespace
618
619INITIALIZE_PASS(LoongArchExpandAtomicPseudo, "loongarch-expand-atomic-pseudo",
620 LoongArch_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
621
622namespace llvm {
623
624FunctionPass *createLoongArchExpandAtomicPseudoPass() {
625 return new LoongArchExpandAtomicPseudo();
626}
627
628} // end namespace llvm
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:110
LivePhysRegs.h
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
#define LoongArch_EXPAND_ATOMIC_PSEUDO_NAME
Definition: LoongArchExpandAtomicPseudoInsts.cpp:26
INITIALIZE_PASS
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::AtomicRMWInst::BinOp
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:730
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:734
llvm::AtomicRMWInst::Min
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:748
llvm::AtomicRMWInst::Or
@ Or
*p = old | v
Definition: Instructions.h:742
llvm::AtomicRMWInst::Sub
@ Sub
*p = old - v
Definition: Instructions.h:736
llvm::AtomicRMWInst::And
@ And
*p = old & v
Definition: Instructions.h:738
llvm::AtomicRMWInst::Xor
@ Xor
*p = old ^ v
Definition: Instructions.h:744
llvm::AtomicRMWInst::Max
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:746
llvm::AtomicRMWInst::UMin
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:752
llvm::AtomicRMWInst::UMax
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:750
llvm::AtomicRMWInst::Nand
@ Nand
*p = ~(old & v)
Definition: Instructions.h:740
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:50
llvm::MachineBasicBlock::transferSuccessors
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
Definition: MachineBasicBlock.cpp:907
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:222
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:788
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:274
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1043
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MachineFunctionPass::runOnMachineFunction
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:454
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:708
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:931
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:132
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:98
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:147
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:68
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
llvm::Pass::getPassName
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:95
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::createLoongArchExpandAtomicPseudoPass
FunctionPass * createLoongArchExpandAtomicPseudoPass()
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:363
llvm::initializeLoongArchExpandAtomicPseudoPass
void initializeLoongArchExpandAtomicPseudoPass(PassRegistry &)
llvm::AtomicOrdering
AtomicOrdering
Atomic ordering for LLVM's memory model.
Definition: AtomicOrdering.h:56
llvm::computeAndAddLiveIns
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
Definition: LivePhysRegs.cpp:336
Generated on Tue Sep 26 2023 20:34:15 for LLVM by doxygen 1.9.6