LLVM 24.0.0git
RISCVExpandPseudoAtomics.cpp
Go to the documentation of this file.
1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains one of the several passes that expand pseudo instructions
10// into target instructions. This pass is run at the last possible moment,
11// avoiding the possibility for other passes to break the requirements for
12// forward progress in the LR/SC block.
13//
14//===----------------------------------------------------------------------===//
15
16#include "RISCV.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSubtarget.h"
23
24using namespace llvm;
25
26#define RISCV_EXPAND_PSEUDO_ATOMICS_NAME \
27 "RISC-V Pseudo Instruction Expansion - Atomics"
28
29namespace {
30
31class RISCVExpandPseudoAtomicsImpl final : public RISCVExpandPseudoImplBase {
33 MachineBasicBlock::iterator &NextMBBI) const override;
34
37 bool IsMasked, int Width,
38 MachineBasicBlock::iterator &NextMBBI) const;
39
40 bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
42 AtomicRMWInst::BinOp, bool IsMasked, int Width,
43 MachineBasicBlock::iterator &NextMBBI) const;
44
45 bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
46 MachineBasicBlock::iterator MBBI, bool IsMasked,
47 int Width,
48 MachineBasicBlock::iterator &NextMBBI) const;
49
50 unsigned getLRForRMW32(AtomicOrdering Ordering) const;
51 unsigned getSCForRMW32(AtomicOrdering Ordering) const;
52 unsigned getLRForRMW64(AtomicOrdering Ordering) const;
53 unsigned getSCForRMW64(AtomicOrdering Ordering) const;
54 unsigned getLRForRMW(AtomicOrdering Ordering, int Width) const;
55 unsigned getSCForRMW(AtomicOrdering Ordering, int Width) const;
56
57 void doAtomicBinOpExpansion(MachineInstr &MI, MachineBasicBlock *LoopMBB,
58 AtomicRMWInst::BinOp BinOp, int Width) const;
59 void insertMaskedMerge(DebugLoc DL, MachineBasicBlock *MBB, Register DestReg,
60 Register OldValReg, Register NewValReg,
61 Register MaskReg, Register ScratchReg) const;
62 void doMaskedAtomicBinOpExpansion(MachineInstr &MI,
63 MachineBasicBlock *LoopMBB,
65 int Width) const;
66 void insertSext(DebugLoc DL, MachineBasicBlock *MBB, Register ValReg,
67 Register ShamtReg) const;
68 void doAtomicMinMaxOpExpansion(MachineInstr &MI,
69 MachineBasicBlock *LoopHeadMBB,
70 MachineBasicBlock *LoopIfBodyMBB,
71 MachineBasicBlock *LoopTailMBB,
72 AtomicRMWInst::BinOp BinOp, int Width) const;
73 void doMaskedAtomicMinMaxOpExpansion(MachineInstr &MI,
74 MachineBasicBlock *LoopHeadMBB,
75 MachineBasicBlock *LoopIfBodyMBB,
76 MachineBasicBlock *LoopTailMBB,
78 int Width) const;
79 bool tryToFoldBNEOnCmpXchgResult(MachineBasicBlock &MBB,
81 Register DestReg, Register CmpValReg,
82 Register MaskReg,
83 MachineBasicBlock *&LoopHeadBNETarget) const;
84};
85
86class RISCVExpandPseudoAtomicsLegacy : public MachineFunctionPass {
87public:
88 static char ID;
89
90 RISCVExpandPseudoAtomicsLegacy() : MachineFunctionPass(ID) {}
91
92 bool runOnMachineFunction(MachineFunction &MF) override {
93 return RISCVExpandPseudoAtomicsImpl().run(MF);
94 }
95
96 StringRef getPassName() const override {
98 }
99};
100
101} // anonymous namespace
102
103bool RISCVExpandPseudoAtomicsImpl::expandMI(
105 MachineBasicBlock::iterator &NextMBBI) const {
106 // RISCVInstrInfo::getInstSizeInBytes expects that the total size of the
107 // expanded instructions for each pseudo is correct in the Size field of the
108 // tablegen definition for the pseudo.
109 switch (MBBI->getOpcode()) {
110 case RISCV::PseudoAtomicSwap32:
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
112 NextMBBI);
113 case RISCV::PseudoAtomicSwap64:
114 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 64,
115 NextMBBI);
116 case RISCV::PseudoAtomicLoadAdd32:
117 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
118 NextMBBI);
119 case RISCV::PseudoAtomicLoadAdd64:
120 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 64,
121 NextMBBI);
122 case RISCV::PseudoAtomicLoadSub32:
123 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
124 NextMBBI);
125 case RISCV::PseudoAtomicLoadSub64:
126 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 64,
127 NextMBBI);
128 case RISCV::PseudoAtomicLoadAnd32:
129 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
130 NextMBBI);
131 case RISCV::PseudoAtomicLoadAnd64:
132 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 64,
133 NextMBBI);
134 case RISCV::PseudoAtomicLoadOr32:
135 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
136 case RISCV::PseudoAtomicLoadOr64:
137 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 64, NextMBBI);
138 case RISCV::PseudoAtomicLoadXor32:
139 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
140 NextMBBI);
141 case RISCV::PseudoAtomicLoadXor64:
142 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 64,
143 NextMBBI);
144 case RISCV::PseudoAtomicLoadNand32:
145 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
146 NextMBBI);
147 case RISCV::PseudoAtomicLoadNand64:
148 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
149 NextMBBI);
150 case RISCV::PseudoAtomicLoadMin32:
151 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 32,
152 NextMBBI);
153 case RISCV::PseudoAtomicLoadMin64:
154 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 64,
155 NextMBBI);
156 case RISCV::PseudoAtomicLoadMax32:
157 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 32,
158 NextMBBI);
159 case RISCV::PseudoAtomicLoadMax64:
160 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 64,
161 NextMBBI);
162 case RISCV::PseudoAtomicLoadUMin32:
163 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 32,
164 NextMBBI);
165 case RISCV::PseudoAtomicLoadUMin64:
166 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 64,
167 NextMBBI);
168 case RISCV::PseudoAtomicLoadUMax32:
169 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 32,
170 NextMBBI);
171 case RISCV::PseudoAtomicLoadUMax64:
172 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 64,
173 NextMBBI);
174 case RISCV::PseudoMaskedAtomicSwap32:
176 NextMBBI);
177 case RISCV::PseudoMaskedAtomicLoadAdd32:
178 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
179 case RISCV::PseudoMaskedAtomicLoadSub32:
180 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
181 case RISCV::PseudoMaskedAtomicLoadNand32:
183 NextMBBI);
184 case RISCV::PseudoMaskedAtomicLoadMax32:
185 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
186 NextMBBI);
187 case RISCV::PseudoMaskedAtomicLoadMin32:
188 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
189 NextMBBI);
190 case RISCV::PseudoMaskedAtomicLoadUMax32:
191 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
192 NextMBBI);
193 case RISCV::PseudoMaskedAtomicLoadUMin32:
194 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
195 NextMBBI);
196 case RISCV::PseudoCmpXchg32:
197 return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
198 case RISCV::PseudoCmpXchg64:
199 return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
200 case RISCV::PseudoMaskedCmpXchg32:
201 return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
202 }
203
204 return false;
205}
206
207unsigned
208RISCVExpandPseudoAtomicsImpl::getLRForRMW32(AtomicOrdering Ordering) const {
209 switch (Ordering) {
210 default:
211 llvm_unreachable("Unexpected AtomicOrdering");
212 case AtomicOrdering::Monotonic:
213 return RISCV::LR_W;
214 case AtomicOrdering::Acquire:
215 if (STI->hasStdExtZtso())
216 return RISCV::LR_W;
217 return RISCV::LR_W_AQ;
218 case AtomicOrdering::Release:
219 return RISCV::LR_W;
220 case AtomicOrdering::AcquireRelease:
221 if (STI->hasStdExtZtso())
222 return RISCV::LR_W;
223 return RISCV::LR_W_AQ;
224 case AtomicOrdering::SequentiallyConsistent:
225 return RISCV::LR_W_AQRL;
226 }
227}
228
229unsigned
230RISCVExpandPseudoAtomicsImpl::getSCForRMW32(AtomicOrdering Ordering) const {
231 switch (Ordering) {
232 default:
233 llvm_unreachable("Unexpected AtomicOrdering");
234 case AtomicOrdering::Monotonic:
235 return RISCV::SC_W;
236 case AtomicOrdering::Acquire:
237 return RISCV::SC_W;
238 case AtomicOrdering::Release:
239 if (STI->hasStdExtZtso())
240 return RISCV::SC_W;
241 return RISCV::SC_W_RL;
242 case AtomicOrdering::AcquireRelease:
243 if (STI->hasStdExtZtso())
244 return RISCV::SC_W;
245 return RISCV::SC_W_RL;
246 case AtomicOrdering::SequentiallyConsistent:
247 return RISCV::SC_W_RL;
248 }
249}
250
251unsigned
252RISCVExpandPseudoAtomicsImpl::getLRForRMW64(AtomicOrdering Ordering) const {
253 switch (Ordering) {
254 default:
255 llvm_unreachable("Unexpected AtomicOrdering");
256 case AtomicOrdering::Monotonic:
257 return RISCV::LR_D;
258 case AtomicOrdering::Acquire:
259 if (STI->hasStdExtZtso())
260 return RISCV::LR_D;
261 return RISCV::LR_D_AQ;
262 case AtomicOrdering::Release:
263 return RISCV::LR_D;
264 case AtomicOrdering::AcquireRelease:
265 if (STI->hasStdExtZtso())
266 return RISCV::LR_D;
267 return RISCV::LR_D_AQ;
268 case AtomicOrdering::SequentiallyConsistent:
269 return RISCV::LR_D_AQRL;
270 }
271}
272
273unsigned
274RISCVExpandPseudoAtomicsImpl::getSCForRMW64(AtomicOrdering Ordering) const {
275 switch (Ordering) {
276 default:
277 llvm_unreachable("Unexpected AtomicOrdering");
278 case AtomicOrdering::Monotonic:
279 return RISCV::SC_D;
280 case AtomicOrdering::Acquire:
281 return RISCV::SC_D;
282 case AtomicOrdering::Release:
283 if (STI->hasStdExtZtso())
284 return RISCV::SC_D;
285 return RISCV::SC_D_RL;
286 case AtomicOrdering::AcquireRelease:
287 if (STI->hasStdExtZtso())
288 return RISCV::SC_D;
289 return RISCV::SC_D_RL;
290 case AtomicOrdering::SequentiallyConsistent:
291 return RISCV::SC_D_RL;
292 }
293}
294
295unsigned RISCVExpandPseudoAtomicsImpl::getLRForRMW(AtomicOrdering Ordering,
296 int Width) const {
297 if (Width == 32)
298 return getLRForRMW32(Ordering);
299 if (Width == 64)
300 return getLRForRMW64(Ordering);
301 llvm_unreachable("Unexpected LR width\n");
302}
303
304unsigned RISCVExpandPseudoAtomicsImpl::getSCForRMW(AtomicOrdering Ordering,
305 int Width) const {
306 if (Width == 32)
307 return getSCForRMW32(Ordering);
308 if (Width == 64)
309 return getSCForRMW64(Ordering);
310 llvm_unreachable("Unexpected SC width\n");
311}
312
313void RISCVExpandPseudoAtomicsImpl::doAtomicBinOpExpansion(
314 MachineInstr &MI, MachineBasicBlock *LoopMBB, AtomicRMWInst::BinOp BinOp,
315 int Width) const {
316 DebugLoc DL = MI.getDebugLoc();
317 Register DestReg = MI.getOperand(0).getReg();
318 Register ScratchReg = MI.getOperand(1).getReg();
319 Register AddrReg = MI.getOperand(2).getReg();
320 Register IncrReg = MI.getOperand(3).getReg();
322 static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
323
324 // .loop:
325 // lr.[w|d] dest, (addr)
326 // binop scratch, dest, val
327 // sc.[w|d] scratch, scratch, (addr)
328 // bnez scratch, loop
329 BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
330 .addReg(AddrReg);
331 switch (BinOp) {
332 default:
333 llvm_unreachable("Unexpected AtomicRMW BinOp");
335 BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
336 .addReg(IncrReg)
337 .addImm(0);
338 break;
340 BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
341 .addReg(DestReg)
342 .addReg(IncrReg);
343 break;
345 BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
346 .addReg(DestReg)
347 .addReg(IncrReg);
348 break;
350 BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
351 .addReg(DestReg)
352 .addReg(IncrReg);
353 break;
355 BuildMI(LoopMBB, DL, TII->get(RISCV::OR), ScratchReg)
356 .addReg(DestReg)
357 .addReg(IncrReg);
358 break;
360 BuildMI(LoopMBB, DL, TII->get(RISCV::XOR), ScratchReg)
361 .addReg(DestReg)
362 .addReg(IncrReg);
363 break;
365 BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
366 .addReg(DestReg)
367 .addReg(IncrReg);
368 BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
369 .addReg(ScratchReg)
370 .addImm(-1);
371 break;
373 BuildMI(LoopMBB, DL, TII->get(RISCV::MAX), ScratchReg)
374 .addReg(DestReg)
375 .addReg(IncrReg);
376 break;
378 BuildMI(LoopMBB, DL, TII->get(RISCV::MIN), ScratchReg)
379 .addReg(DestReg)
380 .addReg(IncrReg);
381 break;
383 BuildMI(LoopMBB, DL, TII->get(RISCV::MAXU), ScratchReg)
384 .addReg(DestReg)
385 .addReg(IncrReg);
386 break;
388 BuildMI(LoopMBB, DL, TII->get(RISCV::MINU), ScratchReg)
389 .addReg(DestReg)
390 .addReg(IncrReg);
391 break;
392 }
393 BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
394 .addReg(ScratchReg)
395 .addReg(AddrReg);
396 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
397 .addReg(ScratchReg)
398 .addReg(RISCV::X0)
399 .addMBB(LoopMBB);
400}
401
402void RISCVExpandPseudoAtomicsImpl::insertMaskedMerge(
403 DebugLoc DL, MachineBasicBlock *MBB, Register DestReg, Register OldValReg,
404 Register NewValReg, Register MaskReg, Register ScratchReg) const {
405 assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
406 assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
407 assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
408
409 // We select bits from newval and oldval using:
410 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
411 // r = oldval ^ ((oldval ^ newval) & masktargetdata);
412 // A zero newval reduces this to r = oldval ^ (oldval & masktargetdata).
413 if (NewValReg == RISCV::X0) {
414 BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
415 .addReg(OldValReg)
416 .addReg(MaskReg);
417 } else {
418 BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
419 .addReg(OldValReg)
420 .addReg(NewValReg);
421 BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
422 .addReg(ScratchReg)
423 .addReg(MaskReg);
424 }
425 BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
426 .addReg(OldValReg)
427 .addReg(ScratchReg);
428}
429
430void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicBinOpExpansion(
431 MachineInstr &MI, MachineBasicBlock *LoopMBB, AtomicRMWInst::BinOp BinOp,
432 int Width) const {
433 DebugLoc DL = MI.getDebugLoc();
434 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
435 Register DestReg = MI.getOperand(0).getReg();
436 Register ScratchReg = MI.getOperand(1).getReg();
437 Register AddrReg = MI.getOperand(2).getReg();
438 Register IncrReg = MI.getOperand(3).getReg();
439 Register MaskReg = MI.getOperand(4).getReg();
441 static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
442
443 // .loop:
444 // lr.w destreg, (alignedaddr)
445 // binop scratch, destreg, incr
446 // xor scratch, destreg, scratch
447 // and scratch, scratch, masktargetdata
448 // xor scratch, destreg, scratch
449 // sc.w scratch, scratch, (alignedaddr)
450 // bnez scratch, loop
451 BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
452 .addReg(AddrReg);
453 switch (BinOp) {
454 default:
455 llvm_unreachable("Unexpected AtomicRMW BinOp");
457 BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
458 .addReg(IncrReg)
459 .addImm(0);
460 break;
462 BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
463 .addReg(DestReg)
464 .addReg(IncrReg);
465 break;
467 BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
468 .addReg(DestReg)
469 .addReg(IncrReg);
470 break;
472 BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
473 .addReg(DestReg)
474 .addReg(IncrReg);
475 BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
476 .addReg(ScratchReg)
477 .addImm(-1);
478 break;
479 }
480
481 insertMaskedMerge(DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
482 ScratchReg);
483
484 BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
485 .addReg(ScratchReg)
486 .addReg(AddrReg);
487 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
488 .addReg(ScratchReg)
489 .addReg(RISCV::X0)
490 .addMBB(LoopMBB);
491}
492
493bool RISCVExpandPseudoAtomicsImpl::expandAtomicBinOp(
494 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
495 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
496 MachineBasicBlock::iterator &NextMBBI) const {
497 MachineInstr &MI = *MBBI;
499 auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
500 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
501
502 // Insert new MBBs.
503 MF->insert(++MBB.getIterator(), LoopMBB);
504 MF->insert(++LoopMBB->getIterator(), DoneMBB);
505
506 // Set up successors and transfer remaining instructions to DoneMBB.
507 LoopMBB->addSuccessor(LoopMBB);
508 LoopMBB->addSuccessor(DoneMBB);
509 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
510 DoneMBB->transferSuccessors(&MBB);
511 MBB.addSuccessor(LoopMBB);
512
513 if (!IsMasked)
514 doAtomicBinOpExpansion(MI, LoopMBB, BinOp, Width);
515 else
516 doMaskedAtomicBinOpExpansion(MI, LoopMBB, BinOp, Width);
517
518 NextMBBI = MBB.end();
519 MI.eraseFromParent();
520
521 LivePhysRegs LiveRegs;
522 computeAndAddLiveIns(LiveRegs, *LoopMBB);
523 computeAndAddLiveIns(LiveRegs, *DoneMBB);
524
525 return true;
526}
527
528void RISCVExpandPseudoAtomicsImpl::insertSext(DebugLoc DL,
529 MachineBasicBlock *MBB,
530 Register ValReg,
531 Register ShamtReg) const {
532 BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
533 .addReg(ValReg)
534 .addReg(ShamtReg);
535 BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
536 .addReg(ValReg)
537 .addReg(ShamtReg);
538}
539
540void RISCVExpandPseudoAtomicsImpl::doAtomicMinMaxOpExpansion(
541 MachineInstr &MI, MachineBasicBlock *LoopHeadMBB,
542 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
543 AtomicRMWInst::BinOp BinOp, int Width) const {
544 DebugLoc DL = MI.getDebugLoc();
545 Register DestReg = MI.getOperand(0).getReg();
546 Register ScratchReg = MI.getOperand(1).getReg();
547 Register AddrReg = MI.getOperand(2).getReg();
548 Register IncrReg = MI.getOperand(3).getReg();
550 static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
551
552 // .loophead:
553 // lr.[w|d] dest, (addr)
554 // mv scratch, dest
555 // ifnochangeneeded scratch, incr, .looptail
556 BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
557 .addReg(AddrReg);
558 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
559 .addReg(DestReg)
560 .addImm(0);
561 switch (BinOp) {
562 default:
563 llvm_unreachable("Unexpected AtomicRMW BinOp");
564 case AtomicRMWInst::Max: {
565 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
566 .addReg(ScratchReg)
567 .addReg(IncrReg)
568 .addMBB(LoopTailMBB);
569 break;
570 }
571 case AtomicRMWInst::Min: {
572 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
573 .addReg(IncrReg)
574 .addReg(ScratchReg)
575 .addMBB(LoopTailMBB);
576 break;
577 }
579 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
580 .addReg(ScratchReg)
581 .addReg(IncrReg)
582 .addMBB(LoopTailMBB);
583 break;
585 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
586 .addReg(IncrReg)
587 .addReg(ScratchReg)
588 .addMBB(LoopTailMBB);
589 break;
590 }
591
592 // .loopifbody:
593 // mv scratch, incr
594 BuildMI(LoopIfBodyMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
595 .addReg(IncrReg)
596 .addImm(0);
597
598 // .looptail:
599 // sc.[w|d] scratch, scratch, (addr)
600 // bnez scratch, loop
601 BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
602 .addReg(ScratchReg)
603 .addReg(AddrReg);
604 BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
605 .addReg(ScratchReg)
606 .addReg(RISCV::X0)
607 .addMBB(LoopHeadMBB);
608}
609
610void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicMinMaxOpExpansion(
611 MachineInstr &MI, MachineBasicBlock *LoopHeadMBB,
612 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
613 AtomicRMWInst::BinOp BinOp, int Width) const {
614 DebugLoc DL = MI.getDebugLoc();
615 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
616 Register DestReg = MI.getOperand(0).getReg();
617 Register Scratch1Reg = MI.getOperand(1).getReg();
618 Register Scratch2Reg = MI.getOperand(2).getReg();
619 Register AddrReg = MI.getOperand(3).getReg();
620 Register IncrReg = MI.getOperand(4).getReg();
621 Register MaskReg = MI.getOperand(5).getReg();
622 bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
624 static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());
625
626 //
627 // .loophead:
628 // lr.w destreg, (alignedaddr)
629 // and scratch2, destreg, mask
630 // mv scratch1, destreg
631 // [sext scratch2 if signed min/max]
632 // ifnochangeneeded scratch2, incr, .looptail
633 BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
634 .addReg(AddrReg);
635 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
636 .addReg(DestReg)
637 .addReg(MaskReg);
638 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
639 .addReg(DestReg)
640 .addImm(0);
641
642 switch (BinOp) {
643 default:
644 llvm_unreachable("Unexpected AtomicRMW BinOp");
645 case AtomicRMWInst::Max: {
646 insertSext(DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
647 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
648 .addReg(Scratch2Reg)
649 .addReg(IncrReg)
650 .addMBB(LoopTailMBB);
651 break;
652 }
653 case AtomicRMWInst::Min: {
654 insertSext(DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
655 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
656 .addReg(IncrReg)
657 .addReg(Scratch2Reg)
658 .addMBB(LoopTailMBB);
659 break;
660 }
662 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
663 .addReg(Scratch2Reg)
664 .addReg(IncrReg)
665 .addMBB(LoopTailMBB);
666 break;
668 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
669 .addReg(IncrReg)
670 .addReg(Scratch2Reg)
671 .addMBB(LoopTailMBB);
672 break;
673 }
674
675 // .loopifbody:
676 // xor scratch1, destreg, incr
677 // and scratch1, scratch1, mask
678 // xor scratch1, destreg, scratch1
679 insertMaskedMerge(DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg, MaskReg,
680 Scratch1Reg);
681
682 // .looptail:
683 // sc.w scratch1, scratch1, (addr)
684 // bnez scratch1, loop
685 BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
686 .addReg(Scratch1Reg)
687 .addReg(AddrReg);
688 BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
689 .addReg(Scratch1Reg)
690 .addReg(RISCV::X0)
691 .addMBB(LoopHeadMBB);
692}
693
694bool RISCVExpandPseudoAtomicsImpl::expandAtomicMinMaxOp(
695 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
696 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
697 MachineBasicBlock::iterator &NextMBBI) const {
698 // Using MIN(U)/MAX(U) is preferrable if permitted
699 if (STI->hasPermissiveZalrsc() && STI->hasStdExtZbb() && !IsMasked)
700 return expandAtomicBinOp(MBB, MBBI, BinOp, IsMasked, Width, NextMBBI);
701
702 MachineInstr &MI = *MBBI;
704 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
705 auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
706 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
707 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
708
709 // Insert new MBBs.
710 MF->insert(++MBB.getIterator(), LoopHeadMBB);
711 MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
712 MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
713 MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
714
715 // Set up successors and transfer remaining instructions to DoneMBB.
716 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
717 LoopHeadMBB->addSuccessor(LoopTailMBB);
718 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
719 LoopTailMBB->addSuccessor(LoopHeadMBB);
720 LoopTailMBB->addSuccessor(DoneMBB);
721 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
722 DoneMBB->transferSuccessors(&MBB);
723 MBB.addSuccessor(LoopHeadMBB);
724
725 if (!IsMasked)
726 doAtomicMinMaxOpExpansion(MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
727 BinOp, Width);
728 else
729 doMaskedAtomicMinMaxOpExpansion(MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
730 BinOp, Width);
731
732 NextMBBI = MBB.end();
733 MI.eraseFromParent();
734
735 LivePhysRegs LiveRegs;
736 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
737 computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
738 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
739 computeAndAddLiveIns(LiveRegs, *DoneMBB);
740
741 return true;
742}
743
744// If a BNE on the cmpxchg comparison result immediately follows the cmpxchg
745// operation, it can be folded into the cmpxchg expansion by
746// modifying the branch within 'LoopHead' (which performs the same
747// comparison). This is a valid transformation because after altering the
748// LoopHead's BNE destination, the BNE following the cmpxchg becomes
749// redundant and and be deleted. In the case of a masked cmpxchg, an
750// appropriate AND and BNE must be matched.
751//
752// On success, returns true and deletes the matching BNE or AND+BNE, sets the
753// LoopHeadBNETarget argument to the target that should be used within the
754// loop head, and removes that block as a successor to MBB.
755bool RISCVExpandPseudoAtomicsImpl::tryToFoldBNEOnCmpXchgResult(
756 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
757 Register CmpValReg, Register MaskReg,
758 MachineBasicBlock *&LoopHeadBNETarget) const {
760 auto E = MBB.end();
761 if (MBBI == E)
762 return false;
764
765 // If we have a masked cmpxchg, match AND dst, DestReg, MaskReg.
766 if (MaskReg.isValid()) {
767 if (MBBI == E || MBBI->getOpcode() != RISCV::AND)
768 return false;
769 Register ANDOp1 = MBBI->getOperand(1).getReg();
770 Register ANDOp2 = MBBI->getOperand(2).getReg();
771 if (!(ANDOp1 == DestReg && ANDOp2 == MaskReg) &&
772 !(ANDOp1 == MaskReg && ANDOp2 == DestReg))
773 return false;
774 // We now expect the BNE to use the result of the AND as an operand.
775 DestReg = MBBI->getOperand(0).getReg();
776 ToErase.push_back(&*MBBI);
778 }
779
780 // Match BNE DestReg, MaskReg.
781 if (MBBI == E || MBBI->getOpcode() != RISCV::BNE)
782 return false;
783 Register BNEOp0 = MBBI->getOperand(0).getReg();
784 Register BNEOp1 = MBBI->getOperand(1).getReg();
785 if (!(BNEOp0 == DestReg && BNEOp1 == CmpValReg) &&
786 !(BNEOp0 == CmpValReg && BNEOp1 == DestReg))
787 return false;
788
789 // Make sure the branch is the only user of the AND.
790 if (MaskReg.isValid()) {
791 if (BNEOp0 == DestReg && !MBBI->getOperand(0).isKill())
792 return false;
793 if (BNEOp1 == DestReg && !MBBI->getOperand(1).isKill())
794 return false;
795 }
796
797 ToErase.push_back(&*MBBI);
798 LoopHeadBNETarget = MBBI->getOperand(2).getMBB();
800 if (MBBI != E)
801 return false;
802
803 MBB.removeSuccessor(LoopHeadBNETarget);
804 for (auto *MI : ToErase)
805 MI->eraseFromParent();
806 return true;
807}
808
809bool RISCVExpandPseudoAtomicsImpl::expandAtomicCmpXchg(
810 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
811 int Width, MachineBasicBlock::iterator &NextMBBI) const {
812 MachineInstr &MI = *MBBI;
813 DebugLoc DL = MI.getDebugLoc();
815 auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
816 auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
817 auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
818
819 Register DestReg = MI.getOperand(0).getReg();
820 Register ScratchReg = MI.getOperand(1).getReg();
821 Register AddrReg = MI.getOperand(2).getReg();
822 Register CmpValReg = MI.getOperand(3).getReg();
823 Register NewValReg = MI.getOperand(4).getReg();
824 Register MaskReg = IsMasked ? MI.getOperand(5).getReg() : Register();
825
826 MachineBasicBlock *LoopHeadBNETarget = DoneMBB;
827 tryToFoldBNEOnCmpXchgResult(MBB, std::next(MBBI), DestReg, CmpValReg, MaskReg,
828 LoopHeadBNETarget);
829
830 // Insert new MBBs.
831 MF->insert(++MBB.getIterator(), LoopHeadMBB);
832 MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
833 MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
834
835 // Set up successors and transfer remaining instructions to DoneMBB.
836 LoopHeadMBB->addSuccessor(LoopTailMBB);
837 LoopHeadMBB->addSuccessor(LoopHeadBNETarget);
838 LoopTailMBB->addSuccessor(DoneMBB);
839 LoopTailMBB->addSuccessor(LoopHeadMBB);
840 DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
841 DoneMBB->transferSuccessors(&MBB);
842 MBB.addSuccessor(LoopHeadMBB);
843
845 static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());
846
847 if (!IsMasked) {
848 // .loophead:
849 // lr.[w|d] dest, (addr)
850 // bne dest, cmpval, done
851 BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
852 .addReg(AddrReg);
853 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
854 .addReg(DestReg)
855 .addReg(CmpValReg)
856 .addMBB(LoopHeadBNETarget);
857 // .looptail:
858 // sc.[w|d] scratch, newval, (addr)
859 // bnez scratch, loophead
860 BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
861 .addReg(NewValReg)
862 .addReg(AddrReg);
863 BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
864 .addReg(ScratchReg)
865 .addReg(RISCV::X0)
866 .addMBB(LoopHeadMBB);
867 } else {
868 // .loophead:
869 // lr.w dest, (addr)
870 // and scratch, dest, mask
871 // bne scratch, cmpval, done
872 Register MaskReg = MI.getOperand(5).getReg();
873 BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
874 .addReg(AddrReg);
875 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
876 .addReg(DestReg)
877 .addReg(MaskReg);
878 BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
879 .addReg(ScratchReg)
880 .addReg(CmpValReg)
881 .addMBB(LoopHeadBNETarget);
882
883 // .looptail:
884 // xor scratch, dest, newval
885 // and scratch, scratch, mask
886 // xor scratch, dest, scratch
887 // sc.w scratch, scratch, (adrr)
888 // bnez scratch, loophead
889 // A zero newval needs only the last xor: scratch still holds dest & mask
890 // from .loophead.
891 if (NewValReg == RISCV::X0)
892 BuildMI(LoopTailMBB, DL, TII->get(RISCV::XOR), ScratchReg)
893 .addReg(DestReg)
894 .addReg(ScratchReg);
895 else
896 insertMaskedMerge(DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
897 MaskReg, ScratchReg);
898 BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
899 .addReg(ScratchReg)
900 .addReg(AddrReg);
901 BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
902 .addReg(ScratchReg)
903 .addReg(RISCV::X0)
904 .addMBB(LoopHeadMBB);
905 }
906
907 NextMBBI = MBB.end();
908 MI.eraseFromParent();
909
910 LivePhysRegs LiveRegs;
911 computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
912 computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
913 computeAndAddLiveIns(LiveRegs, *DoneMBB);
914
915 return true;
916}
917
918char RISCVExpandPseudoAtomicsLegacy::ID = 0;
919
920INITIALIZE_PASS(RISCVExpandPseudoAtomicsLegacy, "riscv-expand-pseudo-atomics",
922
924 return new RISCVExpandPseudoAtomicsLegacy();
925}
926
927PreservedAnalyses
930 bool Changed = RISCVExpandPseudoAtomicsImpl().run(MF);
931 if (!Changed)
932 return PreservedAnalyses::all();
934}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
Promote Memory to Register
Definition Mem2Reg.cpp:110
static bool expandAtomicBinOp(MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI, unsigned Size, const MipsSubtarget *STI, const MipsInstrInfo *TII)
static bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineBasicBlock::iterator &NMBB, const MipsSubtarget *STI, const MipsInstrInfo *TII)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
#define RISCV_EXPAND_PSEUDO_ATOMICS_NAME
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Add
*p = old + v
@ Min
*p = old <signed v ? old : v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ Nand
*p = ~(old & v)
A debug info location.
Definition DebugLoc.h:126
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
void push_back(const T &Elt)
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createRISCVExpandPseudoAtomicsLegacyPass()
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
AtomicOrdering
Atomic ordering for LLVM's memory model.
LLVM_ABI void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().