LLVM 19.0.0git
MipsExpandPseudo.cpp
Go to the documentation of this file.
1//===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling, if-conversion, and other late
11// optimizations. This pass should be run after register allocation but before
12// the post-regalloc scheduling pass.
13//
14// This is currently only used for expanding atomic pseudos after register
15// allocation. We do this to avoid the fast register allocator introducing
16// spills between ll and sc. These stores cause some MIPS implementations to
17// abort the atomic RMW sequence.
18//
19//===----------------------------------------------------------------------===//
20
21#include "Mips.h"
22#include "MipsInstrInfo.h"
23#include "MipsSubtarget.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "mips-pseudo"
31
32namespace {
33 class MipsExpandPseudo : public MachineFunctionPass {
34 public:
35 static char ID;
36 MipsExpandPseudo() : MachineFunctionPass(ID) {}
37
38 const MipsInstrInfo *TII;
39 const MipsSubtarget *STI;
40
41 bool runOnMachineFunction(MachineFunction &Fn) override;
42
45 MachineFunctionProperties::Property::NoVRegs);
46 }
47
48 StringRef getPassName() const override {
49 return "Mips pseudo instruction expansion pass";
50 }
51
52 private:
53 bool expandAtomicCmpSwap(MachineBasicBlock &MBB,
56 bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB,
59
60 bool expandAtomicBinOp(MachineBasicBlock &BB,
62 MachineBasicBlock::iterator &NMBBI, unsigned Size);
63 bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
66
69 bool expandMBB(MachineBasicBlock &MBB);
70 };
71 char MipsExpandPseudo::ID = 0;
72}
73
74bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
77
78 MachineFunction *MF = BB.getParent();
79
80 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
81 DebugLoc DL = I->getDebugLoc();
82 unsigned LL, SC;
83
84 unsigned ZERO = Mips::ZERO;
85 unsigned BNE = Mips::BNE;
86 unsigned BEQ = Mips::BEQ;
87 unsigned SEOp =
88 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
89
90 if (STI->inMicroMipsMode()) {
91 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
92 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
93 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
94 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
95 } else {
96 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
97 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
98 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
99 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
100 }
101
102 Register Dest = I->getOperand(0).getReg();
103 Register Ptr = I->getOperand(1).getReg();
104 Register Mask = I->getOperand(2).getReg();
105 Register ShiftCmpVal = I->getOperand(3).getReg();
106 Register Mask2 = I->getOperand(4).getReg();
107 Register ShiftNewVal = I->getOperand(5).getReg();
108 Register ShiftAmnt = I->getOperand(6).getReg();
109 Register Scratch = I->getOperand(7).getReg();
110 Register Scratch2 = I->getOperand(8).getReg();
111
112 // insert new blocks after the current block
113 const BasicBlock *LLVM_BB = BB.getBasicBlock();
114 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
115 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
116 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
117 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
119 MF->insert(It, loop1MBB);
120 MF->insert(It, loop2MBB);
121 MF->insert(It, sinkMBB);
122 MF->insert(It, exitMBB);
123
124 // Transfer the remainder of BB and its successor edges to exitMBB.
125 exitMBB->splice(exitMBB->begin(), &BB,
126 std::next(MachineBasicBlock::iterator(I)), BB.end());
128
129 // thisMBB:
130 // ...
131 // fallthrough --> loop1MBB
133 loop1MBB->addSuccessor(sinkMBB);
134 loop1MBB->addSuccessor(loop2MBB);
135 loop1MBB->normalizeSuccProbs();
136 loop2MBB->addSuccessor(loop1MBB);
137 loop2MBB->addSuccessor(sinkMBB);
138 loop2MBB->normalizeSuccProbs();
139 sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
140
141 // loop1MBB:
142 // ll dest, 0(ptr)
143 // and Mask', dest, Mask
144 // bne Mask', ShiftCmpVal, exitMBB
145 BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
146 BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
147 .addReg(Scratch)
148 .addReg(Mask);
149 BuildMI(loop1MBB, DL, TII->get(BNE))
150 .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
151
152 // loop2MBB:
153 // and dest, dest, mask2
154 // or dest, dest, ShiftNewVal
155 // sc dest, dest, 0(ptr)
156 // beq dest, $0, loop1MBB
157 BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
158 .addReg(Scratch, RegState::Kill)
159 .addReg(Mask2);
160 BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
161 .addReg(Scratch, RegState::Kill)
162 .addReg(ShiftNewVal);
163 BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
164 .addReg(Scratch, RegState::Kill)
165 .addReg(Ptr)
166 .addImm(0);
167 BuildMI(loop2MBB, DL, TII->get(BEQ))
168 .addReg(Scratch, RegState::Kill)
169 .addReg(ZERO)
170 .addMBB(loop1MBB);
171
172 // sinkMBB:
173 // srl srlres, Mask', shiftamt
174 // sign_extend dest,srlres
175 BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
176 .addReg(Scratch2)
177 .addReg(ShiftAmnt);
178 if (STI->hasMips32r2()) {
179 BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
180 } else {
181 const unsigned ShiftImm =
182 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
183 BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
184 .addReg(Dest, RegState::Kill)
185 .addImm(ShiftImm);
186 BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
187 .addReg(Dest, RegState::Kill)
188 .addImm(ShiftImm);
189 }
190
191 LivePhysRegs LiveRegs;
192 computeAndAddLiveIns(LiveRegs, *loop1MBB);
193 computeAndAddLiveIns(LiveRegs, *loop2MBB);
194 computeAndAddLiveIns(LiveRegs, *sinkMBB);
195 computeAndAddLiveIns(LiveRegs, *exitMBB);
196
197 NMBBI = BB.end();
198 I->eraseFromParent();
199 return true;
200}
201
202bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
205
206 const unsigned Size =
207 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
208 MachineFunction *MF = BB.getParent();
209
210 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
211 DebugLoc DL = I->getDebugLoc();
212
213 unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
214
215 if (Size == 4) {
216 if (STI->inMicroMipsMode()) {
217 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
218 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
219 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
220 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
221 } else {
222 LL = STI->hasMips32r6()
223 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
224 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
225 SC = STI->hasMips32r6()
226 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
227 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
228 BNE = Mips::BNE;
229 BEQ = Mips::BEQ;
230 }
231
232 ZERO = Mips::ZERO;
233 MOVE = Mips::OR;
234 } else {
235 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
236 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
237 ZERO = Mips::ZERO_64;
238 BNE = Mips::BNE64;
239 BEQ = Mips::BEQ64;
240 MOVE = Mips::OR64;
241 }
242
243 Register Dest = I->getOperand(0).getReg();
244 Register Ptr = I->getOperand(1).getReg();
245 Register OldVal = I->getOperand(2).getReg();
246 Register NewVal = I->getOperand(3).getReg();
247 Register Scratch = I->getOperand(4).getReg();
248
249 // insert new blocks after the current block
250 const BasicBlock *LLVM_BB = BB.getBasicBlock();
251 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
252 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
253 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
255 MF->insert(It, loop1MBB);
256 MF->insert(It, loop2MBB);
257 MF->insert(It, exitMBB);
258
259 // Transfer the remainder of BB and its successor edges to exitMBB.
260 exitMBB->splice(exitMBB->begin(), &BB,
261 std::next(MachineBasicBlock::iterator(I)), BB.end());
263
264 // thisMBB:
265 // ...
266 // fallthrough --> loop1MBB
268 loop1MBB->addSuccessor(exitMBB);
269 loop1MBB->addSuccessor(loop2MBB);
270 loop1MBB->normalizeSuccProbs();
271 loop2MBB->addSuccessor(loop1MBB);
272 loop2MBB->addSuccessor(exitMBB);
273 loop2MBB->normalizeSuccProbs();
274
275 // loop1MBB:
276 // ll dest, 0(ptr)
277 // bne dest, oldval, exitMBB
278 BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
279 BuildMI(loop1MBB, DL, TII->get(BNE))
280 .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
281
282 // loop2MBB:
283 // move scratch, NewVal
284 // sc Scratch, Scratch, 0(ptr)
285 // beq Scratch, $0, loop1MBB
286 BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
287 BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
288 .addReg(Scratch).addReg(Ptr).addImm(0);
289 BuildMI(loop2MBB, DL, TII->get(BEQ))
290 .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
291
292 LivePhysRegs LiveRegs;
293 computeAndAddLiveIns(LiveRegs, *loop1MBB);
294 computeAndAddLiveIns(LiveRegs, *loop2MBB);
295 computeAndAddLiveIns(LiveRegs, *exitMBB);
296
297 NMBBI = BB.end();
298 I->eraseFromParent();
299 return true;
300}
301
302bool MipsExpandPseudo::expandAtomicBinOpSubword(
305
306 MachineFunction *MF = BB.getParent();
307
308 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
309 DebugLoc DL = I->getDebugLoc();
310
311 unsigned LL, SC, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
312 unsigned BEQ = Mips::BEQ;
313 unsigned SEOp = Mips::SEH;
314
315 if (STI->inMicroMipsMode()) {
316 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
317 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
318 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
319 SLT = Mips::SLT_MM;
320 SLTu = Mips::SLTu_MM;
321 OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
322 MOVN = Mips::MOVN_I_MM;
323 MOVZ = Mips::MOVZ_I_MM;
324 SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
325 SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
326 } else {
327 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
328 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
329 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
330 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
331 SLT = Mips::SLT;
332 SLTu = Mips::SLTu;
333 OR = Mips::OR;
334 MOVN = Mips::MOVN_I_I;
335 MOVZ = Mips::MOVZ_I_I;
336 SELNEZ = Mips::SELNEZ;
337 SELEQZ = Mips::SELEQZ;
338 }
339
340 bool IsSwap = false;
341 bool IsNand = false;
342 bool IsMin = false;
343 bool IsMax = false;
344 bool IsUnsigned = false;
345
346 unsigned Opcode = 0;
347 switch (I->getOpcode()) {
348 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
349 SEOp = Mips::SEB;
350 [[fallthrough]];
351 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
352 IsNand = true;
353 break;
354 case Mips::ATOMIC_SWAP_I8_POSTRA:
355 SEOp = Mips::SEB;
356 [[fallthrough]];
357 case Mips::ATOMIC_SWAP_I16_POSTRA:
358 IsSwap = true;
359 break;
360 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
361 SEOp = Mips::SEB;
362 [[fallthrough]];
363 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
364 Opcode = Mips::ADDu;
365 break;
366 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
367 SEOp = Mips::SEB;
368 [[fallthrough]];
369 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
370 Opcode = Mips::SUBu;
371 break;
372 case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
373 SEOp = Mips::SEB;
374 [[fallthrough]];
375 case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
376 Opcode = Mips::AND;
377 break;
378 case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
379 SEOp = Mips::SEB;
380 [[fallthrough]];
381 case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
382 Opcode = Mips::OR;
383 break;
384 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
385 SEOp = Mips::SEB;
386 [[fallthrough]];
387 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
388 Opcode = Mips::XOR;
389 break;
390 case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
391 IsUnsigned = true;
392 IsMin = true;
393 break;
394 case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
395 IsUnsigned = true;
396 IsMin = true;
397 break;
398 case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
399 SEOp = Mips::SEB;
400 IsMin = true;
401 break;
402 case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
403 IsMin = true;
404 break;
405 case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
406 IsUnsigned = true;
407 IsMax = true;
408 break;
409 case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
410 IsUnsigned = true;
411 IsMax = true;
412 break;
413 case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
414 SEOp = Mips::SEB;
415 IsMax = true;
416 break;
417 case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
418 IsMax = true;
419 break;
420 default:
421 llvm_unreachable("Unknown subword atomic pseudo for expansion!");
422 }
423
424 Register Dest = I->getOperand(0).getReg();
425 Register Ptr = I->getOperand(1).getReg();
426 Register Incr = I->getOperand(2).getReg();
427 Register Mask = I->getOperand(3).getReg();
428 Register Mask2 = I->getOperand(4).getReg();
429 Register ShiftAmnt = I->getOperand(5).getReg();
430 Register OldVal = I->getOperand(6).getReg();
431 Register BinOpRes = I->getOperand(7).getReg();
432 Register StoreVal = I->getOperand(8).getReg();
433
434 const BasicBlock *LLVM_BB = BB.getBasicBlock();
435 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
436 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
437 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
439 MF->insert(It, loopMBB);
440 MF->insert(It, sinkMBB);
441 MF->insert(It, exitMBB);
442
443 exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
445
447 loopMBB->addSuccessor(sinkMBB);
448 loopMBB->addSuccessor(loopMBB);
449 loopMBB->normalizeSuccProbs();
450
451 BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
452 if (IsNand) {
453 // and andres, oldval, incr2
454 // nor binopres, $0, andres
455 // and newval, binopres, mask
456 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
457 .addReg(OldVal)
458 .addReg(Incr);
459 BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
460 .addReg(Mips::ZERO)
461 .addReg(BinOpRes);
462 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
463 .addReg(BinOpRes)
464 .addReg(Mask);
465 } else if (IsMin || IsMax) {
466
467 assert(I->getNumOperands() == 10 &&
468 "Atomics min|max|umin|umax use an additional register");
469 Register Scratch4 = I->getOperand(9).getReg();
470
471 unsigned SLTScratch4 = IsUnsigned ? SLTu : SLT;
472 unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
473 unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
474 unsigned MOVIncr = IsMax ? MOVN : MOVZ;
475
476 // For little endian we need to clear uninterested bits.
477 if (STI->isLittle()) {
478 if (!IsUnsigned) {
479 BuildMI(loopMBB, DL, TII->get(Mips::SRAV), OldVal)
480 .addReg(OldVal)
481 .addReg(ShiftAmnt);
482 BuildMI(loopMBB, DL, TII->get(Mips::SRAV), Incr)
483 .addReg(Incr)
484 .addReg(ShiftAmnt);
485 if (STI->hasMips32r2()) {
486 BuildMI(loopMBB, DL, TII->get(SEOp), OldVal).addReg(OldVal);
487 BuildMI(loopMBB, DL, TII->get(SEOp), Incr).addReg(Incr);
488 } else {
489 const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
490 BuildMI(loopMBB, DL, TII->get(Mips::SLL), OldVal)
491 .addReg(OldVal, RegState::Kill)
492 .addImm(ShiftImm);
493 BuildMI(loopMBB, DL, TII->get(Mips::SRA), OldVal)
494 .addReg(OldVal, RegState::Kill)
495 .addImm(ShiftImm);
496 BuildMI(loopMBB, DL, TII->get(Mips::SLL), Incr)
497 .addReg(Incr, RegState::Kill)
498 .addImm(ShiftImm);
499 BuildMI(loopMBB, DL, TII->get(Mips::SRA), Incr)
500 .addReg(Incr, RegState::Kill)
501 .addImm(ShiftImm);
502 }
503 } else {
504 // and OldVal, OldVal, Mask
505 // and Incr, Incr, Mask
506 BuildMI(loopMBB, DL, TII->get(Mips::AND), OldVal)
507 .addReg(OldVal)
508 .addReg(Mask);
509 BuildMI(loopMBB, DL, TII->get(Mips::AND), Incr)
510 .addReg(Incr)
511 .addReg(Mask);
512 }
513 }
514 // unsigned: sltu Scratch4, oldVal, Incr
515 // signed: slt Scratch4, oldVal, Incr
516 BuildMI(loopMBB, DL, TII->get(SLTScratch4), Scratch4)
517 .addReg(OldVal)
518 .addReg(Incr);
519
520 if (STI->hasMips64r6() || STI->hasMips32r6()) {
521 // max: seleqz BinOpRes, OldVal, Scratch4
522 // selnez Scratch4, Incr, Scratch4
523 // or BinOpRes, BinOpRes, Scratch4
524 // min: selnqz BinOpRes, OldVal, Scratch4
525 // seleqz Scratch4, Incr, Scratch4
526 // or BinOpRes, BinOpRes, Scratch4
527 BuildMI(loopMBB, DL, TII->get(SELOldVal), BinOpRes)
528 .addReg(OldVal)
529 .addReg(Scratch4);
530 BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch4)
531 .addReg(Incr)
532 .addReg(Scratch4);
533 BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
534 .addReg(BinOpRes)
535 .addReg(Scratch4);
536 } else {
537 // max: move BinOpRes, OldVal
538 // movn BinOpRes, Incr, Scratch4, BinOpRes
539 // min: move BinOpRes, OldVal
540 // movz BinOpRes, Incr, Scratch4, BinOpRes
541 BuildMI(loopMBB, DL, TII->get(OR), BinOpRes)
542 .addReg(OldVal)
543 .addReg(Mips::ZERO);
544 BuildMI(loopMBB, DL, TII->get(MOVIncr), BinOpRes)
545 .addReg(Incr)
546 .addReg(Scratch4)
547 .addReg(BinOpRes);
548 }
549
550 // and BinOpRes, BinOpRes, Mask
551 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
552 .addReg(BinOpRes)
553 .addReg(Mask);
554
555 } else if (!IsSwap) {
556 // <binop> binopres, oldval, incr2
557 // and newval, binopres, mask
558 BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
559 .addReg(OldVal)
560 .addReg(Incr);
561 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
562 .addReg(BinOpRes)
563 .addReg(Mask);
564 } else { // atomic.swap
565 // and newval, incr2, mask
566 BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
567 .addReg(Incr)
568 .addReg(Mask);
569 }
570
571 // and StoreVal, OlddVal, Mask2
572 // or StoreVal, StoreVal, BinOpRes
573 // StoreVal<tied1> = sc StoreVal, 0(Ptr)
574 // beq StoreVal, zero, loopMBB
575 BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
576 .addReg(OldVal).addReg(Mask2);
577 BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
578 .addReg(StoreVal).addReg(BinOpRes);
579 BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
580 .addReg(StoreVal).addReg(Ptr).addImm(0);
581 BuildMI(loopMBB, DL, TII->get(BEQ))
582 .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
583
584 // sinkMBB:
585 // and maskedoldval1,oldval,mask
586 // srl srlres,maskedoldval1,shiftamt
587 // sign_extend dest,srlres
588
589 sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
590
591 BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest)
592 .addReg(OldVal).addReg(Mask);
593 BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
594 .addReg(Dest).addReg(ShiftAmnt);
595
596 if (STI->hasMips32r2()) {
597 BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
598 } else {
599 const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
600 BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
601 .addReg(Dest, RegState::Kill)
602 .addImm(ShiftImm);
603 BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
604 .addReg(Dest, RegState::Kill)
605 .addImm(ShiftImm);
606 }
607
608 LivePhysRegs LiveRegs;
609 computeAndAddLiveIns(LiveRegs, *loopMBB);
610 computeAndAddLiveIns(LiveRegs, *sinkMBB);
611 computeAndAddLiveIns(LiveRegs, *exitMBB);
612
613 NMBBI = BB.end();
614 I->eraseFromParent();
615
616 return true;
617}
618
619bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
622 unsigned Size) {
623 MachineFunction *MF = BB.getParent();
624
625 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
626 DebugLoc DL = I->getDebugLoc();
627
628 unsigned LL, SC, ZERO, BEQ, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
629
630 if (Size == 4) {
631 if (STI->inMicroMipsMode()) {
632 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
633 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
634 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
635 SLT = Mips::SLT_MM;
636 SLTu = Mips::SLTu_MM;
637 OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
638 MOVN = Mips::MOVN_I_MM;
639 MOVZ = Mips::MOVZ_I_MM;
640 SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
641 SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
642 } else {
643 LL = STI->hasMips32r6()
644 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
645 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
646 SC = STI->hasMips32r6()
647 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
648 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
649 BEQ = Mips::BEQ;
650 SLT = Mips::SLT;
651 SLTu = Mips::SLTu;
652 OR = Mips::OR;
653 MOVN = Mips::MOVN_I_I;
654 MOVZ = Mips::MOVZ_I_I;
655 SELNEZ = Mips::SELNEZ;
656 SELEQZ = Mips::SELEQZ;
657 }
658
659 ZERO = Mips::ZERO;
660 } else {
661 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
662 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
663 ZERO = Mips::ZERO_64;
664 BEQ = Mips::BEQ64;
665 SLT = Mips::SLT64;
666 SLTu = Mips::SLTu64;
667 OR = Mips::OR64;
668 MOVN = Mips::MOVN_I64_I64;
669 MOVZ = Mips::MOVZ_I64_I64;
670 SELNEZ = Mips::SELNEZ64;
671 SELEQZ = Mips::SELEQZ64;
672 }
673
674 Register OldVal = I->getOperand(0).getReg();
675 Register Ptr = I->getOperand(1).getReg();
676 Register Incr = I->getOperand(2).getReg();
677 Register Scratch = I->getOperand(3).getReg();
678
679 unsigned Opcode = 0;
680 unsigned AND = 0;
681 unsigned NOR = 0;
682
683 bool IsOr = false;
684 bool IsNand = false;
685 bool IsMin = false;
686 bool IsMax = false;
687 bool IsUnsigned = false;
688
689 switch (I->getOpcode()) {
690 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
691 Opcode = Mips::ADDu;
692 break;
693 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
694 Opcode = Mips::SUBu;
695 break;
696 case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
697 Opcode = Mips::AND;
698 break;
699 case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
700 Opcode = Mips::OR;
701 break;
702 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
703 Opcode = Mips::XOR;
704 break;
705 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
706 IsNand = true;
707 AND = Mips::AND;
708 NOR = Mips::NOR;
709 break;
710 case Mips::ATOMIC_SWAP_I32_POSTRA:
711 IsOr = true;
712 break;
713 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
714 Opcode = Mips::DADDu;
715 break;
716 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
717 Opcode = Mips::DSUBu;
718 break;
719 case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
720 Opcode = Mips::AND64;
721 break;
722 case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
723 Opcode = Mips::OR64;
724 break;
725 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
726 Opcode = Mips::XOR64;
727 break;
728 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
729 IsNand = true;
730 AND = Mips::AND64;
731 NOR = Mips::NOR64;
732 break;
733 case Mips::ATOMIC_SWAP_I64_POSTRA:
734 IsOr = true;
735 break;
736 case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
737 case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
738 IsUnsigned = true;
739 [[fallthrough]];
740 case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
741 case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
742 IsMin = true;
743 break;
744 case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
745 case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
746 IsUnsigned = true;
747 [[fallthrough]];
748 case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
749 case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
750 IsMax = true;
751 break;
752 default:
753 llvm_unreachable("Unknown pseudo atomic!");
754 }
755
756 const BasicBlock *LLVM_BB = BB.getBasicBlock();
757 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
758 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
760 MF->insert(It, loopMBB);
761 MF->insert(It, exitMBB);
762
763 exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
765
767 loopMBB->addSuccessor(exitMBB);
768 loopMBB->addSuccessor(loopMBB);
769 loopMBB->normalizeSuccProbs();
770
771 BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
772 assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
773 assert((OldVal != Incr) && "Clobbered the wrong reg!");
774 if (IsMin || IsMax) {
775
776 assert(I->getNumOperands() == 5 &&
777 "Atomics min|max|umin|umax use an additional register");
778 MCRegister Scratch2 = I->getOperand(4).getReg().asMCReg();
779
780 // On Mips64 result of slt is GPR32.
781 MCRegister Scratch2_32 =
782 (Size == 8) ? STI->getRegisterInfo()->getSubReg(Scratch2, Mips::sub_32)
783 : Scratch2;
784
785 unsigned SLTScratch2 = IsUnsigned ? SLTu : SLT;
786 unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
787 unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
788 unsigned MOVIncr = IsMax ? MOVN : MOVZ;
789
790 // unsigned: sltu Scratch2, oldVal, Incr
791 // signed: slt Scratch2, oldVal, Incr
792 BuildMI(loopMBB, DL, TII->get(SLTScratch2), Scratch2_32)
793 .addReg(OldVal)
794 .addReg(Incr);
795
796 if (STI->hasMips64r6() || STI->hasMips32r6()) {
797 // max: seleqz Scratch, OldVal, Scratch2
798 // selnez Scratch2, Incr, Scratch2
799 // or Scratch, Scratch, Scratch2
800 // min: selnez Scratch, OldVal, Scratch2
801 // seleqz Scratch2, Incr, Scratch2
802 // or Scratch, Scratch, Scratch2
803 BuildMI(loopMBB, DL, TII->get(SELOldVal), Scratch)
804 .addReg(OldVal)
805 .addReg(Scratch2);
806 BuildMI(loopMBB, DL, TII->get(SELIncr), Scratch2)
807 .addReg(Incr)
808 .addReg(Scratch2);
809 BuildMI(loopMBB, DL, TII->get(OR), Scratch)
810 .addReg(Scratch)
811 .addReg(Scratch2);
812 } else {
813 // max: move Scratch, OldVal
814 // movn Scratch, Incr, Scratch2, Scratch
815 // min: move Scratch, OldVal
816 // movz Scratch, Incr, Scratch2, Scratch
817 BuildMI(loopMBB, DL, TII->get(OR), Scratch)
818 .addReg(OldVal)
819 .addReg(ZERO);
820 BuildMI(loopMBB, DL, TII->get(MOVIncr), Scratch)
821 .addReg(Incr)
822 .addReg(Scratch2)
823 .addReg(Scratch);
824 }
825
826 } else if (Opcode) {
827 BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
828 } else if (IsNand) {
829 assert(AND && NOR &&
830 "Unknown nand instruction for atomic pseudo expansion");
831 BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
832 BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
833 } else {
834 assert(IsOr && OR && "Unknown instruction for atomic pseudo expansion!");
835 (void)IsOr;
836 BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
837 }
838
839 BuildMI(loopMBB, DL, TII->get(SC), Scratch)
840 .addReg(Scratch)
841 .addReg(Ptr)
842 .addImm(0);
843 BuildMI(loopMBB, DL, TII->get(BEQ))
844 .addReg(Scratch)
845 .addReg(ZERO)
846 .addMBB(loopMBB);
847
848 NMBBI = BB.end();
849 I->eraseFromParent();
850
851 LivePhysRegs LiveRegs;
852 computeAndAddLiveIns(LiveRegs, *loopMBB);
853 computeAndAddLiveIns(LiveRegs, *exitMBB);
854
855 return true;
856}
857
858bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB,
861
862 bool Modified = false;
863
864 switch (MBBI->getOpcode()) {
865 case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
866 case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
867 return expandAtomicCmpSwap(MBB, MBBI, NMBB);
868 case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
869 case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
870 return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB);
871 case Mips::ATOMIC_SWAP_I8_POSTRA:
872 case Mips::ATOMIC_SWAP_I16_POSTRA:
873 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
874 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
875 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
876 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
877 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
878 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
879 case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
880 case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
881 case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
882 case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
883 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
884 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
885 case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
886 case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
887 case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
888 case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
889 case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
890 case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
891 case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
892 case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
893 return expandAtomicBinOpSubword(MBB, MBBI, NMBB);
894 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
895 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
896 case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
897 case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
898 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
899 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
900 case Mips::ATOMIC_SWAP_I32_POSTRA:
901 case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
902 case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
903 case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
904 case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
905 return expandAtomicBinOp(MBB, MBBI, NMBB, 4);
906 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
907 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
908 case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
909 case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
910 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
911 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
912 case Mips::ATOMIC_SWAP_I64_POSTRA:
913 case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
914 case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
915 case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
916 case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
917 return expandAtomicBinOp(MBB, MBBI, NMBB, 8);
918 default:
919 return Modified;
920 }
921}
922
923bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
924 bool Modified = false;
925
927 while (MBBI != E) {
928 MachineBasicBlock::iterator NMBBI = std::next(MBBI);
929 Modified |= expandMI(MBB, MBBI, NMBBI);
930 MBBI = NMBBI;
931 }
932
933 return Modified;
934}
935
936bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
937 STI = &MF.getSubtarget<MipsSubtarget>();
938 TII = STI->getInstrInfo();
939
940 bool Modified = false;
941 for (MachineBasicBlock &MBB : MF)
942 Modified |= expandMBB(MBB);
943
944 if (Modified)
945 MF.RenumberBlocks();
946
947 return Modified;
948}
949
950/// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
951/// expansion pass.
953 return new MipsExpandPseudo();
954}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
@ ZERO
Special weight used for cases with exact zero probability.
uint64_t Size
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
static BranchProbability getOne()
A debug info location.
Definition: DebugLoc.h:33
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:50
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
self_iterator getIterator()
Definition: ilist_node.h:109
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
@ Kill
The last use of a register.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
FunctionPass * createMipsExpandPseudoPass()
createMipsExpandPseudoPass - returns an instance of the pseudo instruction expansion pass.