LLVM  14.0.0git
PPCExpandAtomicPseudoInsts.cpp
Go to the documentation of this file.
1 //===-- PPCExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands atomic pseudo instructions into
10 // target instructions post RA. With such method, LL/SC loop is considered as
11 // a whole blob and make spilling unlikely happens in the LL/SC loop.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "PPC.h"
17 #include "PPCInstrInfo.h"
18 #include "PPCTargetMachine.h"
19 
23 
24 using namespace llvm;
25 
26 #define DEBUG_TYPE "ppc-atomic-expand"
27 
28 namespace {
29 
30 class PPCExpandAtomicPseudo : public MachineFunctionPass {
31 public:
32  const PPCInstrInfo *TII;
33  const PPCRegisterInfo *TRI;
34  static char ID;
35 
36  PPCExpandAtomicPseudo() : MachineFunctionPass(ID) {
38  }
39 
40  bool runOnMachineFunction(MachineFunction &MF) override;
41 
42 private:
43  bool expandMI(MachineBasicBlock &MBB, MachineInstr &MI,
45  bool expandAtomicRMW128(MachineBasicBlock &MBB, MachineInstr &MI,
47  bool expandAtomicCmpSwap128(MachineBasicBlock &MBB, MachineInstr &MI,
49 };
50 
51 static void PairedCopy(const PPCInstrInfo *TII, MachineBasicBlock &MBB,
53  Register Dest0, Register Dest1, Register Src0,
54  Register Src1) {
55  const MCInstrDesc &OR = TII->get(PPC::OR8);
56  const MCInstrDesc &XOR = TII->get(PPC::XOR8);
57  if (Dest0 == Src1 && Dest1 == Src0) {
58  // The most tricky case, swapping values.
59  BuildMI(MBB, MBBI, DL, XOR, Dest0).addReg(Dest0).addReg(Dest1);
60  BuildMI(MBB, MBBI, DL, XOR, Dest1).addReg(Dest0).addReg(Dest1);
61  BuildMI(MBB, MBBI, DL, XOR, Dest0).addReg(Dest0).addReg(Dest1);
62  } else if (Dest0 != Src0 || Dest1 != Src1) {
63  if (Dest0 == Src1 || Dest1 != Src0) {
64  BuildMI(MBB, MBBI, DL, OR, Dest1).addReg(Src1).addReg(Src1);
65  BuildMI(MBB, MBBI, DL, OR, Dest0).addReg(Src0).addReg(Src0);
66  } else {
67  BuildMI(MBB, MBBI, DL, OR, Dest0).addReg(Src0).addReg(Src0);
68  BuildMI(MBB, MBBI, DL, OR, Dest1).addReg(Src1).addReg(Src1);
69  }
70  }
71 }
72 
73 bool PPCExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
74  bool Changed = false;
75  TII = static_cast<const PPCInstrInfo *>(MF.getSubtarget().getInstrInfo());
76  TRI = &TII->getRegisterInfo();
77  for (MachineBasicBlock &MBB : MF) {
78  for (MachineBasicBlock::iterator MBBI = MBB.begin(), MBBE = MBB.end();
79  MBBI != MBBE;) {
80  MachineInstr &MI = *MBBI;
81  MachineBasicBlock::iterator NMBBI = std::next(MBBI);
82  Changed |= expandMI(MBB, MI, NMBBI);
83  MBBI = NMBBI;
84  }
85  }
86  if (Changed)
87  MF.RenumberBlocks();
88  return Changed;
89 }
90 
91 bool PPCExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB, MachineInstr &MI,
93  switch (MI.getOpcode()) {
94  case PPC::ATOMIC_SWAP_I128:
95  case PPC::ATOMIC_LOAD_ADD_I128:
96  case PPC::ATOMIC_LOAD_SUB_I128:
97  case PPC::ATOMIC_LOAD_XOR_I128:
98  case PPC::ATOMIC_LOAD_NAND_I128:
99  case PPC::ATOMIC_LOAD_AND_I128:
100  case PPC::ATOMIC_LOAD_OR_I128:
101  return expandAtomicRMW128(MBB, MI, NMBBI);
102  case PPC::ATOMIC_CMP_SWAP_I128:
103  return expandAtomicCmpSwap128(MBB, MI, NMBBI);
104  case PPC::BUILD_QUADWORD: {
105  Register Dst = MI.getOperand(0).getReg();
106  Register DstHi = TRI->getSubReg(Dst, PPC::sub_gp8_x0);
107  Register DstLo = TRI->getSubReg(Dst, PPC::sub_gp8_x1);
108  Register Lo = MI.getOperand(1).getReg();
109  Register Hi = MI.getOperand(2).getReg();
110  PairedCopy(TII, MBB, MI, MI.getDebugLoc(), DstHi, DstLo, Hi, Lo);
111  MI.eraseFromParent();
112  return true;
113  }
114  default:
115  return false;
116  }
117 }
118 
119 bool PPCExpandAtomicPseudo::expandAtomicRMW128(
122  const MCInstrDesc &LL = TII->get(PPC::LQARX);
123  const MCInstrDesc &SC = TII->get(PPC::STQCX);
124  DebugLoc DL = MI.getDebugLoc();
125  MachineFunction *MF = MBB.getParent();
126  const BasicBlock *BB = MBB.getBasicBlock();
127  // Create layout of control flow.
131  MF->insert(MFI, LoopMBB);
132  MF->insert(MFI, ExitMBB);
133  ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()),
134  MBB.end());
136  MBB.addSuccessor(LoopMBB);
137 
138  // For non-min/max operations, control flow is kinda like:
139  // MBB:
140  // ...
141  // LoopMBB:
142  // lqarx in, ptr
143  // addc out.sub_x1, in.sub_x1, op.sub_x1
144  // adde out.sub_x0, in.sub_x0, op.sub_x0
145  // stqcx out, ptr
146  // bne- LoopMBB
147  // ExitMBB:
148  // ...
149  Register Old = MI.getOperand(0).getReg();
150  Register OldHi = TRI->getSubReg(Old, PPC::sub_gp8_x0);
151  Register OldLo = TRI->getSubReg(Old, PPC::sub_gp8_x1);
152  Register Scratch = MI.getOperand(1).getReg();
153  Register ScratchHi = TRI->getSubReg(Scratch, PPC::sub_gp8_x0);
154  Register ScratchLo = TRI->getSubReg(Scratch, PPC::sub_gp8_x1);
155  Register RA = MI.getOperand(2).getReg();
156  Register RB = MI.getOperand(3).getReg();
157  Register IncrLo = MI.getOperand(4).getReg();
158  Register IncrHi = MI.getOperand(5).getReg();
159  unsigned RMWOpcode = MI.getOpcode();
160 
161  MachineBasicBlock *CurrentMBB = LoopMBB;
162  BuildMI(CurrentMBB, DL, LL, Old).addReg(RA).addReg(RB);
163 
164  switch (RMWOpcode) {
165  case PPC::ATOMIC_SWAP_I128:
166  PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo,
167  IncrHi, IncrLo);
168  break;
169  case PPC::ATOMIC_LOAD_ADD_I128:
170  BuildMI(CurrentMBB, DL, TII->get(PPC::ADDC8), ScratchLo)
171  .addReg(IncrLo)
172  .addReg(OldLo);
173  BuildMI(CurrentMBB, DL, TII->get(PPC::ADDE8), ScratchHi)
174  .addReg(IncrHi)
175  .addReg(OldHi);
176  break;
177  case PPC::ATOMIC_LOAD_SUB_I128:
178  BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFC8), ScratchLo)
179  .addReg(IncrLo)
180  .addReg(OldLo);
181  BuildMI(CurrentMBB, DL, TII->get(PPC::SUBFE8), ScratchHi)
182  .addReg(IncrHi)
183  .addReg(OldHi);
184  break;
185 
186 #define TRIVIAL_ATOMICRMW(Opcode, Instr) \
187  case Opcode: \
188  BuildMI(CurrentMBB, DL, TII->get((Instr)), ScratchLo) \
189  .addReg(IncrLo) \
190  .addReg(OldLo); \
191  BuildMI(CurrentMBB, DL, TII->get((Instr)), ScratchHi) \
192  .addReg(IncrHi) \
193  .addReg(OldHi); \
194  break
195 
196  TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_OR_I128, PPC::OR8);
197  TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_XOR_I128, PPC::XOR8);
198  TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_AND_I128, PPC::AND8);
199  TRIVIAL_ATOMICRMW(PPC::ATOMIC_LOAD_NAND_I128, PPC::NAND8);
200 #undef TRIVIAL_ATOMICRMW
201  default:
202  llvm_unreachable("Unhandled atomic RMW operation");
203  }
204  BuildMI(CurrentMBB, DL, SC).addReg(Scratch).addReg(RA).addReg(RB);
205  BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
207  .addReg(PPC::CR0)
208  .addMBB(LoopMBB);
209  CurrentMBB->addSuccessor(LoopMBB);
210  CurrentMBB->addSuccessor(ExitMBB);
211  recomputeLiveIns(*LoopMBB);
212  recomputeLiveIns(*ExitMBB);
213  NMBBI = MBB.end();
214  MI.eraseFromParent();
215  return true;
216 }
217 
218 bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128(
221  const MCInstrDesc &LL = TII->get(PPC::LQARX);
222  const MCInstrDesc &SC = TII->get(PPC::STQCX);
223  DebugLoc DL = MI.getDebugLoc();
224  MachineFunction *MF = MBB.getParent();
225  const BasicBlock *BB = MBB.getBasicBlock();
226  Register Old = MI.getOperand(0).getReg();
227  Register OldHi = TRI->getSubReg(Old, PPC::sub_gp8_x0);
228  Register OldLo = TRI->getSubReg(Old, PPC::sub_gp8_x1);
229  Register Scratch = MI.getOperand(1).getReg();
230  Register ScratchHi = TRI->getSubReg(Scratch, PPC::sub_gp8_x0);
231  Register ScratchLo = TRI->getSubReg(Scratch, PPC::sub_gp8_x1);
232  Register RA = MI.getOperand(2).getReg();
233  Register RB = MI.getOperand(3).getReg();
234  Register CmpLo = MI.getOperand(4).getReg();
235  Register CmpHi = MI.getOperand(5).getReg();
236  Register NewLo = MI.getOperand(6).getReg();
237  Register NewHi = MI.getOperand(7).getReg();
238  // Create layout of control flow.
239  // loop:
240  // old = lqarx ptr
241  // <compare old, cmp>
242  // bne 0, fail
243  // succ:
244  // stqcx new ptr
245  // bne 0, loop
246  // b exit
247  // fail:
248  // stqcx old ptr
249  // exit:
250  // ....
252  MachineBasicBlock *LoopCmpMBB = MF->CreateMachineBasicBlock(BB);
253  MachineBasicBlock *CmpSuccMBB = MF->CreateMachineBasicBlock(BB);
254  MachineBasicBlock *CmpFailMBB = MF->CreateMachineBasicBlock(BB);
256  MF->insert(MFI, LoopCmpMBB);
257  MF->insert(MFI, CmpSuccMBB);
258  MF->insert(MFI, CmpFailMBB);
259  MF->insert(MFI, ExitMBB);
260  ExitMBB->splice(ExitMBB->begin(), &MBB, std::next(MI.getIterator()),
261  MBB.end());
263  MBB.addSuccessor(LoopCmpMBB);
264  // Build loop.
265  MachineBasicBlock *CurrentMBB = LoopCmpMBB;
266  BuildMI(CurrentMBB, DL, LL, Old).addReg(RA).addReg(RB);
267  BuildMI(CurrentMBB, DL, TII->get(PPC::XOR8), ScratchLo)
268  .addReg(OldLo)
269  .addReg(CmpLo);
270  BuildMI(CurrentMBB, DL, TII->get(PPC::XOR8), ScratchHi)
271  .addReg(OldHi)
272  .addReg(CmpHi);
273  BuildMI(CurrentMBB, DL, TII->get(PPC::OR8_rec), ScratchLo)
274  .addReg(ScratchLo)
275  .addReg(ScratchHi);
276  BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
278  .addReg(PPC::CR0)
279  .addMBB(CmpFailMBB);
280  CurrentMBB->addSuccessor(CmpSuccMBB);
281  CurrentMBB->addSuccessor(CmpFailMBB);
282  // Build succ.
283  CurrentMBB = CmpSuccMBB;
284  PairedCopy(TII, *CurrentMBB, CurrentMBB->end(), DL, ScratchHi, ScratchLo,
285  NewHi, NewLo);
286  BuildMI(CurrentMBB, DL, SC).addReg(Scratch).addReg(RA).addReg(RB);
287  BuildMI(CurrentMBB, DL, TII->get(PPC::BCC))
289  .addReg(PPC::CR0)
290  .addMBB(LoopCmpMBB);
291  BuildMI(CurrentMBB, DL, TII->get(PPC::B)).addMBB(ExitMBB);
292  CurrentMBB->addSuccessor(LoopCmpMBB);
293  CurrentMBB->addSuccessor(ExitMBB);
294  CurrentMBB = CmpFailMBB;
295  BuildMI(CurrentMBB, DL, SC).addReg(Old).addReg(RA).addReg(RB);
296  CurrentMBB->addSuccessor(ExitMBB);
297 
298  recomputeLiveIns(*LoopCmpMBB);
299  recomputeLiveIns(*CmpSuccMBB);
300  recomputeLiveIns(*CmpFailMBB);
301  recomputeLiveIns(*ExitMBB);
302  NMBBI = MBB.end();
303  MI.eraseFromParent();
304  return true;
305 }
306 
307 } // namespace
308 
309 INITIALIZE_PASS(PPCExpandAtomicPseudo, DEBUG_TYPE, "PowerPC Expand Atomic",
310  false, false)
311 
312 char PPCExpandAtomicPseudo::ID = 0;
314  return new PPCExpandAtomicPseudo();
315 }
llvm::PPCRegisterInfo
Definition: PPCRegisterInfo.h:57
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:105
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:23
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:633
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:206
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::MipsISD::Lo
@ Lo
Definition: MipsISelLowering.h:79
llvm::recomputeLiveIns
static void recomputeLiveIns(MachineBasicBlock &MBB)
Convenience function for recomputing live-in's for MBB.
Definition: LivePhysRegs.h:196
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::PPCInstrInfo
Definition: PPCInstrInfo.h:191
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:842
DEBUG_TYPE
#define DEBUG_TYPE
Definition: PPCExpandAtomicPseudoInsts.cpp:26
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1559
llvm::initializePPCExpandAtomicPseudoPass
void initializePPCExpandAtomicPseudoPass(PassRegistry &)
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::MipsISD::Hi
@ Hi
Definition: MipsISelLowering.h:75
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:750
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:418
INITIALIZE_PASS
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:37
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:127
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:195
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
PPC.h
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:641
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
TRIVIAL_ATOMICRMW
#define TRIVIAL_ATOMICRMW(Opcode, Instr)
PPCInstrInfo.h
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:420
MachineFunctionPass.h
llvm::PPC::PRED_NE
@ PRED_NE
Definition: PPCPredicates.h:32
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:229
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
RA
SI optimize exec mask operations pre RA
Definition: SIOptimizeExecMaskingPreRA.cpp:71
llvm::MachineFunction
Definition: MachineFunction.h:241
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:967
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:634
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:272
MachineInstrBuilder.h
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:889
llvm::TargetRegisterInfo::getSubReg
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Definition: TargetRegisterInfo.h:1094
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
PPCPredicates.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:274
PPCTargetMachine.h
llvm::createPPCExpandAtomicPseudoPass
FunctionPass * createPPCExpandAtomicPseudoPass()
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
LivePhysRegs.h