LLVM  9.0.0svn
AArch64ExpandPseudoInsts.cpp
Go to the documentation of this file.
1 //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands pseudo instructions into target
10 // instructions to allow proper scheduling and other late optimizations. This
11 // pass should be run after register allocation but before the post-regalloc
12 // scheduling pass.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AArch64ExpandImm.h"
17 #include "AArch64InstrInfo.h"
18 #include "AArch64MachineFunctionInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/Triple.h"
24 #include "llvm/CodeGen/LivePhysRegs.h"
25 #include "llvm/CodeGen/MachineBasicBlock.h"
26 #include "llvm/CodeGen/MachineFunction.h"
27 #include "llvm/CodeGen/MachineFunctionPass.h"
28 #include "llvm/CodeGen/MachineInstr.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineOperand.h"
31 #include "llvm/CodeGen/TargetSubtargetInfo.h"
32 #include "llvm/IR/DebugLoc.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/Pass.h"
35 #include "llvm/Support/CodeGen.h"
36 #include "llvm/Support/MathExtras.h"
37 #include "llvm/Target/TargetMachine.h"
38 #include <cassert>
39 #include <cstdint>
40 #include <iterator>
41 #include <limits>
42 #include <utility>
43 
44 using namespace llvm;
45 
46 #define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
47 
48 namespace {
49 
50 class AArch64ExpandPseudo : public MachineFunctionPass {
51 public:
52  const AArch64InstrInfo *TII;
53 
54  static char ID;
55 
56  AArch64ExpandPseudo() : MachineFunctionPass(ID) {
57  initializeAArch64ExpandPseudoPass(*PassRegistry::getPassRegistry());
58  }
59 
60  bool runOnMachineFunction(MachineFunction &Fn) override;
61 
62  StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
63 
64 private:
65  bool expandMBB(MachineBasicBlock &MBB);
66  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
67  MachineBasicBlock::iterator &NextMBBI);
68  bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
69  unsigned BitSize);
70 
71  bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72  unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73  unsigned ExtendImm, unsigned ZeroReg,
74  MachineBasicBlock::iterator &NextMBBI);
75  bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
76  MachineBasicBlock::iterator MBBI,
77  MachineBasicBlock::iterator &NextMBBI);
78  bool expandSetTagLoop(MachineBasicBlock &MBB,
79  MachineBasicBlock::iterator MBBI,
80  MachineBasicBlock::iterator &NextMBBI);
81 };
82 
83 } // end anonymous namespace
84 
85 char AArch64ExpandPseudo::ID = 0;
86 
87 INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo",
88  AARCH64_EXPAND_PSEUDO_NAME, false, false)
89 
90 /// Transfer implicit operands on the pseudo instruction to the
91 /// instructions created from the expansion.
92 static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
93  MachineInstrBuilder &DefMI) {
94  const MCInstrDesc &Desc = OldMI.getDesc();
95  for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands(); i != e;
96  ++i) {
97  const MachineOperand &MO = OldMI.getOperand(i);
98  assert(MO.isReg() && MO.getReg());
99  if (MO.isUse())
100  UseMI.add(MO);
101  else
102  DefMI.add(MO);
103  }
104 }
105 
106 /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
107 /// real move-immediate instructions to synthesize the immediate.
108 bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB,
109  MachineBasicBlock::iterator MBBI,
110  unsigned BitSize) {
111  MachineInstr &MI = *MBBI;
112  unsigned DstReg = MI.getOperand(0).getReg();
113  uint64_t Imm = MI.getOperand(1).getImm();
114 
115  if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
116  // Useless def, and we don't want to risk creating an invalid ORR (which
117  // would really write to sp).
118  MI.eraseFromParent();
119  return true;
120  }
121 
122  SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
123  AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
124  assert(Insn.size() != 0);
125 
126  SmallVector<MachineInstrBuilder, 4> MIBS;
127  for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
128  bool LastItem = std::next(I) == E;
129  switch (I->Opcode)
130  {
131  default: llvm_unreachable("unhandled!"); break;
132 
133  case AArch64::ORRWri:
134  case AArch64::ORRXri:
135  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
136  .add(MI.getOperand(0))
137  .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
138  .addImm(I->Op2));
139  break;
140  case AArch64::MOVNWi:
141  case AArch64::MOVNXi:
142  case AArch64::MOVZWi:
143  case AArch64::MOVZXi: {
144  bool DstIsDead = MI.getOperand(0).isDead();
145  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
146  .addReg(DstReg, RegState::Define |
147  getDeadRegState(DstIsDead && LastItem))
148  .addImm(I->Op1)
149  .addImm(I->Op2));
150  } break;
151  case AArch64::MOVKWi:
152  case AArch64::MOVKXi: {
153  unsigned DstReg = MI.getOperand(0).getReg();
154  bool DstIsDead = MI.getOperand(0).isDead();
155  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
156  .addReg(DstReg,
157  RegState::Define |
158  getDeadRegState(DstIsDead && LastItem))
159  .addReg(DstReg)
160  .addImm(I->Op1)
161  .addImm(I->Op2));
162  } break;
163  }
164  }
165  transferImpOps(MI, MIBS.front(), MIBS.back());
166  MI.eraseFromParent();
167  return true;
168 }
169 
170 bool AArch64ExpandPseudo::expandCMP_SWAP(
171  MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
172  unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
173  MachineBasicBlock::iterator &NextMBBI) {
174  MachineInstr &MI = *MBBI;
175  DebugLoc DL = MI.getDebugLoc();
176  const MachineOperand &Dest = MI.getOperand(0);
177  unsigned StatusReg = MI.getOperand(1).getReg();
178  bool StatusDead = MI.getOperand(1).isDead();
179  // Duplicating undef operands into 2 instructions does not guarantee the same
180  // value on both; However undef should be replaced by xzr anyway.
181  assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
182  unsigned AddrReg = MI.getOperand(2).getReg();
183  unsigned DesiredReg = MI.getOperand(3).getReg();
184  unsigned NewReg = MI.getOperand(4).getReg();
185 
186  MachineFunction *MF = MBB.getParent();
187  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
188  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
189  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
190 
191  MF->insert(++MBB.getIterator(), LoadCmpBB);
192  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
193  MF->insert(++StoreBB->getIterator(), DoneBB);
194 
195  // .Lloadcmp:
196  // mov wStatus, 0
197  // ldaxr xDest, [xAddr]
198  // cmp xDest, xDesired
199  // b.ne .Ldone
200  if (!StatusDead)
201  BuildMI(LoadCmpBB, DL, TII->get(AArch64::MOVZWi), StatusReg)
202  .addImm(0).addImm(0);
203  BuildMI(LoadCmpBB, DL, TII->get(LdarOp), Dest.getReg())
204  .addReg(AddrReg);
205  BuildMI(LoadCmpBB, DL, TII->get(CmpOp), ZeroReg)
206  .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
207  .addReg(DesiredReg)
208  .addImm(ExtendImm);
209  BuildMI(LoadCmpBB, DL, TII->get(AArch64::Bcc))
210  .addImm(AArch64CC::NE)
211  .addMBB(DoneBB)
212  .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);
213  LoadCmpBB->addSuccessor(DoneBB);
214  LoadCmpBB->addSuccessor(StoreBB);
215 
216  // .Lstore:
217  // stlxr wStatus, xNew, [xAddr]
218  // cbnz wStatus, .Lloadcmp
219  BuildMI(StoreBB, DL, TII->get(StlrOp), StatusReg)
220  .addReg(NewReg)
221  .addReg(AddrReg);
222  BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
223  .addReg(StatusReg, getKillRegState(StatusDead))
224  .addMBB(LoadCmpBB);
225  StoreBB->addSuccessor(LoadCmpBB);
226  StoreBB->addSuccessor(DoneBB);
227 
228  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
229  DoneBB->transferSuccessors(&MBB);
230 
231  MBB.addSuccessor(LoadCmpBB);
232 
233  NextMBBI = MBB.end();
234  MI.eraseFromParent();
235 
236  // Recompute livein lists.
237  LivePhysRegs LiveRegs;
238  computeAndAddLiveIns(LiveRegs, *DoneBB);
239  computeAndAddLiveIns(LiveRegs, *StoreBB);
240  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
241  // Do an extra pass around the loop to get loop carried registers right.
242  StoreBB->clearLiveIns();
243  computeAndAddLiveIns(LiveRegs, *StoreBB);
244  LoadCmpBB->clearLiveIns();
245  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
246 
247  return true;
248 }
249 
250 bool AArch64ExpandPseudo::expandCMP_SWAP_128(
251  MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
252  MachineBasicBlock::iterator &NextMBBI) {
253  MachineInstr &MI = *MBBI;
254  DebugLoc DL = MI.getDebugLoc();
255  MachineOperand &DestLo = MI.getOperand(0);
256  MachineOperand &DestHi = MI.getOperand(1);
257  unsigned StatusReg = MI.getOperand(2).getReg();
258  bool StatusDead = MI.getOperand(2).isDead();
259  // Duplicating undef operands into 2 instructions does not guarantee the same
260  // value on both; However undef should be replaced by xzr anyway.
261  assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
262  unsigned AddrReg = MI.getOperand(3).getReg();
263  unsigned DesiredLoReg = MI.getOperand(4).getReg();
264  unsigned DesiredHiReg = MI.getOperand(5).getReg();
265  unsigned NewLoReg = MI.getOperand(6).getReg();
266  unsigned NewHiReg = MI.getOperand(7).getReg();
267 
268  MachineFunction *MF = MBB.getParent();
269  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
270  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
271  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
272 
273  MF->insert(++MBB.getIterator(), LoadCmpBB);
274  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
275  MF->insert(++StoreBB->getIterator(), DoneBB);
276 
277  // .Lloadcmp:
278  // ldaxp xDestLo, xDestHi, [xAddr]
279  // cmp xDestLo, xDesiredLo
280  // sbcs xDestHi, xDesiredHi
281  // b.ne .Ldone
282  BuildMI(LoadCmpBB, DL, TII->get(AArch64::LDAXPX))
283  .addReg(DestLo.getReg(), RegState::Define)
284  .addReg(DestHi.getReg(), RegState::Define)
285  .addReg(AddrReg);
286  BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
287  .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
288  .addReg(DesiredLoReg)
289  .addImm(0);
290  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
291  .addUse(AArch64::WZR)
292  .addUse(AArch64::WZR)
293  .addImm(AArch64CC::EQ);
294  BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
295  .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
296  .addReg(DesiredHiReg)
297  .addImm(0);
298  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
299  .addUse(StatusReg, RegState::Kill)
300  .addUse(StatusReg, RegState::Kill)
301  .addImm(AArch64CC::EQ);
302  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW))
303  .addUse(StatusReg, getKillRegState(StatusDead))
304  .addMBB(DoneBB);
305  LoadCmpBB->addSuccessor(DoneBB);
306  LoadCmpBB->addSuccessor(StoreBB);
307 
308  // .Lstore:
309  // stlxp wStatus, xNewLo, xNewHi, [xAddr]
310  // cbnz wStatus, .Lloadcmp
311  BuildMI(StoreBB, DL, TII->get(AArch64::STLXPX), StatusReg)
312  .addReg(NewLoReg)
313  .addReg(NewHiReg)
314  .addReg(AddrReg);
315  BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
316  .addReg(StatusReg, getKillRegState(StatusDead))
317  .addMBB(LoadCmpBB);
318  StoreBB->addSuccessor(LoadCmpBB);
319  StoreBB->addSuccessor(DoneBB);
320 
321  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
322  DoneBB->transferSuccessors(&MBB);
323 
324  MBB.addSuccessor(LoadCmpBB);
325 
326  NextMBBI = MBB.end();
327  MI.eraseFromParent();
328 
329  // Recompute liveness bottom up.
330  LivePhysRegs LiveRegs;
331  computeAndAddLiveIns(LiveRegs, *DoneBB);
332  computeAndAddLiveIns(LiveRegs, *StoreBB);
333  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
334  // Do an extra pass in the loop to get the loop carried dependencies right.
335  StoreBB->clearLiveIns();
336  computeAndAddLiveIns(LiveRegs, *StoreBB);
337  LoadCmpBB->clearLiveIns();
338  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
339 
340  return true;
341 }
342 
343 bool AArch64ExpandPseudo::expandSetTagLoop(
344  MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
345  MachineBasicBlock::iterator &NextMBBI) {
346  MachineInstr &MI = *MBBI;
347  DebugLoc DL = MI.getDebugLoc();
348  Register SizeReg = MI.getOperand(2).getReg();
349  Register AddressReg = MI.getOperand(3).getReg();
350 
351  MachineFunction *MF = MBB.getParent();
352 
353  bool ZeroData = MI.getOpcode() == AArch64::STZGloop;
354  const unsigned OpCode =
355  ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
356 
357  auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
358  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
359 
360  MF->insert(++MBB.getIterator(), LoopBB);
361  MF->insert(++LoopBB->getIterator(), DoneBB);
362 
363  BuildMI(LoopBB, DL, TII->get(OpCode))
364  .addDef(AddressReg)
365  .addReg(AddressReg)
366  .addReg(AddressReg)
367  .addImm(2)
368  .cloneMemRefs(MI)
369  .setMIFlags(MI.getFlags());
370  BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri))
371  .addDef(SizeReg)
372  .addReg(SizeReg)
373  .addImm(16 * 2)
374  .addImm(0);
375  BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB);
376 
377  LoopBB->addSuccessor(LoopBB);
378  LoopBB->addSuccessor(DoneBB);
379 
380  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
381  DoneBB->transferSuccessors(&MBB);
382 
383  MBB.addSuccessor(LoopBB);
384 
385  NextMBBI = MBB.end();
386  MI.eraseFromParent();
387  // Recompute liveness bottom up.
388  LivePhysRegs LiveRegs;
389  computeAndAddLiveIns(LiveRegs, *DoneBB);
390  computeAndAddLiveIns(LiveRegs, *LoopBB);
391  // Do an extra pass in the loop to get the loop carried dependencies right.
392  // FIXME: is this necessary?
393  LoopBB->clearLiveIns();
394  computeAndAddLiveIns(LiveRegs, *LoopBB);
395  DoneBB->clearLiveIns();
396  computeAndAddLiveIns(LiveRegs, *DoneBB);
397 
398  return true;
399 }
400 
401 /// If MBBI references a pseudo instruction that should be expanded here,
402 /// do the expansion and return true. Otherwise return false.
403 bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
404  MachineBasicBlock::iterator MBBI,
405  MachineBasicBlock::iterator &NextMBBI) {
406  MachineInstr &MI = *MBBI;
407  unsigned Opcode = MI.getOpcode();
408  switch (Opcode) {
409  default:
410  break;
411 
412  case AArch64::ADDWrr:
413  case AArch64::SUBWrr:
414  case AArch64::ADDXrr:
415  case AArch64::SUBXrr:
416  case AArch64::ADDSWrr:
417  case AArch64::SUBSWrr:
418  case AArch64::ADDSXrr:
419  case AArch64::SUBSXrr:
420  case AArch64::ANDWrr:
421  case AArch64::ANDXrr:
422  case AArch64::BICWrr:
423  case AArch64::BICXrr:
424  case AArch64::ANDSWrr:
425  case AArch64::ANDSXrr:
426  case AArch64::BICSWrr:
427  case AArch64::BICSXrr:
428  case AArch64::EONWrr:
429  case AArch64::EONXrr:
430  case AArch64::EORWrr:
431  case AArch64::EORXrr:
432  case AArch64::ORNWrr:
433  case AArch64::ORNXrr:
434  case AArch64::ORRWrr:
435  case AArch64::ORRXrr: {
436  unsigned Opcode;
437  switch (MI.getOpcode()) {
438  default:
439  return false;
440  case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
441  case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
442  case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
443  case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
444  case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
445  case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
446  case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
447  case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
448  case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
449  case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
450  case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
451  case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
452  case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
453  case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
454  case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
455  case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
456  case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
457  case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
458  case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
459  case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
460  case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
461  case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
462  case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
463  case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
464  }
465  MachineInstrBuilder MIB1 =
466  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode),
467  MI.getOperand(0).getReg())
468  .add(MI.getOperand(1))
469  .add(MI.getOperand(2))
470  .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
471  transferImpOps(MI, MIB1, MIB1);
472  MI.eraseFromParent();
473  return true;
474  }
475 
476  case AArch64::LOADgot: {
477  MachineFunction *MF = MBB.getParent();
478  unsigned DstReg = MI.getOperand(0).getReg();
479  const MachineOperand &MO1 = MI.getOperand(1);
480  unsigned Flags = MO1.getTargetFlags();
481 
482  if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
483  // Tiny codemodel expand to LDR
484  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
485  TII->get(AArch64::LDRXl), DstReg);
486 
487  if (MO1.isGlobal()) {
488  MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags);
489  } else if (MO1.isSymbol()) {
490  MIB.addExternalSymbol(MO1.getSymbolName(), Flags);
491  } else {
492  assert(MO1.isCPI() &&
493  "Only expect globals, externalsymbols, or constant pools");
494  MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags);
495  }
496  } else {
497  // Small codemodel expand into ADRP + LDR.
498  MachineInstrBuilder MIB1 =
499  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg);
500  MachineInstrBuilder MIB2 =
501  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui))
502  .add(MI.getOperand(0))
503  .addReg(DstReg);
504 
505  if (MO1.isGlobal()) {
506  MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE);
507  MIB2.addGlobalAddress(MO1.getGlobal(), 0,
508  Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
509  } else if (MO1.isSymbol()) {
510  MIB1.addExternalSymbol(MO1.getSymbolName(), Flags | AArch64II::MO_PAGE);
511  MIB2.addExternalSymbol(MO1.getSymbolName(), Flags |
512  AArch64II::MO_PAGEOFF |
513  AArch64II::MO_NC);
514  } else {
515  assert(MO1.isCPI() &&
516  "Only expect globals, externalsymbols, or constant pools");
517  MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
518  Flags | AArch64II::MO_PAGE);
519  MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
520  Flags | AArch64II::MO_PAGEOFF |
521  AArch64II::MO_NC);
522  }
523 
524  transferImpOps(MI, MIB1, MIB2);
525  }
526  MI.eraseFromParent();
527  return true;
528  }
529 
530  case AArch64::MOVaddr:
531  case AArch64::MOVaddrJT:
532  case AArch64::MOVaddrCP:
533  case AArch64::MOVaddrBA:
534  case AArch64::MOVaddrTLS:
535  case AArch64::MOVaddrEXT: {
536  // Expand into ADRP + ADD.
537  unsigned DstReg = MI.getOperand(0).getReg();
538  MachineInstrBuilder MIB1 =
539  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)
540  .add(MI.getOperand(1));
541 
542  MachineInstrBuilder MIB2 =
543  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
544  .add(MI.getOperand(0))
545  .addReg(DstReg)
546  .add(MI.getOperand(2))
547  .addImm(0);
548 
549  transferImpOps(MI, MIB1, MIB2);
550  MI.eraseFromParent();
551  return true;
552  }
553  case AArch64::ADDlowTLS:
554  // Produce a plain ADD
555  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
556  .add(MI.getOperand(0))
557  .add(MI.getOperand(1))
558  .add(MI.getOperand(2))
559  .addImm(0);
560  MI.eraseFromParent();
561  return true;
562 
563  case AArch64::MOVbaseTLS: {
564  unsigned DstReg = MI.getOperand(0).getReg();
565  auto SysReg = AArch64SysReg::TPIDR_EL0;
566  MachineFunction *MF = MBB.getParent();
567  if (MF->getTarget().getTargetTriple().isOSFuchsia() &&
568  MF->getTarget().getCodeModel() == CodeModel::Kernel)
569  SysReg = AArch64SysReg::TPIDR_EL1;
570  else if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
571  SysReg = AArch64SysReg::TPIDR_EL3;
572  else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
573  SysReg = AArch64SysReg::TPIDR_EL2;
574  else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
575  SysReg = AArch64SysReg::TPIDR_EL1;
576  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg)
577  .addImm(SysReg);
578  MI.eraseFromParent();
579  return true;
580  }
581 
582  case AArch64::MOVi32imm:
583  return expandMOVImm(MBB, MBBI, 32);
584  case AArch64::MOVi64imm:
585  return expandMOVImm(MBB, MBBI, 64);
586  case AArch64::RET_ReallyLR: {
587  // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
588  // function and missing live-ins. We are fine in practice because callee
589  // saved register handling ensures the register value is restored before
590  // RET, but we need the undef flag here to appease the MachineVerifier
591  // liveness checks.
592  MachineInstrBuilder MIB =
593  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET))
594  .addReg(AArch64::LR, RegState::Undef);
595  transferImpOps(MI, MIB, MIB);
596  MI.eraseFromParent();
597  return true;
598  }
599  case AArch64::CMP_SWAP_8:
600  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,
601  AArch64::SUBSWrx,
602  AArch64_AM::getArithExtendImm(AArch64_AM::UXTB, 0),
603  AArch64::WZR, NextMBBI);
604  case AArch64::CMP_SWAP_16:
605  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,
606  AArch64::SUBSWrx,
607  AArch64_AM::getArithExtendImm(AArch64_AM::UXTH, 0),
608  AArch64::WZR, NextMBBI);
609  case AArch64::CMP_SWAP_32:
610  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
611  AArch64::SUBSWrs,
612  AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
613  AArch64::WZR, NextMBBI);
614  case AArch64::CMP_SWAP_64:
615  return expandCMP_SWAP(MBB, MBBI,
616  AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
617  AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
618  AArch64::XZR, NextMBBI);
619  case AArch64::CMP_SWAP_128:
620  return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
621 
622  case AArch64::AESMCrrTied:
623  case AArch64::AESIMCrrTied: {
624  MachineInstrBuilder MIB =
625  BuildMI(MBB, MBBI, MI.getDebugLoc(),
626  TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
627  AArch64::AESIMCrr))
628  .add(MI.getOperand(0))
629  .add(MI.getOperand(1));
630  transferImpOps(MI, MIB, MIB);
631  MI.eraseFromParent();
632  return true;
633  }
634  case AArch64::IRGstack: {
635  MachineFunction &MF = *MBB.getParent();
636  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
637  const AArch64FrameLowering *TFI =
638  MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
639 
640  // IRG does not allow immediate offset. getTaggedBasePointerOffset should
641  // almost always point to SP-after-prologue; if not, emit a longer
642  // instruction sequence.
643  int BaseOffset = -AFI->getTaggedBasePointerOffset();
644  unsigned FrameReg;
645  int FrameRegOffset = TFI->resolveFrameOffsetReference(
646  MF, BaseOffset, false /*isFixed*/, FrameReg, /*PreferFP=*/false,
647  /*ForSimm=*/true);
648  Register SrcReg = FrameReg;
649  if (FrameRegOffset != 0) {
650  // Use output register as temporary.
651  SrcReg = MI.getOperand(0).getReg();
652  emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg,
653  FrameRegOffset, TII);
654  }
655  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG))
656  .add(MI.getOperand(0))
657  .addUse(SrcReg)
658  .add(MI.getOperand(2));
659  MI.eraseFromParent();
660  return true;
661  }
662  case AArch64::TAGPstack: {
663  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDG))
664  .add(MI.getOperand(0))
665  .add(MI.getOperand(1))
666  .add(MI.getOperand(2))
667  .add(MI.getOperand(4));
668  MI.eraseFromParent();
669  return true;
670  }
671  case AArch64::STGloop:
672  case AArch64::STZGloop:
673  return expandSetTagLoop(MBB, MBBI, NextMBBI);
674  }
675  return false;
676 }
677 
678 /// Iterate over the instructions in basic block MBB and expand any
679 /// pseudo instructions. Return true if anything was modified.
680 bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
681  bool Modified = false;
682 
683  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
684  while (MBBI != E) {
685  MachineBasicBlock::iterator NMBBI = std::next(MBBI);
686  Modified |= expandMI(MBB, MBBI, NMBBI);
687  MBBI = NMBBI;
688  }
689 
690  return Modified;
691 }
692 
693 bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
694  TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
695 
696  bool Modified = false;
697  for (auto &MBB : MF)
698  Modified |= expandMBB(MBB);
699  return Modified;
700 }
701 
702 /// Returns an instance of the pseudo instruction expansion pass.
703 FunctionPass *llvm::createAArch64ExpandPseudoPass() {
704  return new AArch64ExpandPseudo();
705 }
llvm::MachineOperand::getTargetFlags
unsigned getTargetFlags() const
Definition: MachineOperand.h:215
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:215
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::AArch64FunctionInfo
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
Definition: AArch64MachineFunctionInfo.h:31
llvm
This class represents lattice values for constants.
Definition: AllocatorList.h:23
llvm::AArch64II::MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
Definition: AArch64BaseInfo.h:573
llvm::emitFrameOffset
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
Definition: AArch64InstrInfo.cpp:2975
llvm::MachineInstr::getDebugLoc
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:164
llvm::AArch64_AM::UXTB
Definition: AArch64AddressingModes.h:40
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::MachineOperand::isCPI
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
Definition: MachineOperand.h:323
llvm::Triple::isOSFuchsia
bool isOSFuchsia() const
Definition: Triple.h:505
llvm::MachineInstrBuilder::addGlobalAddress
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:168
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MCInstrDesc::getNumOperands
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:211
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:127
INITIALIZE_PASS
INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo", AARCH64_EXPAND_PSEUDO_NAME, false, false) static void transferImpOps(MachineInstr &OldMI
Transfer implicit operands on the pseudo instruction to the instructions created from the expansion...
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:114
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
Definition: MachineInstr.cpp:678
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
llvm::MachineOperand::getSymbolName
const char * getSymbolName() const
Definition: MachineOperand.h:592
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:95
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:386
llvm::getKillRegState
unsigned getKillRegState(bool B)
Definition: MachineInstrBuilder.h:471
llvm::AArch64_AM::getShifterImm
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Definition: AArch64AddressingModes.h:98
llvm::getDeadRegState
unsigned getDeadRegState(bool B)
Definition: MachineInstrBuilder.h:474
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:311
llvm::initializeAArch64ExpandPseudoPass
void initializeAArch64ExpandPseudoPass(PassRegistry &)
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:464
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:92
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineOperand::getGlobal
const GlobalValue * getGlobal() const
Definition: MachineOperand.h:552
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:558
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
llvm::TargetMachine::getTargetTriple
const Triple & getTargetTriple() const
Definition: TargetMachine.h:111
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
LivePhysRegs.h
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
llvm::SmallVectorBase::size
size_t size() const
Definition: SmallVector.h:52
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:135
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:649
llvm::MachineOperand::isGlobal
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
Definition: MachineOperand.h:329
add
static uint64_t add(uint64_t LeftOp, uint64_t RightOp)
Definition: FileCheck.cpp:218
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:49
llvm::SmallVector
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
DefMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Definition: AArch64ExpandPseudoInsts.cpp:93
llvm::createAArch64ExpandPseudoPass
FunctionPass * createAArch64ExpandPseudoPass()
Returns an instance of the pseudo instruction expansion pass.
Definition: AArch64ExpandPseudoInsts.cpp:703
llvm::MachineInstrBuilder::addConstantPoolIndex
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:148
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:526
llvm::TargetMachine::getCodeModel
CodeModel::Model getCodeModel() const
Returns the code model.
Definition: TargetMachine.cpp:86
llvm::AArch64_AM::LSL
Definition: AArch64AddressingModes.h:34
llvm::AArch64_AM::getArithExtendImm
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
Definition: AArch64AddressingModes.h:170
llvm::computeAndAddLiveIns
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
Definition: LivePhysRegs.cpp:326
llvm::AArch64_IMM::expandMOVImm
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
Definition: AArch64ExpandImm.cpp:305
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:173
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:122
llvm::AArch64_AM::UXTH
Definition: AArch64AddressingModes.h:41
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition: MachineInstrBuilder.h:259
llvm::MachineOperand::getOffset
int64_t getOffset() const
Return the offset from the symbol in this operand.
Definition: MachineOperand.h:584
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:175
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::MachineInstrBuilder::cloneMemRefs
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Definition: MachineInstrBuilder.h:204
llvm::AArch64II::MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page...
Definition: AArch64BaseInfo.h:578
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:150
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:88
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:460
llvm::MachineOperand::isSymbol
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
Definition: MachineOperand.h:331
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:311
llvm::MachineInstr::getFlags
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:292
AARCH64_EXPAND_PSEUDO_NAME
#define AARCH64_EXPAND_PSEUDO_NAME
Definition: AArch64ExpandPseudoInsts.cpp:46
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:657
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:91
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
llvm::AArch64II::MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow...
Definition: AArch64BaseInfo.h:614
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:349
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
Definition: MachineInstrBuilder.h:137
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:18
Generated on Thu Jul 18 2019 00:37:31 for LLVM by   doxygen 1.8.13