LLVM  10.0.0svn
AArch64ExpandPseudoInsts.cpp
Go to the documentation of this file.
1 //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands pseudo instructions into target
10 // instructions to allow proper scheduling and other late optimizations. This
11 // pass should be run after register allocation but before the post-regalloc
12 // scheduling pass.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AArch64ExpandImm.h"
17 #include "AArch64InstrInfo.h"
19 #include "AArch64Subtarget.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/Triple.h"
32 #include "llvm/IR/DebugLoc.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/Pass.h"
35 #include "llvm/Support/CodeGen.h"
38 #include <cassert>
39 #include <cstdint>
40 #include <iterator>
41 #include <limits>
42 #include <utility>
43 
44 using namespace llvm;
45 
46 #define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
47 
48 namespace {
49 
50 class AArch64ExpandPseudo : public MachineFunctionPass {
51 public:
52  const AArch64InstrInfo *TII;
53 
54  static char ID;
55 
56  AArch64ExpandPseudo() : MachineFunctionPass(ID) {
58  }
59 
60  bool runOnMachineFunction(MachineFunction &Fn) override;
61 
62  StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
63 
64 private:
65  bool expandMBB(MachineBasicBlock &MBB);
66  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
67  MachineBasicBlock::iterator &NextMBBI);
69  unsigned BitSize);
70 
71  bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72  unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73  unsigned ExtendImm, unsigned ZeroReg,
74  MachineBasicBlock::iterator &NextMBBI);
75  bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
77  MachineBasicBlock::iterator &NextMBBI);
78  bool expandSetTagLoop(MachineBasicBlock &MBB,
80  MachineBasicBlock::iterator &NextMBBI);
81 };
82 
83 } // end anonymous namespace
84 
86 
87 INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo",
88  AARCH64_EXPAND_PSEUDO_NAME, false, false)
89 
90 /// Transfer implicit operands on the pseudo instruction to the
91 /// instructions created from the expansion.
92 static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
94  const MCInstrDesc &Desc = OldMI.getDesc();
95  for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands(); i != e;
96  ++i) {
97  const MachineOperand &MO = OldMI.getOperand(i);
98  assert(MO.isReg() && MO.getReg());
99  if (MO.isUse())
100  UseMI.add(MO);
101  else
102  DefMI.add(MO);
103  }
104 }
105 
106 /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
107 /// real move-immediate instructions to synthesize the immediate.
110  unsigned BitSize) {
111  MachineInstr &MI = *MBBI;
112  Register DstReg = MI.getOperand(0).getReg();
113  uint64_t Imm = MI.getOperand(1).getImm();
114 
115  if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
116  // Useless def, and we don't want to risk creating an invalid ORR (which
117  // would really write to sp).
118  MI.eraseFromParent();
119  return true;
120  }
121 
123  AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
124  assert(Insn.size() != 0);
125 
127  for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
128  bool LastItem = std::next(I) == E;
129  switch (I->Opcode)
130  {
131  default: llvm_unreachable("unhandled!"); break;
132 
133  case AArch64::ORRWri:
134  case AArch64::ORRXri:
135  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
136  .add(MI.getOperand(0))
137  .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
138  .addImm(I->Op2));
139  break;
140  case AArch64::MOVNWi:
141  case AArch64::MOVNXi:
142  case AArch64::MOVZWi:
143  case AArch64::MOVZXi: {
144  bool DstIsDead = MI.getOperand(0).isDead();
145  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
146  .addReg(DstReg, RegState::Define |
147  getDeadRegState(DstIsDead && LastItem))
148  .addImm(I->Op1)
149  .addImm(I->Op2));
150  } break;
151  case AArch64::MOVKWi:
152  case AArch64::MOVKXi: {
153  Register DstReg = MI.getOperand(0).getReg();
154  bool DstIsDead = MI.getOperand(0).isDead();
155  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
156  .addReg(DstReg,
158  getDeadRegState(DstIsDead && LastItem))
159  .addReg(DstReg)
160  .addImm(I->Op1)
161  .addImm(I->Op2));
162  } break;
163  }
164  }
165  transferImpOps(MI, MIBS.front(), MIBS.back());
166  MI.eraseFromParent();
167  return true;
168 }
169 
170 bool AArch64ExpandPseudo::expandCMP_SWAP(
171  MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
172  unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
173  MachineBasicBlock::iterator &NextMBBI) {
174  MachineInstr &MI = *MBBI;
175  DebugLoc DL = MI.getDebugLoc();
176  const MachineOperand &Dest = MI.getOperand(0);
177  Register StatusReg = MI.getOperand(1).getReg();
178  bool StatusDead = MI.getOperand(1).isDead();
179  // Duplicating undef operands into 2 instructions does not guarantee the same
180  // value on both; However undef should be replaced by xzr anyway.
181  assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
182  Register AddrReg = MI.getOperand(2).getReg();
183  Register DesiredReg = MI.getOperand(3).getReg();
184  Register NewReg = MI.getOperand(4).getReg();
185 
186  MachineFunction *MF = MBB.getParent();
187  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
188  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
189  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
190 
191  MF->insert(++MBB.getIterator(), LoadCmpBB);
192  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
193  MF->insert(++StoreBB->getIterator(), DoneBB);
194 
195  // .Lloadcmp:
196  // mov wStatus, 0
197  // ldaxr xDest, [xAddr]
198  // cmp xDest, xDesired
199  // b.ne .Ldone
200  if (!StatusDead)
201  BuildMI(LoadCmpBB, DL, TII->get(AArch64::MOVZWi), StatusReg)
202  .addImm(0).addImm(0);
203  BuildMI(LoadCmpBB, DL, TII->get(LdarOp), Dest.getReg())
204  .addReg(AddrReg);
205  BuildMI(LoadCmpBB, DL, TII->get(CmpOp), ZeroReg)
206  .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
207  .addReg(DesiredReg)
208  .addImm(ExtendImm);
209  BuildMI(LoadCmpBB, DL, TII->get(AArch64::Bcc))
210  .addImm(AArch64CC::NE)
211  .addMBB(DoneBB)
212  .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);
213  LoadCmpBB->addSuccessor(DoneBB);
214  LoadCmpBB->addSuccessor(StoreBB);
215 
216  // .Lstore:
217  // stlxr wStatus, xNew, [xAddr]
218  // cbnz wStatus, .Lloadcmp
219  BuildMI(StoreBB, DL, TII->get(StlrOp), StatusReg)
220  .addReg(NewReg)
221  .addReg(AddrReg);
222  BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
223  .addReg(StatusReg, getKillRegState(StatusDead))
224  .addMBB(LoadCmpBB);
225  StoreBB->addSuccessor(LoadCmpBB);
226  StoreBB->addSuccessor(DoneBB);
227 
228  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
229  DoneBB->transferSuccessors(&MBB);
230 
231  MBB.addSuccessor(LoadCmpBB);
232 
233  NextMBBI = MBB.end();
234  MI.eraseFromParent();
235 
236  // Recompute livein lists.
237  LivePhysRegs LiveRegs;
238  computeAndAddLiveIns(LiveRegs, *DoneBB);
239  computeAndAddLiveIns(LiveRegs, *StoreBB);
240  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
241  // Do an extra pass around the loop to get loop carried registers right.
242  StoreBB->clearLiveIns();
243  computeAndAddLiveIns(LiveRegs, *StoreBB);
244  LoadCmpBB->clearLiveIns();
245  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
246 
247  return true;
248 }
249 
250 bool AArch64ExpandPseudo::expandCMP_SWAP_128(
252  MachineBasicBlock::iterator &NextMBBI) {
253  MachineInstr &MI = *MBBI;
254  DebugLoc DL = MI.getDebugLoc();
255  MachineOperand &DestLo = MI.getOperand(0);
256  MachineOperand &DestHi = MI.getOperand(1);
257  Register StatusReg = MI.getOperand(2).getReg();
258  bool StatusDead = MI.getOperand(2).isDead();
259  // Duplicating undef operands into 2 instructions does not guarantee the same
260  // value on both; However undef should be replaced by xzr anyway.
261  assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
262  Register AddrReg = MI.getOperand(3).getReg();
263  Register DesiredLoReg = MI.getOperand(4).getReg();
264  Register DesiredHiReg = MI.getOperand(5).getReg();
265  Register NewLoReg = MI.getOperand(6).getReg();
266  Register NewHiReg = MI.getOperand(7).getReg();
267 
268  MachineFunction *MF = MBB.getParent();
269  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
270  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
271  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
272 
273  MF->insert(++MBB.getIterator(), LoadCmpBB);
274  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
275  MF->insert(++StoreBB->getIterator(), DoneBB);
276 
277  // .Lloadcmp:
278  // ldaxp xDestLo, xDestHi, [xAddr]
279  // cmp xDestLo, xDesiredLo
280  // sbcs xDestHi, xDesiredHi
281  // b.ne .Ldone
282  BuildMI(LoadCmpBB, DL, TII->get(AArch64::LDAXPX))
283  .addReg(DestLo.getReg(), RegState::Define)
284  .addReg(DestHi.getReg(), RegState::Define)
285  .addReg(AddrReg);
286  BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
287  .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
288  .addReg(DesiredLoReg)
289  .addImm(0);
290  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
291  .addUse(AArch64::WZR)
292  .addUse(AArch64::WZR)
294  BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
295  .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
296  .addReg(DesiredHiReg)
297  .addImm(0);
298  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
299  .addUse(StatusReg, RegState::Kill)
300  .addUse(StatusReg, RegState::Kill)
302  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW))
303  .addUse(StatusReg, getKillRegState(StatusDead))
304  .addMBB(DoneBB);
305  LoadCmpBB->addSuccessor(DoneBB);
306  LoadCmpBB->addSuccessor(StoreBB);
307 
308  // .Lstore:
309  // stlxp wStatus, xNewLo, xNewHi, [xAddr]
310  // cbnz wStatus, .Lloadcmp
311  BuildMI(StoreBB, DL, TII->get(AArch64::STLXPX), StatusReg)
312  .addReg(NewLoReg)
313  .addReg(NewHiReg)
314  .addReg(AddrReg);
315  BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
316  .addReg(StatusReg, getKillRegState(StatusDead))
317  .addMBB(LoadCmpBB);
318  StoreBB->addSuccessor(LoadCmpBB);
319  StoreBB->addSuccessor(DoneBB);
320 
321  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
322  DoneBB->transferSuccessors(&MBB);
323 
324  MBB.addSuccessor(LoadCmpBB);
325 
326  NextMBBI = MBB.end();
327  MI.eraseFromParent();
328 
329  // Recompute liveness bottom up.
330  LivePhysRegs LiveRegs;
331  computeAndAddLiveIns(LiveRegs, *DoneBB);
332  computeAndAddLiveIns(LiveRegs, *StoreBB);
333  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
334  // Do an extra pass in the loop to get the loop carried dependencies right.
335  StoreBB->clearLiveIns();
336  computeAndAddLiveIns(LiveRegs, *StoreBB);
337  LoadCmpBB->clearLiveIns();
338  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
339 
340  return true;
341 }
342 
343 bool AArch64ExpandPseudo::expandSetTagLoop(
345  MachineBasicBlock::iterator &NextMBBI) {
346  MachineInstr &MI = *MBBI;
347  DebugLoc DL = MI.getDebugLoc();
348  Register SizeReg = MI.getOperand(2).getReg();
349  Register AddressReg = MI.getOperand(3).getReg();
350 
351  MachineFunction *MF = MBB.getParent();
352 
353  bool ZeroData = MI.getOpcode() == AArch64::STZGloop;
354  const unsigned OpCode =
355  ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
356 
357  auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
358  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
359 
360  MF->insert(++MBB.getIterator(), LoopBB);
361  MF->insert(++LoopBB->getIterator(), DoneBB);
362 
363  BuildMI(LoopBB, DL, TII->get(OpCode))
364  .addDef(AddressReg)
365  .addReg(AddressReg)
366  .addReg(AddressReg)
367  .addImm(2)
368  .cloneMemRefs(MI)
369  .setMIFlags(MI.getFlags());
370  BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri))
371  .addDef(SizeReg)
372  .addReg(SizeReg)
373  .addImm(16 * 2)
374  .addImm(0);
375  BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB);
376 
377  LoopBB->addSuccessor(LoopBB);
378  LoopBB->addSuccessor(DoneBB);
379 
380  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
381  DoneBB->transferSuccessors(&MBB);
382 
383  MBB.addSuccessor(LoopBB);
384 
385  NextMBBI = MBB.end();
386  MI.eraseFromParent();
387  // Recompute liveness bottom up.
388  LivePhysRegs LiveRegs;
389  computeAndAddLiveIns(LiveRegs, *DoneBB);
390  computeAndAddLiveIns(LiveRegs, *LoopBB);
391  // Do an extra pass in the loop to get the loop carried dependencies right.
392  // FIXME: is this necessary?
393  LoopBB->clearLiveIns();
394  computeAndAddLiveIns(LiveRegs, *LoopBB);
395  DoneBB->clearLiveIns();
396  computeAndAddLiveIns(LiveRegs, *DoneBB);
397 
398  return true;
399 }
400 
401 /// If MBBI references a pseudo instruction that should be expanded here,
402 /// do the expansion and return true. Otherwise return false.
403 bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
405  MachineBasicBlock::iterator &NextMBBI) {
406  MachineInstr &MI = *MBBI;
407  unsigned Opcode = MI.getOpcode();
408  switch (Opcode) {
409  default:
410  break;
411 
412  case AArch64::ADDWrr:
413  case AArch64::SUBWrr:
414  case AArch64::ADDXrr:
415  case AArch64::SUBXrr:
416  case AArch64::ADDSWrr:
417  case AArch64::SUBSWrr:
418  case AArch64::ADDSXrr:
419  case AArch64::SUBSXrr:
420  case AArch64::ANDWrr:
421  case AArch64::ANDXrr:
422  case AArch64::BICWrr:
423  case AArch64::BICXrr:
424  case AArch64::ANDSWrr:
425  case AArch64::ANDSXrr:
426  case AArch64::BICSWrr:
427  case AArch64::BICSXrr:
428  case AArch64::EONWrr:
429  case AArch64::EONXrr:
430  case AArch64::EORWrr:
431  case AArch64::EORXrr:
432  case AArch64::ORNWrr:
433  case AArch64::ORNXrr:
434  case AArch64::ORRWrr:
435  case AArch64::ORRXrr: {
436  unsigned Opcode;
437  switch (MI.getOpcode()) {
438  default:
439  return false;
440  case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
441  case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
442  case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
443  case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
444  case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
445  case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
446  case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
447  case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
448  case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
449  case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
450  case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
451  case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
452  case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
453  case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
454  case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
455  case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
456  case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
457  case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
458  case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
459  case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
460  case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
461  case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
462  case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
463  case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
464  }
465  MachineInstrBuilder MIB1 =
466  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode),
467  MI.getOperand(0).getReg())
468  .add(MI.getOperand(1))
469  .add(MI.getOperand(2))
471  transferImpOps(MI, MIB1, MIB1);
472  MI.eraseFromParent();
473  return true;
474  }
475 
476  case AArch64::LOADgot: {
477  MachineFunction *MF = MBB.getParent();
478  Register DstReg = MI.getOperand(0).getReg();
479  const MachineOperand &MO1 = MI.getOperand(1);
480  unsigned Flags = MO1.getTargetFlags();
481 
482  if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
483  // Tiny codemodel expand to LDR
484  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
485  TII->get(AArch64::LDRXl), DstReg);
486 
487  if (MO1.isGlobal()) {
488  MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags);
489  } else if (MO1.isSymbol()) {
490  MIB.addExternalSymbol(MO1.getSymbolName(), Flags);
491  } else {
492  assert(MO1.isCPI() &&
493  "Only expect globals, externalsymbols, or constant pools");
494  MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags);
495  }
496  } else {
497  // Small codemodel expand into ADRP + LDR.
498  MachineInstrBuilder MIB1 =
499  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg);
500  MachineInstrBuilder MIB2 =
501  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui))
502  .add(MI.getOperand(0))
503  .addReg(DstReg);
504 
505  if (MO1.isGlobal()) {
506  MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE);
507  MIB2.addGlobalAddress(MO1.getGlobal(), 0,
509  } else if (MO1.isSymbol()) {
511  MIB2.addExternalSymbol(MO1.getSymbolName(), Flags |
514  } else {
515  assert(MO1.isCPI() &&
516  "Only expect globals, externalsymbols, or constant pools");
517  MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
518  Flags | AArch64II::MO_PAGE);
519  MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
520  Flags | AArch64II::MO_PAGEOFF |
522  }
523 
524  transferImpOps(MI, MIB1, MIB2);
525  }
526  MI.eraseFromParent();
527  return true;
528  }
529 
530  case AArch64::MOVaddr:
531  case AArch64::MOVaddrJT:
532  case AArch64::MOVaddrCP:
533  case AArch64::MOVaddrBA:
534  case AArch64::MOVaddrTLS:
535  case AArch64::MOVaddrEXT: {
536  // Expand into ADRP + ADD.
537  Register DstReg = MI.getOperand(0).getReg();
538  MachineInstrBuilder MIB1 =
539  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)
540  .add(MI.getOperand(1));
541 
543  // MO_TAGGED on the page indicates a tagged address. Set the tag now.
544  // We do so by creating a MOVK that sets bits 48-63 of the register to
545  // (global address + 0x100000000 - PC) >> 48. This assumes that we're in
546  // the small code model so we can assume a binary size of <= 4GB, which
547  // makes the untagged PC relative offset positive. The binary must also be
548  // loaded into address range [0, 2^48). Both of these properties need to
549  // be ensured at runtime when using tagged addresses.
550  auto Tag = MI.getOperand(1);
551  Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3);
552  Tag.setOffset(0x100000000);
553  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi), DstReg)
554  .addReg(DstReg)
555  .add(Tag)
556  .addImm(48);
557  }
558 
559  MachineInstrBuilder MIB2 =
560  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
561  .add(MI.getOperand(0))
562  .addReg(DstReg)
563  .add(MI.getOperand(2))
564  .addImm(0);
565 
566  transferImpOps(MI, MIB1, MIB2);
567  MI.eraseFromParent();
568  return true;
569  }
570  case AArch64::ADDlowTLS:
571  // Produce a plain ADD
572  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
573  .add(MI.getOperand(0))
574  .add(MI.getOperand(1))
575  .add(MI.getOperand(2))
576  .addImm(0);
577  MI.eraseFromParent();
578  return true;
579 
580  case AArch64::MOVbaseTLS: {
581  Register DstReg = MI.getOperand(0).getReg();
582  auto SysReg = AArch64SysReg::TPIDR_EL0;
583  MachineFunction *MF = MBB.getParent();
584  if (MF->getTarget().getTargetTriple().isOSFuchsia() &&
586  SysReg = AArch64SysReg::TPIDR_EL1;
587  else if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
588  SysReg = AArch64SysReg::TPIDR_EL3;
589  else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
590  SysReg = AArch64SysReg::TPIDR_EL2;
591  else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
592  SysReg = AArch64SysReg::TPIDR_EL1;
593  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg)
594  .addImm(SysReg);
595  MI.eraseFromParent();
596  return true;
597  }
598 
599  case AArch64::MOVi32imm:
600  return expandMOVImm(MBB, MBBI, 32);
601  case AArch64::MOVi64imm:
602  return expandMOVImm(MBB, MBBI, 64);
603  case AArch64::RET_ReallyLR: {
604  // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
605  // function and missing live-ins. We are fine in practice because callee
606  // saved register handling ensures the register value is restored before
607  // RET, but we need the undef flag here to appease the MachineVerifier
608  // liveness checks.
609  MachineInstrBuilder MIB =
610  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET))
611  .addReg(AArch64::LR, RegState::Undef);
612  transferImpOps(MI, MIB, MIB);
613  MI.eraseFromParent();
614  return true;
615  }
616  case AArch64::CMP_SWAP_8:
617  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,
618  AArch64::SUBSWrx,
620  AArch64::WZR, NextMBBI);
621  case AArch64::CMP_SWAP_16:
622  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,
623  AArch64::SUBSWrx,
625  AArch64::WZR, NextMBBI);
626  case AArch64::CMP_SWAP_32:
627  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
628  AArch64::SUBSWrs,
630  AArch64::WZR, NextMBBI);
631  case AArch64::CMP_SWAP_64:
632  return expandCMP_SWAP(MBB, MBBI,
633  AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
635  AArch64::XZR, NextMBBI);
636  case AArch64::CMP_SWAP_128:
637  return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
638 
639  case AArch64::AESMCrrTied:
640  case AArch64::AESIMCrrTied: {
641  MachineInstrBuilder MIB =
642  BuildMI(MBB, MBBI, MI.getDebugLoc(),
643  TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
644  AArch64::AESIMCrr))
645  .add(MI.getOperand(0))
646  .add(MI.getOperand(1));
647  transferImpOps(MI, MIB, MIB);
648  MI.eraseFromParent();
649  return true;
650  }
651  case AArch64::IRGstack: {
652  MachineFunction &MF = *MBB.getParent();
654  const AArch64FrameLowering *TFI =
655  MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
656 
657  // IRG does not allow immediate offset. getTaggedBasePointerOffset should
658  // almost always point to SP-after-prologue; if not, emit a longer
659  // instruction sequence.
660  int BaseOffset = -AFI->getTaggedBasePointerOffset();
661  unsigned FrameReg;
662  StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
663  MF, BaseOffset, false /*isFixed*/, FrameReg,
664  /*PreferFP=*/false,
665  /*ForSimm=*/true);
666  Register SrcReg = FrameReg;
667  if (FrameRegOffset) {
668  // Use output register as temporary.
669  SrcReg = MI.getOperand(0).getReg();
670  emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg,
671  FrameRegOffset, TII);
672  }
673  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG))
674  .add(MI.getOperand(0))
675  .addUse(SrcReg)
676  .add(MI.getOperand(2));
677  MI.eraseFromParent();
678  return true;
679  }
680  case AArch64::TAGPstack: {
681  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDG))
682  .add(MI.getOperand(0))
683  .add(MI.getOperand(1))
684  .add(MI.getOperand(2))
685  .add(MI.getOperand(4));
686  MI.eraseFromParent();
687  return true;
688  }
689  case AArch64::STGloop:
690  case AArch64::STZGloop:
691  return expandSetTagLoop(MBB, MBBI, NextMBBI);
692  }
693  return false;
694 }
695 
696 /// Iterate over the instructions in basic block MBB and expand any
697 /// pseudo instructions. Return true if anything was modified.
698 bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
699  bool Modified = false;
700 
701  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
702  while (MBBI != E) {
703  MachineBasicBlock::iterator NMBBI = std::next(MBBI);
704  Modified |= expandMI(MBB, MBBI, NMBBI);
705  MBBI = NMBBI;
706  }
707 
708  return Modified;
709 }
710 
711 bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
712  TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
713 
714  bool Modified = false;
715  for (auto &MBB : MF)
716  Modified |= expandMBB(MBB);
717  return Modified;
718 }
719 
720 /// Returns an instance of the pseudo instruction expansion pass.
722  return new AArch64ExpandPseudo();
723 }
unsigned getTargetFlags() const
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative...
const MachineInstrBuilder & add(const MachineOperand &MO) const
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
StackOffset is a wrapper around scalable and non-scalable offsets and is used in several functions su...
This class represents lattice values for constants.
Definition: AllocatorList.h:23
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:178
A debug info location.
Definition: DebugLoc.h:33
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
bool isOSFuchsia() const
Definition: Triple.h:505
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:225
const HexagonInstrInfo * TII
INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo", AARCH64_EXPAND_PSEUDO_NAME, false, false) static void transferImpOps(MachineInstr &OldMI
Transfer implicit operands on the pseudo instruction to the instructions created from the expansion...
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
const char * getSymbolName() const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
virtual const TargetInstrInfo * getInstrInfo() const
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
unsigned getKillRegState(bool B)
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
unsigned getDeadRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
void initializeAArch64ExpandPseudoPass(PassRegistry &)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineInstrBuilder & UseMI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const GlobalValue * getGlobal() const
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
const Triple & getTargetTriple() const
self_iterator getIterator()
Definition: ilist_node.h:81
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
size_t size() const
Definition: SmallVector.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
static uint64_t add(uint64_t LeftOp, uint64_t RightOp)
Definition: FileCheck.cpp:243
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
MachineInstrBuilder MachineInstrBuilder & DefMI
FunctionPass * createAArch64ExpandPseudoPass()
Returns an instance of the pseudo instruction expansion pass.
int64_t getImm() const
CodeModel::Model getCodeModel() const
Returns the code model.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:48
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
int64_t getOffset() const
Return the offset from the symbol in this operand.
#define I(x, y, z)
Definition: MD5.cpp:58
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:292
#define AARCH64_EXPAND_PSEUDO_NAME
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow...
Register getReg() const
getReg - Returns the register number.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Wrapper class representing virtual and physical registers.
Definition: Register.h:19