LLVM  16.0.0git
AArch64ExpandPseudoInsts.cpp
Go to the documentation of this file.
1 //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass that expands pseudo instructions into target
10 // instructions to allow proper scheduling and other late optimizations. This
11 // pass should be run after register allocation but before the post-regalloc
12 // scheduling pass.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AArch64ExpandImm.h"
17 #include "AArch64InstrInfo.h"
19 #include "AArch64Subtarget.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/Triple.h"
33 #include "llvm/IR/DebugLoc.h"
34 #include "llvm/MC/MCInstrDesc.h"
35 #include "llvm/Pass.h"
36 #include "llvm/Support/CodeGen.h"
39 #include <cassert>
40 #include <cstdint>
41 #include <iterator>
42 #include <limits>
43 #include <utility>
44 
45 using namespace llvm;
46 
47 #define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
48 
49 namespace {
50 
51 class AArch64ExpandPseudo : public MachineFunctionPass {
52 public:
53  const AArch64InstrInfo *TII;
54 
55  static char ID;
56 
57  AArch64ExpandPseudo() : MachineFunctionPass(ID) {
59  }
60 
61  bool runOnMachineFunction(MachineFunction &Fn) override;
62 
63  StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
64 
65 private:
66  bool expandMBB(MachineBasicBlock &MBB);
68  MachineBasicBlock::iterator &NextMBBI);
70  unsigned BitSize);
71 
72  bool expand_DestructiveOp(MachineInstr &MI, MachineBasicBlock &MBB,
75  unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
76  unsigned ExtendImm, unsigned ZeroReg,
77  MachineBasicBlock::iterator &NextMBBI);
78  bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
80  MachineBasicBlock::iterator &NextMBBI);
81  bool expandSetTagLoop(MachineBasicBlock &MBB,
83  MachineBasicBlock::iterator &NextMBBI);
84  bool expandSVESpillFill(MachineBasicBlock &MBB,
85  MachineBasicBlock::iterator MBBI, unsigned Opc,
86  unsigned N);
87  bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
90  bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
92  MachineBasicBlock *expandRestoreZA(MachineBasicBlock &MBB,
94  MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB,
96 };
97 
98 } // end anonymous namespace
99 
100 char AArch64ExpandPseudo::ID = 0;
101 
102 INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo",
103  AARCH64_EXPAND_PSEUDO_NAME, false, false)
104 
105 /// Transfer implicit operands on the pseudo instruction to the
106 /// instructions created from the expansion.
107 static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
109  const MCInstrDesc &Desc = OldMI.getDesc();
110  for (const MachineOperand &MO :
111  llvm::drop_begin(OldMI.operands(), Desc.getNumOperands())) {
112  assert(MO.isReg() && MO.getReg());
113  if (MO.isUse())
114  UseMI.add(MO);
115  else
116  DefMI.add(MO);
117  }
118 }
119 
120 /// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
121 /// real move-immediate instructions to synthesize the immediate.
124  unsigned BitSize) {
125  MachineInstr &MI = *MBBI;
126  Register DstReg = MI.getOperand(0).getReg();
127  uint64_t RenamableState =
128  MI.getOperand(0).isRenamable() ? RegState::Renamable : 0;
129  uint64_t Imm = MI.getOperand(1).getImm();
130 
131  if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
132  // Useless def, and we don't want to risk creating an invalid ORR (which
133  // would really write to sp).
134  MI.eraseFromParent();
135  return true;
136  }
137 
140  assert(Insn.size() != 0);
141 
143  for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
144  bool LastItem = std::next(I) == E;
145  switch (I->Opcode)
146  {
147  default: llvm_unreachable("unhandled!"); break;
148 
149  case AArch64::ORRWri:
150  case AArch64::ORRXri:
151  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
152  .add(MI.getOperand(0))
153  .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
154  .addImm(I->Op2));
155  break;
156  case AArch64::MOVNWi:
157  case AArch64::MOVNXi:
158  case AArch64::MOVZWi:
159  case AArch64::MOVZXi: {
160  bool DstIsDead = MI.getOperand(0).isDead();
161  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
162  .addReg(DstReg, RegState::Define |
163  getDeadRegState(DstIsDead && LastItem) |
164  RenamableState)
165  .addImm(I->Op1)
166  .addImm(I->Op2));
167  } break;
168  case AArch64::MOVKWi:
169  case AArch64::MOVKXi: {
170  Register DstReg = MI.getOperand(0).getReg();
171  bool DstIsDead = MI.getOperand(0).isDead();
172  MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
173  .addReg(DstReg,
175  getDeadRegState(DstIsDead && LastItem) |
176  RenamableState)
177  .addReg(DstReg)
178  .addImm(I->Op1)
179  .addImm(I->Op2));
180  } break;
181  }
182  }
183  transferImpOps(MI, MIBS.front(), MIBS.back());
184  MI.eraseFromParent();
185  return true;
186 }
187 
188 bool AArch64ExpandPseudo::expandCMP_SWAP(
190  unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
191  MachineBasicBlock::iterator &NextMBBI) {
192  MachineInstr &MI = *MBBI;
193  MIMetadata MIMD(MI);
194  const MachineOperand &Dest = MI.getOperand(0);
195  Register StatusReg = MI.getOperand(1).getReg();
196  bool StatusDead = MI.getOperand(1).isDead();
197  // Duplicating undef operands into 2 instructions does not guarantee the same
198  // value on both; However undef should be replaced by xzr anyway.
199  assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
200  Register AddrReg = MI.getOperand(2).getReg();
201  Register DesiredReg = MI.getOperand(3).getReg();
202  Register NewReg = MI.getOperand(4).getReg();
203 
204  MachineFunction *MF = MBB.getParent();
205  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
206  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
207  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
208 
209  MF->insert(++MBB.getIterator(), LoadCmpBB);
210  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
211  MF->insert(++StoreBB->getIterator(), DoneBB);
212 
213  // .Lloadcmp:
214  // mov wStatus, 0
215  // ldaxr xDest, [xAddr]
216  // cmp xDest, xDesired
217  // b.ne .Ldone
218  if (!StatusDead)
219  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::MOVZWi), StatusReg)
220  .addImm(0).addImm(0);
221  BuildMI(LoadCmpBB, MIMD, TII->get(LdarOp), Dest.getReg())
222  .addReg(AddrReg);
223  BuildMI(LoadCmpBB, MIMD, TII->get(CmpOp), ZeroReg)
224  .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
225  .addReg(DesiredReg)
226  .addImm(ExtendImm);
227  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::Bcc))
229  .addMBB(DoneBB)
230  .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);
231  LoadCmpBB->addSuccessor(DoneBB);
232  LoadCmpBB->addSuccessor(StoreBB);
233 
234  // .Lstore:
235  // stlxr wStatus, xNew, [xAddr]
236  // cbnz wStatus, .Lloadcmp
237  BuildMI(StoreBB, MIMD, TII->get(StlrOp), StatusReg)
238  .addReg(NewReg)
239  .addReg(AddrReg);
240  BuildMI(StoreBB, MIMD, TII->get(AArch64::CBNZW))
241  .addReg(StatusReg, getKillRegState(StatusDead))
242  .addMBB(LoadCmpBB);
243  StoreBB->addSuccessor(LoadCmpBB);
244  StoreBB->addSuccessor(DoneBB);
245 
246  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
247  DoneBB->transferSuccessors(&MBB);
248 
249  MBB.addSuccessor(LoadCmpBB);
250 
251  NextMBBI = MBB.end();
252  MI.eraseFromParent();
253 
254  // Recompute livein lists.
255  LivePhysRegs LiveRegs;
256  computeAndAddLiveIns(LiveRegs, *DoneBB);
257  computeAndAddLiveIns(LiveRegs, *StoreBB);
258  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
259  // Do an extra pass around the loop to get loop carried registers right.
260  StoreBB->clearLiveIns();
261  computeAndAddLiveIns(LiveRegs, *StoreBB);
262  LoadCmpBB->clearLiveIns();
263  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
264 
265  return true;
266 }
267 
268 bool AArch64ExpandPseudo::expandCMP_SWAP_128(
270  MachineBasicBlock::iterator &NextMBBI) {
271  MachineInstr &MI = *MBBI;
272  MIMetadata MIMD(MI);
273  MachineOperand &DestLo = MI.getOperand(0);
274  MachineOperand &DestHi = MI.getOperand(1);
275  Register StatusReg = MI.getOperand(2).getReg();
276  bool StatusDead = MI.getOperand(2).isDead();
277  // Duplicating undef operands into 2 instructions does not guarantee the same
278  // value on both; However undef should be replaced by xzr anyway.
279  assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
280  Register AddrReg = MI.getOperand(3).getReg();
281  Register DesiredLoReg = MI.getOperand(4).getReg();
282  Register DesiredHiReg = MI.getOperand(5).getReg();
283  Register NewLoReg = MI.getOperand(6).getReg();
284  Register NewHiReg = MI.getOperand(7).getReg();
285 
286  unsigned LdxpOp, StxpOp;
287 
288  switch (MI.getOpcode()) {
289  case AArch64::CMP_SWAP_128_MONOTONIC:
290  LdxpOp = AArch64::LDXPX;
291  StxpOp = AArch64::STXPX;
292  break;
293  case AArch64::CMP_SWAP_128_RELEASE:
294  LdxpOp = AArch64::LDXPX;
295  StxpOp = AArch64::STLXPX;
296  break;
297  case AArch64::CMP_SWAP_128_ACQUIRE:
298  LdxpOp = AArch64::LDAXPX;
299  StxpOp = AArch64::STXPX;
300  break;
301  case AArch64::CMP_SWAP_128:
302  LdxpOp = AArch64::LDAXPX;
303  StxpOp = AArch64::STLXPX;
304  break;
305  default:
306  llvm_unreachable("Unexpected opcode");
307  }
308 
309  MachineFunction *MF = MBB.getParent();
310  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
311  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
312  auto FailBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
313  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
314 
315  MF->insert(++MBB.getIterator(), LoadCmpBB);
316  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
317  MF->insert(++StoreBB->getIterator(), FailBB);
318  MF->insert(++FailBB->getIterator(), DoneBB);
319 
320  // .Lloadcmp:
321  // ldaxp xDestLo, xDestHi, [xAddr]
322  // cmp xDestLo, xDesiredLo
323  // sbcs xDestHi, xDesiredHi
324  // b.ne .Ldone
325  BuildMI(LoadCmpBB, MIMD, TII->get(LdxpOp))
326  .addReg(DestLo.getReg(), RegState::Define)
327  .addReg(DestHi.getReg(), RegState::Define)
328  .addReg(AddrReg);
329  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR)
330  .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
331  .addReg(DesiredLoReg)
332  .addImm(0);
333  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg)
334  .addUse(AArch64::WZR)
335  .addUse(AArch64::WZR)
337  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR)
338  .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
339  .addReg(DesiredHiReg)
340  .addImm(0);
341  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg)
342  .addUse(StatusReg, RegState::Kill)
343  .addUse(StatusReg, RegState::Kill)
345  BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CBNZW))
346  .addUse(StatusReg, getKillRegState(StatusDead))
347  .addMBB(FailBB);
348  LoadCmpBB->addSuccessor(FailBB);
349  LoadCmpBB->addSuccessor(StoreBB);
350 
351  // .Lstore:
352  // stlxp wStatus, xNewLo, xNewHi, [xAddr]
353  // cbnz wStatus, .Lloadcmp
354  BuildMI(StoreBB, MIMD, TII->get(StxpOp), StatusReg)
355  .addReg(NewLoReg)
356  .addReg(NewHiReg)
357  .addReg(AddrReg);
358  BuildMI(StoreBB, MIMD, TII->get(AArch64::CBNZW))
359  .addReg(StatusReg, getKillRegState(StatusDead))
360  .addMBB(LoadCmpBB);
361  BuildMI(StoreBB, MIMD, TII->get(AArch64::B)).addMBB(DoneBB);
362  StoreBB->addSuccessor(LoadCmpBB);
363  StoreBB->addSuccessor(DoneBB);
364 
365  // .Lfail:
366  // stlxp wStatus, xDestLo, xDestHi, [xAddr]
367  // cbnz wStatus, .Lloadcmp
368  BuildMI(FailBB, MIMD, TII->get(StxpOp), StatusReg)
369  .addReg(DestLo.getReg())
370  .addReg(DestHi.getReg())
371  .addReg(AddrReg);
372  BuildMI(FailBB, MIMD, TII->get(AArch64::CBNZW))
373  .addReg(StatusReg, getKillRegState(StatusDead))
374  .addMBB(LoadCmpBB);
375  FailBB->addSuccessor(LoadCmpBB);
376  FailBB->addSuccessor(DoneBB);
377 
378  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
379  DoneBB->transferSuccessors(&MBB);
380 
381  MBB.addSuccessor(LoadCmpBB);
382 
383  NextMBBI = MBB.end();
384  MI.eraseFromParent();
385 
386  // Recompute liveness bottom up.
387  LivePhysRegs LiveRegs;
388  computeAndAddLiveIns(LiveRegs, *DoneBB);
389  computeAndAddLiveIns(LiveRegs, *FailBB);
390  computeAndAddLiveIns(LiveRegs, *StoreBB);
391  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
392 
393  // Do an extra pass in the loop to get the loop carried dependencies right.
394  FailBB->clearLiveIns();
395  computeAndAddLiveIns(LiveRegs, *FailBB);
396  StoreBB->clearLiveIns();
397  computeAndAddLiveIns(LiveRegs, *StoreBB);
398  LoadCmpBB->clearLiveIns();
399  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
400 
401  return true;
402 }
403 
404 /// \brief Expand Pseudos to Instructions with destructive operands.
405 ///
406 /// This mechanism uses MOVPRFX instructions for zeroing the false lanes
407 /// or for fixing relaxed register allocation conditions to comply with
408 /// the instructions register constraints. The latter case may be cheaper
409 /// than setting the register constraints in the register allocator,
410 /// since that will insert regular MOV instructions rather than MOVPRFX.
411 ///
412 /// Example (after register allocation):
413 ///
414 /// FSUB_ZPZZ_ZERO_B Z0, Pg, Z1, Z0
415 ///
416 /// * The Pseudo FSUB_ZPZZ_ZERO_B maps to FSUB_ZPmZ_B.
417 /// * We cannot map directly to FSUB_ZPmZ_B because the register
418 /// constraints of the instruction are not met.
419 /// * Also the _ZERO specifies the false lanes need to be zeroed.
420 ///
421 /// We first try to see if the destructive operand == result operand,
422 /// if not, we try to swap the operands, e.g.
423 ///
424 /// FSUB_ZPmZ_B Z0, Pg/m, Z0, Z1
425 ///
426 /// But because FSUB_ZPmZ is not commutative, this is semantically
427 /// different, so we need a reverse instruction:
428 ///
429 /// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
430 ///
431 /// Then we implement the zeroing of the false lanes of Z0 by adding
432 /// a zeroing MOVPRFX instruction:
433 ///
434 /// MOVPRFX_ZPzZ_B Z0, Pg/z, Z0
435 /// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
436 ///
437 /// Note that this can only be done for _ZERO or _UNDEF variants where
438 /// we can guarantee the false lanes to be zeroed (by implementing this)
439 /// or that they are undef (don't care / not used), otherwise the
440 /// swapping of operands is illegal because the operation is not
441 /// (or cannot be emulated to be) fully commutative.
442 bool AArch64ExpandPseudo::expand_DestructiveOp(
443  MachineInstr &MI,
446  unsigned Opcode = AArch64::getSVEPseudoMap(MI.getOpcode());
447  uint64_t DType = TII->get(Opcode).TSFlags & AArch64::DestructiveInstTypeMask;
448  uint64_t FalseLanes = MI.getDesc().TSFlags & AArch64::FalseLanesMask;
449  bool FalseZero = FalseLanes == AArch64::FalseLanesZero;
450  Register DstReg = MI.getOperand(0).getReg();
451  bool DstIsDead = MI.getOperand(0).isDead();
452  bool UseRev = false;
453  unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;
454 
455  switch (DType) {
458  if (DstReg == MI.getOperand(3).getReg()) {
459  // FSUB Zd, Pg, Zs1, Zd ==> FSUBR Zd, Pg/m, Zd, Zs1
460  std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 3, 2);
461  UseRev = true;
462  break;
463  }
464  [[fallthrough]];
467  std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 2, 3);
468  break;
470  std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(2, 3, 3);
471  break;
473  std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 2, 3, 4);
474  if (DstReg == MI.getOperand(3).getReg()) {
475  // FMLA Zd, Pg, Za, Zd, Zm ==> FMAD Zdn, Pg, Zm, Za
476  std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 3, 4, 2);
477  UseRev = true;
478  } else if (DstReg == MI.getOperand(4).getReg()) {
479  // FMLA Zd, Pg, Za, Zm, Zd ==> FMAD Zdn, Pg, Zm, Za
480  std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 4, 3, 2);
481  UseRev = true;
482  }
483  break;
484  default:
485  llvm_unreachable("Unsupported Destructive Operand type");
486  }
487 
488  // MOVPRFX can only be used if the destination operand
489  // is the destructive operand, not as any other operand,
490  // so the Destructive Operand must be unique.
491  bool DOPRegIsUnique = false;
492  switch (DType) {
494  DOPRegIsUnique = DstReg != MI.getOperand(SrcIdx).getReg();
495  break;
498  DOPRegIsUnique =
499  DstReg != MI.getOperand(DOPIdx).getReg() ||
500  MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg();
501  break;
504  DOPRegIsUnique = true;
505  break;
507  DOPRegIsUnique =
508  DstReg != MI.getOperand(DOPIdx).getReg() ||
509  (MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg() &&
510  MI.getOperand(DOPIdx).getReg() != MI.getOperand(Src2Idx).getReg());
511  break;
512  }
513 
514  // Resolve the reverse opcode
515  if (UseRev) {
516  int NewOpcode;
517  // e.g. DIV -> DIVR
518  if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1)
519  Opcode = NewOpcode;
520  // e.g. DIVR -> DIV
521  else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1)
522  Opcode = NewOpcode;
523  }
524 
525  // Get the right MOVPRFX
526  uint64_t ElementSize = TII->getElementSizeForOpcode(Opcode);
527  unsigned MovPrfx, LSLZero, MovPrfxZero;
528  switch (ElementSize) {
531  MovPrfx = AArch64::MOVPRFX_ZZ;
532  LSLZero = AArch64::LSL_ZPmI_B;
533  MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;
534  break;
536  MovPrfx = AArch64::MOVPRFX_ZZ;
537  LSLZero = AArch64::LSL_ZPmI_H;
538  MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;
539  break;
541  MovPrfx = AArch64::MOVPRFX_ZZ;
542  LSLZero = AArch64::LSL_ZPmI_S;
543  MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;
544  break;
546  MovPrfx = AArch64::MOVPRFX_ZZ;
547  LSLZero = AArch64::LSL_ZPmI_D;
548  MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;
549  break;
550  default:
551  llvm_unreachable("Unsupported ElementSize");
552  }
553 
554  //
555  // Create the destructive operation (if required)
556  //
557  MachineInstrBuilder PRFX, DOP;
558  if (FalseZero) {
559  // If we cannot prefix the requested instruction we'll instead emit a
560  // prefixed_zeroing_mov for DestructiveBinary.
561  assert((DOPRegIsUnique || AArch64::DestructiveBinary == DType) &&
562  "The destructive operand should be unique");
563  assert(ElementSize != AArch64::ElementSizeNone &&
564  "This instruction is unpredicated");
565 
566  // Merge source operand into destination register
567  PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfxZero))
568  .addReg(DstReg, RegState::Define)
569  .addReg(MI.getOperand(PredIdx).getReg())
570  .addReg(MI.getOperand(DOPIdx).getReg());
571 
572  // After the movprfx, the destructive operand is same as Dst
573  DOPIdx = 0;
574 
575  // Create the additional LSL to zero the lanes when the DstReg is not
576  // unique. Zeros the lanes in z0 that aren't active in p0 with sequence
577  // movprfx z0.b, p0/z, z0.b; lsl z0.b, p0/m, z0.b, #0;
578  if (DType == AArch64::DestructiveBinary && !DOPRegIsUnique) {
579  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LSLZero))
580  .addReg(DstReg, RegState::Define)
581  .add(MI.getOperand(PredIdx))
582  .addReg(DstReg)
583  .addImm(0);
584  }
585  } else if (DstReg != MI.getOperand(DOPIdx).getReg()) {
586  assert(DOPRegIsUnique && "The destructive operand should be unique");
587  PRFX = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(MovPrfx))
588  .addReg(DstReg, RegState::Define)
589  .addReg(MI.getOperand(DOPIdx).getReg());
590  DOPIdx = 0;
591  }
592 
593  //
594  // Create the destructive operation
595  //
596  DOP = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode))
597  .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead));
598 
599  switch (DType) {
601  DOP.addReg(MI.getOperand(DOPIdx).getReg(), RegState::Kill)
602  .add(MI.getOperand(PredIdx))
603  .add(MI.getOperand(SrcIdx));
604  break;
609  DOP.add(MI.getOperand(PredIdx))
610  .addReg(MI.getOperand(DOPIdx).getReg(), RegState::Kill)
611  .add(MI.getOperand(SrcIdx));
612  break;
614  DOP.add(MI.getOperand(PredIdx))
615  .addReg(MI.getOperand(DOPIdx).getReg(), RegState::Kill)
616  .add(MI.getOperand(SrcIdx))
617  .add(MI.getOperand(Src2Idx));
618  break;
619  }
620 
621  if (PRFX) {
623  transferImpOps(MI, PRFX, DOP);
624  } else
625  transferImpOps(MI, DOP, DOP);
626 
627  MI.eraseFromParent();
628  return true;
629 }
630 
631 bool AArch64ExpandPseudo::expandSetTagLoop(
633  MachineBasicBlock::iterator &NextMBBI) {
634  MachineInstr &MI = *MBBI;
635  DebugLoc DL = MI.getDebugLoc();
636  Register SizeReg = MI.getOperand(0).getReg();
637  Register AddressReg = MI.getOperand(1).getReg();
638 
639  MachineFunction *MF = MBB.getParent();
640 
641  bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback;
642  const unsigned OpCode1 =
643  ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;
644  const unsigned OpCode2 =
645  ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
646 
647  unsigned Size = MI.getOperand(2).getImm();
648  assert(Size > 0 && Size % 16 == 0);
649  if (Size % (16 * 2) != 0) {
650  BuildMI(MBB, MBBI, DL, TII->get(OpCode1), AddressReg)
651  .addReg(AddressReg)
652  .addReg(AddressReg)
653  .addImm(1);
654  Size -= 16;
655  }
657  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), SizeReg)
658  .addImm(Size);
659  expandMOVImm(MBB, I, 64);
660 
661  auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
662  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
663 
664  MF->insert(++MBB.getIterator(), LoopBB);
665  MF->insert(++LoopBB->getIterator(), DoneBB);
666 
667  BuildMI(LoopBB, DL, TII->get(OpCode2))
668  .addDef(AddressReg)
669  .addReg(AddressReg)
670  .addReg(AddressReg)
671  .addImm(2)
672  .cloneMemRefs(MI)
673  .setMIFlags(MI.getFlags());
674  BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri))
675  .addDef(SizeReg)
676  .addReg(SizeReg)
677  .addImm(16 * 2)
678  .addImm(0);
679  BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB);
680 
681  LoopBB->addSuccessor(LoopBB);
682  LoopBB->addSuccessor(DoneBB);
683 
684  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
685  DoneBB->transferSuccessors(&MBB);
686 
687  MBB.addSuccessor(LoopBB);
688 
689  NextMBBI = MBB.end();
690  MI.eraseFromParent();
691  // Recompute liveness bottom up.
692  LivePhysRegs LiveRegs;
693  computeAndAddLiveIns(LiveRegs, *DoneBB);
694  computeAndAddLiveIns(LiveRegs, *LoopBB);
695  // Do an extra pass in the loop to get the loop carried dependencies right.
696  // FIXME: is this necessary?
697  LoopBB->clearLiveIns();
698  computeAndAddLiveIns(LiveRegs, *LoopBB);
699  DoneBB->clearLiveIns();
700  computeAndAddLiveIns(LiveRegs, *DoneBB);
701 
702  return true;
703 }
704 
705 bool AArch64ExpandPseudo::expandSVESpillFill(MachineBasicBlock &MBB,
707  unsigned Opc, unsigned N) {
708  const TargetRegisterInfo *TRI =
710  MachineInstr &MI = *MBBI;
711  for (unsigned Offset = 0; Offset < N; ++Offset) {
712  int ImmOffset = MI.getOperand(2).getImm() + Offset;
713  bool Kill = (Offset + 1 == N) ? MI.getOperand(1).isKill() : false;
714  assert(ImmOffset >= -256 && ImmOffset < 256 &&
715  "Immediate spill offset out of range");
716  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc))
717  .addReg(
718  TRI->getSubReg(MI.getOperand(0).getReg(), AArch64::zsub0 + Offset),
719  Opc == AArch64::LDR_ZXI ? RegState::Define : 0)
720  .addReg(MI.getOperand(1).getReg(), getKillRegState(Kill))
721  .addImm(ImmOffset);
722  }
723  MI.eraseFromParent();
724  return true;
725 }
726 
727 bool AArch64ExpandPseudo::expandCALL_RVMARKER(
729  // Expand CALL_RVMARKER pseudo to:
730  // - a branch to the call target, followed by
731  // - the special `mov x29, x29` marker, and
732  // - another branch, to the runtime function
733  // Mark the sequence as bundle, to avoid passes moving other code in between.
734  MachineInstr &MI = *MBBI;
735 
736  MachineInstr *OriginalCall;
737  MachineOperand &RVTarget = MI.getOperand(0);
738  MachineOperand &CallTarget = MI.getOperand(1);
739  assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
740  "invalid operand for regular call");
741  assert(RVTarget.isGlobal() && "invalid operand for attached call");
742  unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
743  OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr();
744  OriginalCall->addOperand(CallTarget);
745 
746  unsigned RegMaskStartIdx = 2;
747  // Skip register arguments. Those are added during ISel, but are not
748  // needed for the concrete branch.
749  while (!MI.getOperand(RegMaskStartIdx).isRegMask()) {
750  auto MOP = MI.getOperand(RegMaskStartIdx);
751  assert(MOP.isReg() && "can only add register operands");
753  MOP.getReg(), /*Def=*/false, /*Implicit=*/true));
754  RegMaskStartIdx++;
755  }
756  for (const MachineOperand &MO :
757  llvm::drop_begin(MI.operands(), RegMaskStartIdx))
758  OriginalCall->addOperand(MO);
759 
760  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))
761  .addReg(AArch64::FP, RegState::Define)
762  .addReg(AArch64::XZR)
763  .addReg(AArch64::FP)
764  .addImm(0);
765 
766  auto *RVCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BL))
767  .add(RVTarget)
768  .getInstr();
769 
770  if (MI.shouldUpdateCallSiteInfo())
771  MBB.getParent()->moveCallSiteInfo(&MI, OriginalCall);
772 
773  MI.eraseFromParent();
774  finalizeBundle(MBB, OriginalCall->getIterator(),
775  std::next(RVCall->getIterator()));
776  return true;
777 }
778 
779 bool AArch64ExpandPseudo::expandCALL_BTI(MachineBasicBlock &MBB,
781  // Expand CALL_BTI pseudo to:
782  // - a branch to the call target
783  // - a BTI instruction
784  // Mark the sequence as a bundle, to avoid passes moving other code in
785  // between.
786 
787  MachineInstr &MI = *MBBI;
788  MachineOperand &CallTarget = MI.getOperand(0);
789  assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
790  "invalid operand for regular call");
791  unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
792  MachineInstr *Call =
793  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr();
794  Call->addOperand(CallTarget);
795  Call->setCFIType(*MBB.getParent(), MI.getCFIType());
796 
797  MachineInstr *BTI =
798  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::HINT))
799  // BTI J so that setjmp can to BR to this.
800  .addImm(36)
801  .getInstr();
802 
803  if (MI.shouldUpdateCallSiteInfo())
804  MBB.getParent()->moveCallSiteInfo(&MI, Call);
805 
806  MI.eraseFromParent();
807  finalizeBundle(MBB, Call->getIterator(), std::next(BTI->getIterator()));
808  return true;
809 }
810 
811 bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
813  Register CtxReg = MBBI->getOperand(0).getReg();
814  Register BaseReg = MBBI->getOperand(1).getReg();
815  int Offset = MBBI->getOperand(2).getImm();
816  DebugLoc DL(MBBI->getDebugLoc());
817  auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
818 
819  if (STI.getTargetTriple().getArchName() != "arm64e") {
820  BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))
821  .addUse(CtxReg)
822  .addUse(BaseReg)
823  .addImm(Offset / 8)
826  return true;
827  }
828 
829  // We need to sign the context in an address-discriminated way. 0xc31a is a
830  // fixed random value, chosen as part of the ABI.
831  // add x16, xBase, #Offset
832  // movk x16, #0xc31a, lsl #48
833  // mov x17, x22/xzr
834  // pacdb x17, x16
835  // str x17, [xBase, #Offset]
836  unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
837  BuildMI(MBB, MBBI, DL, TII->get(Opc), AArch64::X16)
838  .addUse(BaseReg)
839  .addImm(abs(Offset))
840  .addImm(0)
842  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X16)
843  .addUse(AArch64::X16)
844  .addImm(0xc31a)
845  .addImm(48)
847  // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so
848  // move it somewhere before signing.
849  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::X17)
850  .addUse(AArch64::XZR)
851  .addUse(CtxReg)
852  .addImm(0)
854  BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACDB), AArch64::X17)
855  .addUse(AArch64::X17)
856  .addUse(AArch64::X16)
858  BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui))
859  .addUse(AArch64::X17)
860  .addUse(BaseReg)
861  .addImm(Offset / 8)
863 
865  return true;
866 }
867 
869 AArch64ExpandPseudo::expandRestoreZA(MachineBasicBlock &MBB,
871  MachineInstr &MI = *MBBI;
872  assert((std::next(MBBI) != MBB.end() ||
873  MI.getParent()->successors().begin() !=
874  MI.getParent()->successors().end()) &&
875  "Unexpected unreachable in block that restores ZA");
876 
877  // Compare TPIDR2_EL0 value against 0.
878  DebugLoc DL = MI.getDebugLoc();
879  MachineInstrBuilder Cbz = BuildMI(MBB, MBBI, DL, TII->get(AArch64::CBZX))
880  .add(MI.getOperand(0));
881 
882  // Split MBB and create two new blocks:
883  // - MBB now contains all instructions before RestoreZAPseudo.
884  // - SMBB contains the RestoreZAPseudo instruction only.
885  // - EndBB contains all instructions after RestoreZAPseudo.
886  MachineInstr &PrevMI = *std::prev(MBBI);
887  MachineBasicBlock *SMBB = MBB.splitAt(PrevMI, /*UpdateLiveIns*/ true);
888  MachineBasicBlock *EndBB = std::next(MI.getIterator()) == SMBB->end()
889  ? *SMBB->successors().begin()
890  : SMBB->splitAt(MI, /*UpdateLiveIns*/ true);
891 
892  // Add the SMBB label to the TB[N]Z instruction & create a branch to EndBB.
893  Cbz.addMBB(SMBB);
894  BuildMI(&MBB, DL, TII->get(AArch64::B))
895  .addMBB(EndBB);
896  MBB.addSuccessor(EndBB);
897 
898  // Replace the pseudo with a call (BL).
899  MachineInstrBuilder MIB =
900  BuildMI(*SMBB, SMBB->end(), DL, TII->get(AArch64::BL));
901  MIB.addReg(MI.getOperand(1).getReg(), RegState::Implicit);
902  for (unsigned I = 2; I < MI.getNumOperands(); ++I)
903  MIB.add(MI.getOperand(I));
904  BuildMI(SMBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
905 
906  MI.eraseFromParent();
907  return EndBB;
908 }
909 
911 AArch64ExpandPseudo::expandCondSMToggle(MachineBasicBlock &MBB,
913  MachineInstr &MI = *MBBI;
914  // In the case of a smstart/smstop before a unreachable, just remove the pseudo.
915  // Exception handling code generated by Clang may introduce unreachables and it
916  // seems unnecessary to restore pstate.sm when that happens. Note that it is
917  // not just an optimisation, the code below expects a successor instruction/block
918  // in order to split the block at MBBI.
919  if (std::next(MBBI) == MBB.end() &&
920  MI.getParent()->successors().begin() ==
921  MI.getParent()->successors().end()) {
922  MI.eraseFromParent();
923  return &MBB;
924  }
925 
926  // Expand the pseudo into smstart or smstop instruction. The pseudo has the
927  // following operands:
928  //
929  // MSRpstatePseudo <za|sm|both>, <0|1>, pstate.sm, expectedval, <regmask>
930  //
931  // The pseudo is expanded into a conditional smstart/smstop, with a
932  // check if pstate.sm (register) equals the expected value, and if not,
933  // invokes the smstart/smstop.
934  //
935  // As an example, the following block contains a normal call from a
936  // streaming-compatible function:
937  //
938  // OrigBB:
939  // MSRpstatePseudo 3, 0, %0, 0, <regmask> <- Conditional SMSTOP
940  // bl @normal_callee
941  // MSRpstatePseudo 3, 1, %0, 0, <regmask> <- Conditional SMSTART
942  //
943  // ...which will be transformed into:
944  //
945  // OrigBB:
946  // TBNZx %0:gpr64, 0, SMBB
947  // b EndBB
948  //
949  // SMBB:
950  // MSRpstatesvcrImm1 3, 0, <regmask> <- SMSTOP
951  //
952  // EndBB:
953  // bl @normal_callee
954  // MSRcond_pstatesvcrImm1 3, 1, <regmask> <- SMSTART
955  //
956  DebugLoc DL = MI.getDebugLoc();
957 
958  // Create the conditional branch based on the third operand of the
959  // instruction, which tells us if we are wrapping a normal or streaming
960  // function.
961  // We test the live value of pstate.sm and toggle pstate.sm if this is not the
962  // expected value for the callee (0 for a normal callee and 1 for a streaming
963  // callee).
964  auto PStateSM = MI.getOperand(2).getReg();
965  bool IsStreamingCallee = MI.getOperand(3).getImm();
966  unsigned Opc = IsStreamingCallee ? AArch64::TBZX : AArch64::TBNZX;
967  MachineInstrBuilder Tbx =
968  BuildMI(MBB, MBBI, DL, TII->get(Opc)).addReg(PStateSM).addImm(0);
969 
970  // Split MBB and create two new blocks:
971  // - MBB now contains all instructions before MSRcond_pstatesvcrImm1.
972  // - SMBB contains the MSRcond_pstatesvcrImm1 instruction only.
973  // - EndBB contains all instructions after MSRcond_pstatesvcrImm1.
974  MachineInstr &PrevMI = *std::prev(MBBI);
975  MachineBasicBlock *SMBB = MBB.splitAt(PrevMI, /*UpdateLiveIns*/ true);
976  MachineBasicBlock *EndBB = std::next(MI.getIterator()) == SMBB->end()
977  ? *SMBB->successors().begin()
978  : SMBB->splitAt(MI, /*UpdateLiveIns*/ true);
979 
980  // Add the SMBB label to the TB[N]Z instruction & create a branch to EndBB.
981  Tbx.addMBB(SMBB);
982  BuildMI(&MBB, DL, TII->get(AArch64::B))
983  .addMBB(EndBB);
984  MBB.addSuccessor(EndBB);
985 
986  // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.
987  MachineInstrBuilder MIB = BuildMI(*SMBB, SMBB->begin(), MI.getDebugLoc(),
988  TII->get(AArch64::MSRpstatesvcrImm1));
989  // Copy all but the second and third operands of MSRcond_pstatesvcrImm1 (as
990  // these contain the CopyFromReg for the first argument and the flag to
991  // indicate whether the callee is streaming or normal).
992  MIB.add(MI.getOperand(0));
993  MIB.add(MI.getOperand(1));
994  for (unsigned i = 4; i < MI.getNumOperands(); ++i)
995  MIB.add(MI.getOperand(i));
996 
997  BuildMI(SMBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
998 
999  MI.eraseFromParent();
1000  return EndBB;
1001 }
1002 
1003 /// If MBBI references a pseudo instruction that should be expanded here,
1004 /// do the expansion and return true. Otherwise return false.
1005 bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
1007  MachineBasicBlock::iterator &NextMBBI) {
1008  MachineInstr &MI = *MBBI;
1009  unsigned Opcode = MI.getOpcode();
1010 
1011  // Check if we can expand the destructive op
1012  int OrigInstr = AArch64::getSVEPseudoMap(MI.getOpcode());
1013  if (OrigInstr != -1) {
1014  auto &Orig = TII->get(OrigInstr);
1015  if ((Orig.TSFlags & AArch64::DestructiveInstTypeMask)
1017  return expand_DestructiveOp(MI, MBB, MBBI);
1018  }
1019  }
1020 
1021  switch (Opcode) {
1022  default:
1023  break;
1024 
1025  case AArch64::BSPv8i8:
1026  case AArch64::BSPv16i8: {
1027  Register DstReg = MI.getOperand(0).getReg();
1028  if (DstReg == MI.getOperand(3).getReg()) {
1029  // Expand to BIT
1030  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1031  TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8
1032  : AArch64::BITv16i8))
1033  .add(MI.getOperand(0))
1034  .add(MI.getOperand(3))
1035  .add(MI.getOperand(2))
1036  .add(MI.getOperand(1));
1037  } else if (DstReg == MI.getOperand(2).getReg()) {
1038  // Expand to BIF
1039  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1040  TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8
1041  : AArch64::BIFv16i8))
1042  .add(MI.getOperand(0))
1043  .add(MI.getOperand(2))
1044  .add(MI.getOperand(3))
1045  .add(MI.getOperand(1));
1046  } else {
1047  // Expand to BSL, use additional move if required
1048  if (DstReg == MI.getOperand(1).getReg()) {
1049  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1050  TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1051  : AArch64::BSLv16i8))
1052  .add(MI.getOperand(0))
1053  .add(MI.getOperand(1))
1054  .add(MI.getOperand(2))
1055  .add(MI.getOperand(3));
1056  } else {
1057  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1058  TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
1059  : AArch64::ORRv16i8))
1060  .addReg(DstReg,
1062  getRenamableRegState(MI.getOperand(0).isRenamable()))
1063  .add(MI.getOperand(1))
1064  .add(MI.getOperand(1));
1065  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1066  TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1067  : AArch64::BSLv16i8))
1068  .add(MI.getOperand(0))
1069  .addReg(DstReg,
1070  RegState::Kill |
1071  getRenamableRegState(MI.getOperand(0).isRenamable()))
1072  .add(MI.getOperand(2))
1073  .add(MI.getOperand(3));
1074  }
1075  }
1076  MI.eraseFromParent();
1077  return true;
1078  }
1079 
1080  case AArch64::ADDWrr:
1081  case AArch64::SUBWrr:
1082  case AArch64::ADDXrr:
1083  case AArch64::SUBXrr:
1084  case AArch64::ADDSWrr:
1085  case AArch64::SUBSWrr:
1086  case AArch64::ADDSXrr:
1087  case AArch64::SUBSXrr:
1088  case AArch64::ANDWrr:
1089  case AArch64::ANDXrr:
1090  case AArch64::BICWrr:
1091  case AArch64::BICXrr:
1092  case AArch64::ANDSWrr:
1093  case AArch64::ANDSXrr:
1094  case AArch64::BICSWrr:
1095  case AArch64::BICSXrr:
1096  case AArch64::EONWrr:
1097  case AArch64::EONXrr:
1098  case AArch64::EORWrr:
1099  case AArch64::EORXrr:
1100  case AArch64::ORNWrr:
1101  case AArch64::ORNXrr:
1102  case AArch64::ORRWrr:
1103  case AArch64::ORRXrr: {
1104  unsigned Opcode;
1105  switch (MI.getOpcode()) {
1106  default:
1107  return false;
1108  case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
1109  case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
1110  case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
1111  case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
1112  case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
1113  case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
1114  case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
1115  case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
1116  case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
1117  case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
1118  case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
1119  case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
1120  case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
1121  case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
1122  case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
1123  case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
1124  case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
1125  case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
1126  case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
1127  case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
1128  case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
1129  case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
1130  case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
1131  case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
1132  }
1133  MachineFunction &MF = *MBB.getParent();
1134  // Try to create new inst without implicit operands added.
1135  MachineInstr *NewMI = MF.CreateMachineInstr(
1136  TII->get(Opcode), MI.getDebugLoc(), /*NoImplicit=*/true);
1137  MBB.insert(MBBI, NewMI);
1138  MachineInstrBuilder MIB1(MF, NewMI);
1139  MIB1->setPCSections(MF, MI.getPCSections());
1140  MIB1.addReg(MI.getOperand(0).getReg(), RegState::Define)
1141  .add(MI.getOperand(1))
1142  .add(MI.getOperand(2))
1144  transferImpOps(MI, MIB1, MIB1);
1145  MI.eraseFromParent();
1146  return true;
1147  }
1148 
1149  case AArch64::LOADgot: {
1150  MachineFunction *MF = MBB.getParent();
1151  Register DstReg = MI.getOperand(0).getReg();
1152  const MachineOperand &MO1 = MI.getOperand(1);
1153  unsigned Flags = MO1.getTargetFlags();
1154 
1155  if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
1156  // Tiny codemodel expand to LDR
1157  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1158  TII->get(AArch64::LDRXl), DstReg);
1159 
1160  if (MO1.isGlobal()) {
1161  MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags);
1162  } else if (MO1.isSymbol()) {
1163  MIB.addExternalSymbol(MO1.getSymbolName(), Flags);
1164  } else {
1165  assert(MO1.isCPI() &&
1166  "Only expect globals, externalsymbols, or constant pools");
1167  MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags);
1168  }
1169  } else {
1170  // Small codemodel expand into ADRP + LDR.
1171  MachineFunction &MF = *MI.getParent()->getParent();
1172  DebugLoc DL = MI.getDebugLoc();
1173  MachineInstrBuilder MIB1 =
1174  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg);
1175 
1176  MachineInstrBuilder MIB2;
1179  unsigned Reg32 = TRI->getSubReg(DstReg, AArch64::sub_32);
1180  unsigned DstFlags = MI.getOperand(0).getTargetFlags();
1181  MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRWui))
1182  .addDef(Reg32)
1183  .addReg(DstReg, RegState::Kill)
1184  .addReg(DstReg, DstFlags | RegState::Implicit);
1185  } else {
1186  Register DstReg = MI.getOperand(0).getReg();
1187  MIB2 = BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRXui))
1188  .add(MI.getOperand(0))
1189  .addUse(DstReg, RegState::Kill);
1190  }
1191 
1192  if (MO1.isGlobal()) {
1193  MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE);
1194  MIB2.addGlobalAddress(MO1.getGlobal(), 0,
1196  } else if (MO1.isSymbol()) {
1197  MIB1.addExternalSymbol(MO1.getSymbolName(), Flags | AArch64II::MO_PAGE);
1198  MIB2.addExternalSymbol(MO1.getSymbolName(), Flags |
1201  } else {
1202  assert(MO1.isCPI() &&
1203  "Only expect globals, externalsymbols, or constant pools");
1204  MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
1205  Flags | AArch64II::MO_PAGE);
1206  MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
1207  Flags | AArch64II::MO_PAGEOFF |
1209  }
1210 
1211  transferImpOps(MI, MIB1, MIB2);
1212  }
1213  MI.eraseFromParent();
1214  return true;
1215  }
1216  case AArch64::MOVaddrBA: {
1217  MachineFunction &MF = *MI.getParent()->getParent();
1219  // blockaddress expressions have to come from a constant pool because the
1220  // largest addend (and hence offset within a function) allowed for ADRP is
1221  // only 8MB.
1222  const BlockAddress *BA = MI.getOperand(1).getBlockAddress();
1223  assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset");
1224 
1226  unsigned CPIdx = MCP->getConstantPoolIndex(BA, Align(8));
1227 
1228  Register DstReg = MI.getOperand(0).getReg();
1229  auto MIB1 =
1230  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)
1232  auto MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1233  TII->get(AArch64::LDRXui), DstReg)
1234  .addUse(DstReg)
1237  transferImpOps(MI, MIB1, MIB2);
1238  MI.eraseFromParent();
1239  return true;
1240  }
1241  }
1242  [[fallthrough]];
1243  case AArch64::MOVaddr:
1244  case AArch64::MOVaddrJT:
1245  case AArch64::MOVaddrCP:
1246  case AArch64::MOVaddrTLS:
1247  case AArch64::MOVaddrEXT: {
1248  // Expand into ADRP + ADD.
1249  Register DstReg = MI.getOperand(0).getReg();
1250  assert(DstReg != AArch64::XZR);
1251  MachineInstrBuilder MIB1 =
1252  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)
1253  .add(MI.getOperand(1));
1254 
1255  if (MI.getOperand(1).getTargetFlags() & AArch64II::MO_TAGGED) {
1256  // MO_TAGGED on the page indicates a tagged address. Set the tag now.
1257  // We do so by creating a MOVK that sets bits 48-63 of the register to
1258  // (global address + 0x100000000 - PC) >> 48. This assumes that we're in
1259  // the small code model so we can assume a binary size of <= 4GB, which
1260  // makes the untagged PC relative offset positive. The binary must also be
1261  // loaded into address range [0, 2^48). Both of these properties need to
1262  // be ensured at runtime when using tagged addresses.
1263  auto Tag = MI.getOperand(1);
1264  Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3);
1265  Tag.setOffset(0x100000000);
1266  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MOVKXi), DstReg)
1267  .addReg(DstReg)
1268  .add(Tag)
1269  .addImm(48);
1270  }
1271 
1272  MachineInstrBuilder MIB2 =
1273  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
1274  .add(MI.getOperand(0))
1275  .addReg(DstReg)
1276  .add(MI.getOperand(2))
1277  .addImm(0);
1278 
1279  transferImpOps(MI, MIB1, MIB2);
1280  MI.eraseFromParent();
1281  return true;
1282  }
1283  case AArch64::ADDlowTLS:
1284  // Produce a plain ADD
1285  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
1286  .add(MI.getOperand(0))
1287  .add(MI.getOperand(1))
1288  .add(MI.getOperand(2))
1289  .addImm(0);
1290  MI.eraseFromParent();
1291  return true;
1292 
1293  case AArch64::MOVbaseTLS: {
1294  Register DstReg = MI.getOperand(0).getReg();
1295  auto SysReg = AArch64SysReg::TPIDR_EL0;
1296  MachineFunction *MF = MBB.getParent();
1297  if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
1298  SysReg = AArch64SysReg::TPIDR_EL3;
1299  else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
1300  SysReg = AArch64SysReg::TPIDR_EL2;
1301  else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
1302  SysReg = AArch64SysReg::TPIDR_EL1;
1303  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg)
1304  .addImm(SysReg);
1305  MI.eraseFromParent();
1306  return true;
1307  }
1308 
1309  case AArch64::MOVi32imm:
1310  return expandMOVImm(MBB, MBBI, 32);
1311  case AArch64::MOVi64imm:
1312  return expandMOVImm(MBB, MBBI, 64);
1313  case AArch64::RET_ReallyLR: {
1314  // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
1315  // function and missing live-ins. We are fine in practice because callee
1316  // saved register handling ensures the register value is restored before
1317  // RET, but we need the undef flag here to appease the MachineVerifier
1318  // liveness checks.
1319  MachineInstrBuilder MIB =
1320  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET))
1321  .addReg(AArch64::LR, RegState::Undef);
1322  transferImpOps(MI, MIB, MIB);
1323  MI.eraseFromParent();
1324  return true;
1325  }
1326  case AArch64::CMP_SWAP_8:
1327  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,
1328  AArch64::SUBSWrx,
1330  AArch64::WZR, NextMBBI);
1331  case AArch64::CMP_SWAP_16:
1332  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,
1333  AArch64::SUBSWrx,
1335  AArch64::WZR, NextMBBI);
1336  case AArch64::CMP_SWAP_32:
1337  return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
1338  AArch64::SUBSWrs,
1340  AArch64::WZR, NextMBBI);
1341  case AArch64::CMP_SWAP_64:
1342  return expandCMP_SWAP(MBB, MBBI,
1343  AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
1345  AArch64::XZR, NextMBBI);
1346  case AArch64::CMP_SWAP_128:
1347  case AArch64::CMP_SWAP_128_RELEASE:
1348  case AArch64::CMP_SWAP_128_ACQUIRE:
1349  case AArch64::CMP_SWAP_128_MONOTONIC:
1350  return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
1351 
1352  case AArch64::AESMCrrTied:
1353  case AArch64::AESIMCrrTied: {
1354  MachineInstrBuilder MIB =
1355  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1356  TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
1357  AArch64::AESIMCrr))
1358  .add(MI.getOperand(0))
1359  .add(MI.getOperand(1));
1360  transferImpOps(MI, MIB, MIB);
1361  MI.eraseFromParent();
1362  return true;
1363  }
1364  case AArch64::IRGstack: {
1365  MachineFunction &MF = *MBB.getParent();
1366  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1367  const AArch64FrameLowering *TFI =
1368  MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
1369 
1370  // IRG does not allow immediate offset. getTaggedBasePointerOffset should
1371  // almost always point to SP-after-prologue; if not, emit a longer
1372  // instruction sequence.
1373  int BaseOffset = -AFI->getTaggedBasePointerOffset();
1374  Register FrameReg;
1375  StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1376  MF, BaseOffset, false /*isFixed*/, false /*isSVE*/, FrameReg,
1377  /*PreferFP=*/false,
1378  /*ForSimm=*/true);
1379  Register SrcReg = FrameReg;
1380  if (FrameRegOffset) {
1381  // Use output register as temporary.
1382  SrcReg = MI.getOperand(0).getReg();
1383  emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg,
1384  FrameRegOffset, TII);
1385  }
1386  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG))
1387  .add(MI.getOperand(0))
1388  .addUse(SrcReg)
1389  .add(MI.getOperand(2));
1390  MI.eraseFromParent();
1391  return true;
1392  }
1393  case AArch64::TAGPstack: {
1394  int64_t Offset = MI.getOperand(2).getImm();
1395  BuildMI(MBB, MBBI, MI.getDebugLoc(),
1396  TII->get(Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))
1397  .add(MI.getOperand(0))
1398  .add(MI.getOperand(1))
1399  .addImm(std::abs(Offset))
1400  .add(MI.getOperand(4));
1401  MI.eraseFromParent();
1402  return true;
1403  }
1404  case AArch64::STGloop_wback:
1405  case AArch64::STZGloop_wback:
1406  return expandSetTagLoop(MBB, MBBI, NextMBBI);
1407  case AArch64::STGloop:
1408  case AArch64::STZGloop:
1410  "Non-writeback variants of STGloop / STZGloop should not "
1411  "survive past PrologEpilogInserter.");
1412  case AArch64::STR_ZZZZXI:
1413  return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 4);
1414  case AArch64::STR_ZZZXI:
1415  return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 3);
1416  case AArch64::STR_ZZXI:
1417  return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 2);
1418  case AArch64::LDR_ZZZZXI:
1419  return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 4);
1420  case AArch64::LDR_ZZZXI:
1421  return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 3);
1422  case AArch64::LDR_ZZXI:
1423  return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);
1424  case AArch64::BLR_RVMARKER:
1425  return expandCALL_RVMARKER(MBB, MBBI);
1426  case AArch64::BLR_BTI:
1427  return expandCALL_BTI(MBB, MBBI);
1428  case AArch64::StoreSwiftAsyncContext:
1429  return expandStoreSwiftAsyncContext(MBB, MBBI);
1430  case AArch64::RestoreZAPseudo: {
1431  auto *NewMBB = expandRestoreZA(MBB, MBBI);
1432  if (NewMBB != &MBB)
1433  NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
1434  return true;
1435  }
1436  case AArch64::MSRpstatePseudo: {
1437  auto *NewMBB = expandCondSMToggle(MBB, MBBI);
1438  if (NewMBB != &MBB)
1439  NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
1440  return true;
1441  }
1442  case AArch64::OBSCURE_COPY: {
1443  if (MI.getOperand(0).getReg() != MI.getOperand(1).getReg()) {
1444  BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))
1445  .add(MI.getOperand(0))
1446  .addReg(AArch64::XZR)
1447  .add(MI.getOperand(1))
1448  .addImm(0);
1449  }
1450  MI.eraseFromParent();
1451  return true;
1452  }
1453  }
1454  return false;
1455 }
1456 
1457 /// Iterate over the instructions in basic block MBB and expand any
1458 /// pseudo instructions. Return true if anything was modified.
1459 bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
1460  bool Modified = false;
1461 
1463  while (MBBI != E) {
1464  MachineBasicBlock::iterator NMBBI = std::next(MBBI);
1465  Modified |= expandMI(MBB, MBBI, NMBBI);
1466  MBBI = NMBBI;
1467  }
1468 
1469  return Modified;
1470 }
1471 
1472 bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
1473  TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
1474 
1475  bool Modified = false;
1476  for (auto &MBB : MF)
1477  Modified |= expandMBB(MBB);
1478  return Modified;
1479 }
1480 
1481 /// Returns an instance of the pseudo instruction expansion pass.
1483  return new AArch64ExpandPseudo();
1484 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:77
i
i
Definition: README.txt:29
llvm::AArch64II::MO_G3
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
Definition: AArch64BaseInfo.h:743
llvm::AArch64ISD::LOADgot
@ LOADgot
Definition: AArch64ISelLowering.h:78
llvm::AArch64::ElementSizeNone
@ ElementSizeNone
Definition: AArch64InstrInfo.h:549
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
MachineInstr.h
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm::MachineConstantPool::getConstantPoolIndex
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
Definition: MachineFunction.cpp:1479
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
AArch64MachineFunctionInfo.h
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:107
llvm::drop_begin
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:387
llvm::AArch64CC::NE
@ NE
Definition: AArch64BaseInfo.h:256
llvm::MachineOperand::CreateReg
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
Definition: MachineOperand.h:800
llvm::AArch64PACKey::ID
ID
Definition: AArch64BaseInfo.h:818
llvm::MachineBasicBlock::getBasicBlock
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Definition: MachineBasicBlock.h:209
llvm::AArch64_AM::LSL
@ LSL
Definition: AArch64AddressingModes.h:35
AARCH64_EXPAND_PSEUDO_NAME
#define AARCH64_EXPAND_PSEUDO_NAME
Definition: AArch64ExpandPseudoInsts.cpp:47
MCInstrDesc.h
llvm::MachineOperand::getGlobal
const GlobalValue * getGlobal() const
Definition: MachineOperand.h:572
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:224
llvm::AArch64::FalseLanesZero
@ FalseLanesZero
Definition: AArch64InstrInfo.h:572
Pass.h
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:95
llvm::AArch64::ElementSizeS
@ ElementSizeS
Definition: AArch64InstrInfo.h:552
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
AArch64BaseInfo.h
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
MachineBasicBlock.h
llvm::LivePhysRegs
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:50
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:127
llvm::AArch64II::MO_PREL
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
Definition: AArch64BaseInfo.h:794
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:237
llvm::MachineFunction::moveCallSiteInfo
void moveCallSiteInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
Definition: MachineFunction.cpp:954
llvm::MachineOperand::isSymbol
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
Definition: MachineOperand.h:340
llvm::MachineFunction::CreateMachineInstr
MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)
CreateMachineInstr - Allocate a new MachineInstr.
Definition: MachineFunction.cpp:373
DenseMap.h
llvm::MachineFunction::insert
void insert(iterator MBBI, MachineBasicBlock *MBB)
Definition: MachineFunction.h:873
llvm::getDeadRegState
unsigned getDeadRegState(bool B)
Definition: MachineInstrBuilder.h:549
llvm::AArch64::ElementSizeB
@ ElementSizeB
Definition: AArch64InstrInfo.h:550
llvm::AArch64::DestructiveTernaryCommWithRev
@ DestructiveTernaryCommWithRev
Definition: AArch64InstrInfo.h:566
llvm::dwarf::Tag
Tag
Definition: Dwarf.h:105
llvm::MachineOperand::getOffset
int64_t getOffset() const
Return the offset from the symbol in this operand.
Definition: MachineOperand.h:609
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::ARCISD::BL
@ BL
Definition: ARCISelLowering.h:34
llvm::AArch64FrameLowering
Definition: AArch64FrameLowering.h:21
llvm::MachineBasicBlock::addSuccessor
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:762
llvm::AArch64::getSVERevInstr
int getSVERevInstr(uint16_t Opcode)
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition: MachineInstrBuilder.h:116
llvm::finalizeBundle
void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
Definition: MachineInstrBundle.cpp:124
AArch64InstrInfo.h
llvm::getRenamableRegState
unsigned getRenamableRegState(bool B)
Definition: MachineInstrBuilder.h:561
TargetMachine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::AArch64InstrInfo
Definition: AArch64InstrInfo.h:36
llvm::AArch64ISD::MRS
@ MRS
Definition: AArch64ISelLowering.h:301
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::initializeAArch64ExpandPseudoPass
void initializeAArch64ExpandPseudoPass(PassRegistry &)
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:755
llvm::AArch64Subtarget::isTargetILP32
bool isTargetILP32() const
Definition: AArch64Subtarget.h:273
llvm::MachineBasicBlock::eraseFromParent
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
Definition: MachineBasicBlock.cpp:1347
llvm::AArch64ISD::OBSCURE_COPY
@ OBSCURE_COPY
Definition: AArch64ISelLowering.h:67
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
llvm::MachineInstr::FrameSetup
@ FrameSetup
Definition: MachineInstr.h:84
llvm::AArch64::DestructiveUnaryPassthru
@ DestructiveUnaryPassthru
Definition: AArch64InstrInfo.h:567
llvm::AArch64_AM::getShifterImm
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Definition: AArch64AddressingModes.h:99
llvm::MachineBasicBlock::splitAt
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
Definition: MachineBasicBlock.cpp:985
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
LoopDeletionResult::Modified
@ Modified
llvm::computeAndAddLiveIns
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().
Definition: LivePhysRegs.cpp:341
DebugLoc.h
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
llvm::MachineInstrBuilder::addExternalSymbol
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:184
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::MachineInstrBuilder::cloneMemRefs
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Definition: MachineInstrBuilder.h:213
AArch64AddressingModes.h
llvm::MIMetadata
Set of metadata that should be preserved when using BuildMI().
Definition: MachineInstrBuilder.h:335
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:657
llvm::AArch64::NotDestructive
@ NotDestructive
Definition: AArch64InstrInfo.h:558
llvm::MachineInstrBuilder::setMIFlag
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
Definition: MachineInstrBuilder.h:278
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:52
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:416
llvm::MachineOperand::getTargetFlags
unsigned getTargetFlags() const
Definition: MachineOperand.h:220
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:320
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
uint64_t
llvm::AArch64FunctionInfo::getTaggedBasePointerOffset
unsigned getTaggedBasePointerOffset() const
Definition: AArch64MachineFunctionInfo.h:415
AArch64ExpandImm.h
llvm::createAArch64ExpandPseudoPass
FunctionPass * createAArch64ExpandPseudoPass()
Returns an instance of the pseudo instruction expansion pass.
Definition: AArch64ExpandPseudoInsts.cpp:1482
llvm::AArch64II::MO_NC
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
Definition: AArch64BaseInfo.h:775
llvm::AArch64II::MO_PAGEOFF
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
Definition: AArch64BaseInfo.h:739
llvm::MachineConstantPool
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
Definition: MachineConstantPool.h:117
llvm::MachineOperand::isDead
bool isDead() const
Definition: MachineOperand.h:384
llvm::AArch64FunctionInfo
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
Definition: AArch64MachineFunctionInfo.h:38
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::RegState::Renamable
@ Renamable
Register that may be renamed.
Definition: MachineInstrBuilder.h:61
MachineConstantPool.h
llvm::MachineOperand::isCPI
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
Definition: MachineOperand.h:332
llvm::MachineFunction::CreateMachineBasicBlock
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Definition: MachineFunction.cpp:439
MachineFunctionPass.h
llvm::MachineFunction::getConstantPool
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
Definition: MachineFunction.h:689
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:261
llvm::emitFrameOffset
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
Definition: AArch64InstrInfo.cpp:4398
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:123
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::MachineFunction
Definition: MachineFunction.h:257
Triple.h
llvm::BlockAddress
The address of a basic block.
Definition: Constants.h:849
llvm::MachineBasicBlock::successors
iterator_range< succ_iterator > successors()
Definition: MachineBasicBlock.h:392
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition: AArch64SLSHardening.cpp:75
llvm::AArch64::FalseLanesMask
@ FalseLanesMask
Definition: AArch64InstrInfo.h:571
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::RegState::Implicit
@ Implicit
Not emitted register (e.g. carry, or temporary result).
Definition: MachineInstrBuilder.h:46
llvm::StackOffset
StackOffset is a class to represent an offset with 2 dimensions, named fixed and scalable,...
Definition: TypeSize.h:134
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
TargetSubtargetInfo.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::CodeModel::Tiny
@ Tiny
Definition: CodeGen.h:28
llvm::AArch64CC::EQ
@ EQ
Definition: AArch64BaseInfo.h:255
llvm::MachineInstrBuilder::getInstr
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Definition: MachineInstrBuilder.h:89
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::AArch64_AM::getArithExtendImm
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
Definition: AArch64AddressingModes.h:171
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::AArch64::ElementSizeD
@ ElementSizeD
Definition: AArch64InstrInfo.h:553
Insn
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
Definition: AArch64MIPeepholeOpt.cpp:130
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:653
llvm::AArch64_IMM::expandMOVImm
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
Definition: AArch64ExpandImm.cpp:303
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Definition: MachineBasicBlock.cpp:1327
llvm::ARCISD::RET
@ RET
Definition: ARCISelLowering.h:52
llvm::MachineOperand::getIndex
int getIndex() const
Definition: MachineOperand.h:566
llvm::MachineInstrBuilder::addConstantPoolIndex
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:158
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
llvm::AArch64::DestructiveInstTypeMask
@ DestructiveInstTypeMask
Definition: AArch64InstrInfo.h:557
llvm::AArch64ISD::ADRP
@ ADRP
Definition: AArch64ISelLowering.h:75
llvm::AArch64::DestructiveBinaryComm
@ DestructiveBinaryComm
Definition: AArch64InstrInfo.h:564
llvm::RISCVMatInt::Imm
@ Imm
Definition: RISCVMatInt.h:23
llvm::MachineInstrBuilder::addGlobalAddress
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:177
CodeGen.h
llvm::AArch64Subtarget::isTargetMachO
bool isTargetMachO() const
Definition: AArch64Subtarget.h:271
INITIALIZE_PASS
INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo", AARCH64_EXPAND_PSEUDO_NAME, false, false) static void transferImpOps(MachineInstr &OldMI
Transfer implicit operands on the pseudo instruction to the instructions created from the expansion.
llvm::getKillRegState
unsigned getKillRegState(bool B)
Definition: MachineInstrBuilder.h:546
AArch64Subtarget.h
llvm::AArch64::DestructiveBinary
@ DestructiveBinary
Definition: AArch64InstrInfo.h:563
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:357
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:305
MachineInstrBuilder.h
llvm::MachineOperand::getSymbolName
const char * getSymbolName() const
Definition: MachineOperand.h:617
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition: MachineInstrBuilder.h:273
N
#define N
DefMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Definition: AArch64ExpandPseudoInsts.cpp:108
llvm::MachineInstr::addOperand
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
Definition: MachineInstr.cpp:192
llvm::AArch64::getSVEPseudoMap
int getSVEPseudoMap(uint16_t Opcode)
llvm::AArch64_AM::UXTB
@ UXTB
Definition: AArch64AddressingModes.h:41
MachineOperand.h
llvm::TargetRegisterInfo::getSubReg
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Definition: TargetRegisterInfo.h:1142
llvm::AArch64::DestructiveBinaryImm
@ DestructiveBinaryImm
Definition: AArch64InstrInfo.h:561
llvm::AArch64::DestructiveBinaryCommWithRev
@ DestructiveBinaryCommWithRev
Definition: AArch64InstrInfo.h:565
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::omp::RTLDependInfoFields::Flags
@ Flags
llvm::TargetMachine::getCodeModel
CodeModel::Model getCodeModel() const
Returns the code model.
Definition: TargetMachine.h:224
llvm::AArch64_AM::UXTH
@ UXTH
Definition: AArch64AddressingModes.h:42
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::AArch64II::MO_TAGGED
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
Definition: AArch64BaseInfo.h:802
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
MachineFunction.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::abs
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1297
llvm::AArch64::getSVENonRevInstr
int getSVENonRevInstr(uint16_t Opcode)
llvm::MCInstrDesc::getNumOperands
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:230
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:307
llvm::AArch64II::MO_PAGE
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
Definition: AArch64BaseInfo.h:734
llvm::MachineOperand::isGlobal
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
Definition: MachineOperand.h:338
LivePhysRegs.h
llvm::AArch64::ElementSizeH
@ ElementSizeH
Definition: AArch64InstrInfo.h:551