LLVM  8.0.0svn
ARMLoadStoreOptimizer.cpp
Go to the documentation of this file.
1 //===- ARMLoadStoreOptimizer.cpp - ARM load / store opt. pass -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file contains a pass that performs load / store related peephole
11 /// optimizations. This pass should be run after register allocation.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "ARM.h"
16 #include "ARMBaseInstrInfo.h"
17 #include "ARMBaseRegisterInfo.h"
18 #include "ARMISelLowering.h"
19 #include "ARMMachineFunctionInfo.h"
20 #include "ARMSubtarget.h"
23 #include "Utils/ARMBaseInfo.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/DenseMap.h"
26 #include "llvm/ADT/DenseSet.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/SmallPtrSet.h"
29 #include "llvm/ADT/SmallSet.h"
30 #include "llvm/ADT/SmallVector.h"
31 #include "llvm/ADT/Statistic.h"
49 #include "llvm/IR/DataLayout.h"
50 #include "llvm/IR/DebugLoc.h"
51 #include "llvm/IR/DerivedTypes.h"
52 #include "llvm/IR/Function.h"
53 #include "llvm/IR/Type.h"
54 #include "llvm/MC/MCInstrDesc.h"
55 #include "llvm/Pass.h"
56 #include "llvm/Support/Allocator.h"
58 #include "llvm/Support/Debug.h"
61 #include <algorithm>
62 #include <cassert>
63 #include <cstddef>
64 #include <cstdlib>
65 #include <iterator>
66 #include <limits>
67 #include <utility>
68 
69 using namespace llvm;
70 
71 #define DEBUG_TYPE "arm-ldst-opt"
72 
73 STATISTIC(NumLDMGened , "Number of ldm instructions generated");
74 STATISTIC(NumSTMGened , "Number of stm instructions generated");
75 STATISTIC(NumVLDMGened, "Number of vldm instructions generated");
76 STATISTIC(NumVSTMGened, "Number of vstm instructions generated");
77 STATISTIC(NumLdStMoved, "Number of load / store instructions moved");
78 STATISTIC(NumLDRDFormed,"Number of ldrd created before allocation");
79 STATISTIC(NumSTRDFormed,"Number of strd created before allocation");
80 STATISTIC(NumLDRD2LDM, "Number of ldrd instructions turned back into ldm");
81 STATISTIC(NumSTRD2STM, "Number of strd instructions turned back into stm");
82 STATISTIC(NumLDRD2LDR, "Number of ldrd instructions turned back into ldr's");
83 STATISTIC(NumSTRD2STR, "Number of strd instructions turned back into str's");
84 
85 /// This switch disables formation of double/multi instructions that could
86 /// potentially lead to (new) alignment traps even with CCR.UNALIGN_TRP
87 /// disabled. This can be used to create libraries that are robust even when
88 /// users provoke undefined behaviour by supplying misaligned pointers.
89 /// \see mayCombineMisaligned()
90 static cl::opt<bool>
91 AssumeMisalignedLoadStores("arm-assume-misaligned-load-store", cl::Hidden,
92  cl::init(false), cl::desc("Be more conservative in ARM load/store opt"));
93 
94 #define ARM_LOAD_STORE_OPT_NAME "ARM load / store optimization pass"
95 
96 namespace {
97 
98  /// Post- register allocation pass the combine load / store instructions to
99  /// form ldm / stm instructions.
100  struct ARMLoadStoreOpt : public MachineFunctionPass {
101  static char ID;
102 
103  const MachineFunction *MF;
104  const TargetInstrInfo *TII;
105  const TargetRegisterInfo *TRI;
106  const ARMSubtarget *STI;
107  const TargetLowering *TL;
108  ARMFunctionInfo *AFI;
109  LivePhysRegs LiveRegs;
110  RegisterClassInfo RegClassInfo;
112  bool LiveRegsValid;
113  bool RegClassInfoValid;
114  bool isThumb1, isThumb2;
115 
116  ARMLoadStoreOpt() : MachineFunctionPass(ID) {}
117 
118  bool runOnMachineFunction(MachineFunction &Fn) override;
119 
120  MachineFunctionProperties getRequiredProperties() const override {
123  }
124 
125  StringRef getPassName() const override { return ARM_LOAD_STORE_OPT_NAME; }
126 
127  private:
128  /// A set of load/store MachineInstrs with same base register sorted by
129  /// offset.
130  struct MemOpQueueEntry {
131  MachineInstr *MI;
132  int Offset; ///< Load/Store offset.
133  unsigned Position; ///< Position as counted from end of basic block.
134 
135  MemOpQueueEntry(MachineInstr &MI, int Offset, unsigned Position)
136  : MI(&MI), Offset(Offset), Position(Position) {}
137  };
138  using MemOpQueue = SmallVector<MemOpQueueEntry, 8>;
139 
140  /// A set of MachineInstrs that fulfill (nearly all) conditions to get
141  /// merged into a LDM/STM.
142  struct MergeCandidate {
143  /// List of instructions ordered by load/store offset.
145 
146  /// Index in Instrs of the instruction being latest in the schedule.
147  unsigned LatestMIIdx;
148 
149  /// Index in Instrs of the instruction being earliest in the schedule.
150  unsigned EarliestMIIdx;
151 
152  /// Index into the basic block where the merged instruction will be
153  /// inserted. (See MemOpQueueEntry.Position)
154  unsigned InsertPos;
155 
156  /// Whether the instructions can be merged into a ldm/stm instruction.
157  bool CanMergeToLSMulti;
158 
159  /// Whether the instructions can be merged into a ldrd/strd instruction.
160  bool CanMergeToLSDouble;
161  };
164  SmallVector<MachineInstr*,4> MergeBaseCandidates;
165 
166  void moveLiveRegsBefore(const MachineBasicBlock &MBB,
168  unsigned findFreeReg(const TargetRegisterClass &RegClass);
169  void UpdateBaseRegUses(MachineBasicBlock &MBB,
170  MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
171  unsigned Base, unsigned WordOffset,
172  ARMCC::CondCodes Pred, unsigned PredReg);
173  MachineInstr *CreateLoadStoreMulti(
175  int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
176  ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
177  ArrayRef<std::pair<unsigned, bool>> Regs);
178  MachineInstr *CreateLoadStoreDouble(
180  int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
181  ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
182  ArrayRef<std::pair<unsigned, bool>> Regs) const;
183  void FormCandidates(const MemOpQueue &MemOps);
184  MachineInstr *MergeOpsUpdate(const MergeCandidate &Cand);
185  bool FixInvalidRegPairOp(MachineBasicBlock &MBB,
187  bool MergeBaseUpdateLoadStore(MachineInstr *MI);
188  bool MergeBaseUpdateLSMultiple(MachineInstr *MI);
189  bool MergeBaseUpdateLSDouble(MachineInstr &MI) const;
190  bool LoadStoreMultipleOpti(MachineBasicBlock &MBB);
191  bool MergeReturnIntoLDM(MachineBasicBlock &MBB);
192  bool CombineMovBx(MachineBasicBlock &MBB);
193  };
194 
195 } // end anonymous namespace
196 
197 char ARMLoadStoreOpt::ID = 0;
198 
199 INITIALIZE_PASS(ARMLoadStoreOpt, "arm-ldst-opt", ARM_LOAD_STORE_OPT_NAME, false,
200  false)
201 
202 static bool definesCPSR(const MachineInstr &MI) {
203  for (const auto &MO : MI.operands()) {
204  if (!MO.isReg())
205  continue;
206  if (MO.isDef() && MO.getReg() == ARM::CPSR && !MO.isDead())
207  // If the instruction has live CPSR def, then it's not safe to fold it
208  // into load / store.
209  return true;
210  }
211 
212  return false;
213 }
214 
215 static int getMemoryOpOffset(const MachineInstr &MI) {
216  unsigned Opcode = MI.getOpcode();
217  bool isAM3 = Opcode == ARM::LDRD || Opcode == ARM::STRD;
218  unsigned NumOperands = MI.getDesc().getNumOperands();
219  unsigned OffField = MI.getOperand(NumOperands - 3).getImm();
220 
221  if (Opcode == ARM::t2LDRi12 || Opcode == ARM::t2LDRi8 ||
222  Opcode == ARM::t2STRi12 || Opcode == ARM::t2STRi8 ||
223  Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8 ||
224  Opcode == ARM::LDRi12 || Opcode == ARM::STRi12)
225  return OffField;
226 
227  // Thumb1 immediate offsets are scaled by 4
228  if (Opcode == ARM::tLDRi || Opcode == ARM::tSTRi ||
229  Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi)
230  return OffField * 4;
231 
232  int Offset = isAM3 ? ARM_AM::getAM3Offset(OffField)
233  : ARM_AM::getAM5Offset(OffField) * 4;
234  ARM_AM::AddrOpc Op = isAM3 ? ARM_AM::getAM3Op(OffField)
235  : ARM_AM::getAM5Op(OffField);
236 
237  if (Op == ARM_AM::sub)
238  return -Offset;
239 
240  return Offset;
241 }
242 
244  return MI.getOperand(1);
245 }
246 
248  return MI.getOperand(0);
249 }
250 
251 static int getLoadStoreMultipleOpcode(unsigned Opcode, ARM_AM::AMSubMode Mode) {
252  switch (Opcode) {
253  default: llvm_unreachable("Unhandled opcode!");
254  case ARM::LDRi12:
255  ++NumLDMGened;
256  switch (Mode) {
257  default: llvm_unreachable("Unhandled submode!");
258  case ARM_AM::ia: return ARM::LDMIA;
259  case ARM_AM::da: return ARM::LDMDA;
260  case ARM_AM::db: return ARM::LDMDB;
261  case ARM_AM::ib: return ARM::LDMIB;
262  }
263  case ARM::STRi12:
264  ++NumSTMGened;
265  switch (Mode) {
266  default: llvm_unreachable("Unhandled submode!");
267  case ARM_AM::ia: return ARM::STMIA;
268  case ARM_AM::da: return ARM::STMDA;
269  case ARM_AM::db: return ARM::STMDB;
270  case ARM_AM::ib: return ARM::STMIB;
271  }
272  case ARM::tLDRi:
273  case ARM::tLDRspi:
274  // tLDMIA is writeback-only - unless the base register is in the input
275  // reglist.
276  ++NumLDMGened;
277  switch (Mode) {
278  default: llvm_unreachable("Unhandled submode!");
279  case ARM_AM::ia: return ARM::tLDMIA;
280  }
281  case ARM::tSTRi:
282  case ARM::tSTRspi:
283  // There is no non-writeback tSTMIA either.
284  ++NumSTMGened;
285  switch (Mode) {
286  default: llvm_unreachable("Unhandled submode!");
287  case ARM_AM::ia: return ARM::tSTMIA_UPD;
288  }
289  case ARM::t2LDRi8:
290  case ARM::t2LDRi12:
291  ++NumLDMGened;
292  switch (Mode) {
293  default: llvm_unreachable("Unhandled submode!");
294  case ARM_AM::ia: return ARM::t2LDMIA;
295  case ARM_AM::db: return ARM::t2LDMDB;
296  }
297  case ARM::t2STRi8:
298  case ARM::t2STRi12:
299  ++NumSTMGened;
300  switch (Mode) {
301  default: llvm_unreachable("Unhandled submode!");
302  case ARM_AM::ia: return ARM::t2STMIA;
303  case ARM_AM::db: return ARM::t2STMDB;
304  }
305  case ARM::VLDRS:
306  ++NumVLDMGened;
307  switch (Mode) {
308  default: llvm_unreachable("Unhandled submode!");
309  case ARM_AM::ia: return ARM::VLDMSIA;
310  case ARM_AM::db: return 0; // Only VLDMSDB_UPD exists.
311  }
312  case ARM::VSTRS:
313  ++NumVSTMGened;
314  switch (Mode) {
315  default: llvm_unreachable("Unhandled submode!");
316  case ARM_AM::ia: return ARM::VSTMSIA;
317  case ARM_AM::db: return 0; // Only VSTMSDB_UPD exists.
318  }
319  case ARM::VLDRD:
320  ++NumVLDMGened;
321  switch (Mode) {
322  default: llvm_unreachable("Unhandled submode!");
323  case ARM_AM::ia: return ARM::VLDMDIA;
324  case ARM_AM::db: return 0; // Only VLDMDDB_UPD exists.
325  }
326  case ARM::VSTRD:
327  ++NumVSTMGened;
328  switch (Mode) {
329  default: llvm_unreachable("Unhandled submode!");
330  case ARM_AM::ia: return ARM::VSTMDIA;
331  case ARM_AM::db: return 0; // Only VSTMDDB_UPD exists.
332  }
333  }
334 }
335 
337  switch (Opcode) {
338  default: llvm_unreachable("Unhandled opcode!");
339  case ARM::LDMIA_RET:
340  case ARM::LDMIA:
341  case ARM::LDMIA_UPD:
342  case ARM::STMIA:
343  case ARM::STMIA_UPD:
344  case ARM::tLDMIA:
345  case ARM::tLDMIA_UPD:
346  case ARM::tSTMIA_UPD:
347  case ARM::t2LDMIA_RET:
348  case ARM::t2LDMIA:
349  case ARM::t2LDMIA_UPD:
350  case ARM::t2STMIA:
351  case ARM::t2STMIA_UPD:
352  case ARM::VLDMSIA:
353  case ARM::VLDMSIA_UPD:
354  case ARM::VSTMSIA:
355  case ARM::VSTMSIA_UPD:
356  case ARM::VLDMDIA:
357  case ARM::VLDMDIA_UPD:
358  case ARM::VSTMDIA:
359  case ARM::VSTMDIA_UPD:
360  return ARM_AM::ia;
361 
362  case ARM::LDMDA:
363  case ARM::LDMDA_UPD:
364  case ARM::STMDA:
365  case ARM::STMDA_UPD:
366  return ARM_AM::da;
367 
368  case ARM::LDMDB:
369  case ARM::LDMDB_UPD:
370  case ARM::STMDB:
371  case ARM::STMDB_UPD:
372  case ARM::t2LDMDB:
373  case ARM::t2LDMDB_UPD:
374  case ARM::t2STMDB:
375  case ARM::t2STMDB_UPD:
376  case ARM::VLDMSDB_UPD:
377  case ARM::VSTMSDB_UPD:
378  case ARM::VLDMDDB_UPD:
379  case ARM::VSTMDDB_UPD:
380  return ARM_AM::db;
381 
382  case ARM::LDMIB:
383  case ARM::LDMIB_UPD:
384  case ARM::STMIB:
385  case ARM::STMIB_UPD:
386  return ARM_AM::ib;
387  }
388 }
389 
390 static bool isT1i32Load(unsigned Opc) {
391  return Opc == ARM::tLDRi || Opc == ARM::tLDRspi;
392 }
393 
394 static bool isT2i32Load(unsigned Opc) {
395  return Opc == ARM::t2LDRi12 || Opc == ARM::t2LDRi8;
396 }
397 
398 static bool isi32Load(unsigned Opc) {
399  return Opc == ARM::LDRi12 || isT1i32Load(Opc) || isT2i32Load(Opc) ;
400 }
401 
402 static bool isT1i32Store(unsigned Opc) {
403  return Opc == ARM::tSTRi || Opc == ARM::tSTRspi;
404 }
405 
406 static bool isT2i32Store(unsigned Opc) {
407  return Opc == ARM::t2STRi12 || Opc == ARM::t2STRi8;
408 }
409 
410 static bool isi32Store(unsigned Opc) {
411  return Opc == ARM::STRi12 || isT1i32Store(Opc) || isT2i32Store(Opc);
412 }
413 
414 static bool isLoadSingle(unsigned Opc) {
415  return isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD;
416 }
417 
418 static unsigned getImmScale(unsigned Opc) {
419  switch (Opc) {
420  default: llvm_unreachable("Unhandled opcode!");
421  case ARM::tLDRi:
422  case ARM::tSTRi:
423  case ARM::tLDRspi:
424  case ARM::tSTRspi:
425  return 1;
426  case ARM::tLDRHi:
427  case ARM::tSTRHi:
428  return 2;
429  case ARM::tLDRBi:
430  case ARM::tSTRBi:
431  return 4;
432  }
433 }
434 
435 static unsigned getLSMultipleTransferSize(const MachineInstr *MI) {
436  switch (MI->getOpcode()) {
437  default: return 0;
438  case ARM::LDRi12:
439  case ARM::STRi12:
440  case ARM::tLDRi:
441  case ARM::tSTRi:
442  case ARM::tLDRspi:
443  case ARM::tSTRspi:
444  case ARM::t2LDRi8:
445  case ARM::t2LDRi12:
446  case ARM::t2STRi8:
447  case ARM::t2STRi12:
448  case ARM::VLDRS:
449  case ARM::VSTRS:
450  return 4;
451  case ARM::VLDRD:
452  case ARM::VSTRD:
453  return 8;
454  case ARM::LDMIA:
455  case ARM::LDMDA:
456  case ARM::LDMDB:
457  case ARM::LDMIB:
458  case ARM::STMIA:
459  case ARM::STMDA:
460  case ARM::STMDB:
461  case ARM::STMIB:
462  case ARM::tLDMIA:
463  case ARM::tLDMIA_UPD:
464  case ARM::tSTMIA_UPD:
465  case ARM::t2LDMIA:
466  case ARM::t2LDMDB:
467  case ARM::t2STMIA:
468  case ARM::t2STMDB:
469  case ARM::VLDMSIA:
470  case ARM::VSTMSIA:
471  return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 4;
472  case ARM::VLDMDIA:
473  case ARM::VSTMDIA:
474  return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 8;
475  }
476 }
477 
478 /// Update future uses of the base register with the offset introduced
479 /// due to writeback. This function only works on Thumb1.
480 void ARMLoadStoreOpt::UpdateBaseRegUses(MachineBasicBlock &MBB,
482  const DebugLoc &DL, unsigned Base,
483  unsigned WordOffset,
484  ARMCC::CondCodes Pred,
485  unsigned PredReg) {
486  assert(isThumb1 && "Can only update base register uses for Thumb1!");
487  // Start updating any instructions with immediate offsets. Insert a SUB before
488  // the first non-updateable instruction (if any).
489  for (; MBBI != MBB.end(); ++MBBI) {
490  bool InsertSub = false;
491  unsigned Opc = MBBI->getOpcode();
492 
493  if (MBBI->readsRegister(Base)) {
494  int Offset;
495  bool IsLoad =
496  Opc == ARM::tLDRi || Opc == ARM::tLDRHi || Opc == ARM::tLDRBi;
497  bool IsStore =
498  Opc == ARM::tSTRi || Opc == ARM::tSTRHi || Opc == ARM::tSTRBi;
499 
500  if (IsLoad || IsStore) {
501  // Loads and stores with immediate offsets can be updated, but only if
502  // the new offset isn't negative.
503  // The MachineOperand containing the offset immediate is the last one
504  // before predicates.
505  MachineOperand &MO =
506  MBBI->getOperand(MBBI->getDesc().getNumOperands() - 3);
507  // The offsets are scaled by 1, 2 or 4 depending on the Opcode.
508  Offset = MO.getImm() - WordOffset * getImmScale(Opc);
509 
510  // If storing the base register, it needs to be reset first.
511  unsigned InstrSrcReg = getLoadStoreRegOp(*MBBI).getReg();
512 
513  if (Offset >= 0 && !(IsStore && InstrSrcReg == Base))
514  MO.setImm(Offset);
515  else
516  InsertSub = true;
517  } else if ((Opc == ARM::tSUBi8 || Opc == ARM::tADDi8) &&
518  !definesCPSR(*MBBI)) {
519  // SUBS/ADDS using this register, with a dead def of the CPSR.
520  // Merge it with the update; if the merged offset is too large,
521  // insert a new sub instead.
522  MachineOperand &MO =
523  MBBI->getOperand(MBBI->getDesc().getNumOperands() - 3);
524  Offset = (Opc == ARM::tSUBi8) ?
525  MO.getImm() + WordOffset * 4 :
526  MO.getImm() - WordOffset * 4 ;
527  if (Offset >= 0 && TL->isLegalAddImmediate(Offset)) {
528  // FIXME: Swap ADDS<->SUBS if Offset < 0, erase instruction if
529  // Offset == 0.
530  MO.setImm(Offset);
531  // The base register has now been reset, so exit early.
532  return;
533  } else {
534  InsertSub = true;
535  }
536  } else {
537  // Can't update the instruction.
538  InsertSub = true;
539  }
540  } else if (definesCPSR(*MBBI) || MBBI->isCall() || MBBI->isBranch()) {
541  // Since SUBS sets the condition flags, we can't place the base reset
542  // after an instruction that has a live CPSR def.
543  // The base register might also contain an argument for a function call.
544  InsertSub = true;
545  }
546 
547  if (InsertSub) {
548  // An instruction above couldn't be updated, so insert a sub.
549  BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBi8), Base)
550  .add(t1CondCodeOp(true))
551  .addReg(Base)
552  .addImm(WordOffset * 4)
553  .addImm(Pred)
554  .addReg(PredReg);
555  return;
556  }
557 
558  if (MBBI->killsRegister(Base) || MBBI->definesRegister(Base))
559  // Register got killed. Stop updating.
560  return;
561  }
562 
563  // End of block was reached.
564  if (MBB.succ_size() > 0) {
565  // FIXME: Because of a bug, live registers are sometimes missing from
566  // the successor blocks' live-in sets. This means we can't trust that
567  // information and *always* have to reset at the end of a block.
568  // See PR21029.
569  if (MBBI != MBB.end()) --MBBI;
570  BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBi8), Base)
571  .add(t1CondCodeOp(true))
572  .addReg(Base)
573  .addImm(WordOffset * 4)
574  .addImm(Pred)
575  .addReg(PredReg);
576  }
577 }
578 
579 /// Return the first register of class \p RegClass that is not in \p Regs.
580 unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) {
581  if (!RegClassInfoValid) {
582  RegClassInfo.runOnMachineFunction(*MF);
583  RegClassInfoValid = true;
584  }
585 
586  for (unsigned Reg : RegClassInfo.getOrder(&RegClass))
587  if (!LiveRegs.contains(Reg))
588  return Reg;
589  return 0;
590 }
591 
592 /// Compute live registers just before instruction \p Before (in normal schedule
593 /// direction). Computes backwards so multiple queries in the same block must
594 /// come in reverse order.
595 void ARMLoadStoreOpt::moveLiveRegsBefore(const MachineBasicBlock &MBB,
597  // Initialize if we never queried in this block.
598  if (!LiveRegsValid) {
599  LiveRegs.init(*TRI);
600  LiveRegs.addLiveOuts(MBB);
601  LiveRegPos = MBB.end();
602  LiveRegsValid = true;
603  }
604  // Move backward just before the "Before" position.
605  while (LiveRegPos != Before) {
606  --LiveRegPos;
607  LiveRegs.stepBackward(*LiveRegPos);
608  }
609 }
610 
611 static bool ContainsReg(const ArrayRef<std::pair<unsigned, bool>> &Regs,
612  unsigned Reg) {
613  for (const std::pair<unsigned, bool> &R : Regs)
614  if (R.first == Reg)
615  return true;
616  return false;
617 }
618 
619 /// Create and insert a LDM or STM with Base as base register and registers in
620 /// Regs as the register operands that would be loaded / stored. It returns
621 /// true if the transformation is done.
622 MachineInstr *ARMLoadStoreOpt::CreateLoadStoreMulti(
624  int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
625  ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
626  ArrayRef<std::pair<unsigned, bool>> Regs) {
627  unsigned NumRegs = Regs.size();
628  assert(NumRegs > 1);
629 
630  // For Thumb1 targets, it might be necessary to clobber the CPSR to merge.
631  // Compute liveness information for that register to make the decision.
632  bool SafeToClobberCPSR = !isThumb1 ||
633  (MBB.computeRegisterLiveness(TRI, ARM::CPSR, InsertBefore, 20) ==
635 
636  bool Writeback = isThumb1; // Thumb1 LDM/STM have base reg writeback.
637 
638  // Exception: If the base register is in the input reglist, Thumb1 LDM is
639  // non-writeback.
640  // It's also not possible to merge an STR of the base register in Thumb1.
641  if (isThumb1 && ContainsReg(Regs, Base)) {
642  assert(Base != ARM::SP && "Thumb1 does not allow SP in register list");
643  if (Opcode == ARM::tLDRi)
644  Writeback = false;
645  else if (Opcode == ARM::tSTRi)
646  return nullptr;
647  }
648 
650  // VFP and Thumb2 do not support IB or DA modes. Thumb1 only supports IA.
651  bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
652  bool haveIBAndDA = isNotVFP && !isThumb2 && !isThumb1;
653 
654  if (Offset == 4 && haveIBAndDA) {
655  Mode = ARM_AM::ib;
656  } else if (Offset == -4 * (int)NumRegs + 4 && haveIBAndDA) {
657  Mode = ARM_AM::da;
658  } else if (Offset == -4 * (int)NumRegs && isNotVFP && !isThumb1) {
659  // VLDM/VSTM do not support DB mode without also updating the base reg.
660  Mode = ARM_AM::db;
661  } else if (Offset != 0 || Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi) {
662  // Check if this is a supported opcode before inserting instructions to
663  // calculate a new base register.
664  if (!getLoadStoreMultipleOpcode(Opcode, Mode)) return nullptr;
665 
666  // If starting offset isn't zero, insert a MI to materialize a new base.
667  // But only do so if it is cost effective, i.e. merging more than two
668  // loads / stores.
669  if (NumRegs <= 2)
670  return nullptr;
671 
672  // On Thumb1, it's not worth materializing a new base register without
673  // clobbering the CPSR (i.e. not using ADDS/SUBS).
674  if (!SafeToClobberCPSR)
675  return nullptr;
676 
677  unsigned NewBase;
678  if (isi32Load(Opcode)) {
679  // If it is a load, then just use one of the destination registers
680  // as the new base. Will no longer be writeback in Thumb1.
681  NewBase = Regs[NumRegs-1].first;
682  Writeback = false;
683  } else {
684  // Find a free register that we can use as scratch register.
685  moveLiveRegsBefore(MBB, InsertBefore);
686  // The merged instruction does not exist yet but will use several Regs if
687  // it is a Store.
688  if (!isLoadSingle(Opcode))
689  for (const std::pair<unsigned, bool> &R : Regs)
690  LiveRegs.addReg(R.first);
691 
692  NewBase = findFreeReg(isThumb1 ? ARM::tGPRRegClass : ARM::GPRRegClass);
693  if (NewBase == 0)
694  return nullptr;
695  }
696 
697  int BaseOpc =
698  isThumb2 ? ARM::t2ADDri :
699  (isThumb1 && Base == ARM::SP) ? ARM::tADDrSPi :
700  (isThumb1 && Offset < 8) ? ARM::tADDi3 :
701  isThumb1 ? ARM::tADDi8 : ARM::ADDri;
702 
703  if (Offset < 0) {
704  Offset = - Offset;
705  BaseOpc =
706  isThumb2 ? ARM::t2SUBri :
707  (isThumb1 && Offset < 8 && Base != ARM::SP) ? ARM::tSUBi3 :
708  isThumb1 ? ARM::tSUBi8 : ARM::SUBri;
709  }
710 
711  if (!TL->isLegalAddImmediate(Offset))
712  // FIXME: Try add with register operand?
713  return nullptr; // Probably not worth it then.
714 
715  // We can only append a kill flag to the add/sub input if the value is not
716  // used in the register list of the stm as well.
717  bool KillOldBase = BaseKill &&
718  (!isi32Store(Opcode) || !ContainsReg(Regs, Base));
719 
720  if (isThumb1) {
721  // Thumb1: depending on immediate size, use either
722  // ADDS NewBase, Base, #imm3
723  // or
724  // MOV NewBase, Base
725  // ADDS NewBase, #imm8.
726  if (Base != NewBase &&
727  (BaseOpc == ARM::tADDi8 || BaseOpc == ARM::tSUBi8)) {
728  // Need to insert a MOV to the new base first.
729  if (isARMLowRegister(NewBase) && isARMLowRegister(Base) &&
730  !STI->hasV6Ops()) {
731  // thumbv4t doesn't have lo->lo copies, and we can't predicate tMOVSr
732  if (Pred != ARMCC::AL)
733  return nullptr;
734  BuildMI(MBB, InsertBefore, DL, TII->get(ARM::tMOVSr), NewBase)
735  .addReg(Base, getKillRegState(KillOldBase));
736  } else
737  BuildMI(MBB, InsertBefore, DL, TII->get(ARM::tMOVr), NewBase)
738  .addReg(Base, getKillRegState(KillOldBase))
739  .add(predOps(Pred, PredReg));
740 
741  // The following ADDS/SUBS becomes an update.
742  Base = NewBase;
743  KillOldBase = true;
744  }
745  if (BaseOpc == ARM::tADDrSPi) {
746  assert(Offset % 4 == 0 && "tADDrSPi offset is scaled by 4");
747  BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
748  .addReg(Base, getKillRegState(KillOldBase))
749  .addImm(Offset / 4)
750  .add(predOps(Pred, PredReg));
751  } else
752  BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
753  .add(t1CondCodeOp(true))
754  .addReg(Base, getKillRegState(KillOldBase))
755  .addImm(Offset)
756  .add(predOps(Pred, PredReg));
757  } else {
758  BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
759  .addReg(Base, getKillRegState(KillOldBase))
760  .addImm(Offset)
761  .add(predOps(Pred, PredReg))
762  .add(condCodeOp());
763  }
764  Base = NewBase;
765  BaseKill = true; // New base is always killed straight away.
766  }
767 
768  bool isDef = isLoadSingle(Opcode);
769 
770  // Get LS multiple opcode. Note that for Thumb1 this might be an opcode with
771  // base register writeback.
772  Opcode = getLoadStoreMultipleOpcode(Opcode, Mode);
773  if (!Opcode)
774  return nullptr;
775 
776  // Check if a Thumb1 LDM/STM merge is safe. This is the case if:
777  // - There is no writeback (LDM of base register),
778  // - the base register is killed by the merged instruction,
779  // - or it's safe to overwrite the condition flags, i.e. to insert a SUBS
780  // to reset the base register.
781  // Otherwise, don't merge.
782  // It's safe to return here since the code to materialize a new base register
783  // above is also conditional on SafeToClobberCPSR.
784  if (isThumb1 && !SafeToClobberCPSR && Writeback && !BaseKill)
785  return nullptr;
786 
788 
789  if (Writeback) {
790  assert(isThumb1 && "expected Writeback only inThumb1");
791  if (Opcode == ARM::tLDMIA) {
792  assert(!(ContainsReg(Regs, Base)) && "Thumb1 can't LDM ! with Base in Regs");
793  // Update tLDMIA with writeback if necessary.
794  Opcode = ARM::tLDMIA_UPD;
795  }
796 
797  MIB = BuildMI(MBB, InsertBefore, DL, TII->get(Opcode));
798 
799  // Thumb1: we might need to set base writeback when building the MI.
800  MIB.addReg(Base, getDefRegState(true))
801  .addReg(Base, getKillRegState(BaseKill));
802 
803  // The base isn't dead after a merged instruction with writeback.
804  // Insert a sub instruction after the newly formed instruction to reset.
805  if (!BaseKill)
806  UpdateBaseRegUses(MBB, InsertBefore, DL, Base, NumRegs, Pred, PredReg);
807  } else {
808  // No writeback, simply build the MachineInstr.
809  MIB = BuildMI(MBB, InsertBefore, DL, TII->get(Opcode));
810  MIB.addReg(Base, getKillRegState(BaseKill));
811  }
812 
813  MIB.addImm(Pred).addReg(PredReg);
814 
815  for (const std::pair<unsigned, bool> &R : Regs)
816  MIB.addReg(R.first, getDefRegState(isDef) | getKillRegState(R.second));
817 
818  return MIB.getInstr();
819 }
820 
821 MachineInstr *ARMLoadStoreOpt::CreateLoadStoreDouble(
823  int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
824  ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
825  ArrayRef<std::pair<unsigned, bool>> Regs) const {
826  bool IsLoad = isi32Load(Opcode);
827  assert((IsLoad || isi32Store(Opcode)) && "Must have integer load or store");
828  unsigned LoadStoreOpcode = IsLoad ? ARM::t2LDRDi8 : ARM::t2STRDi8;
829 
830  assert(Regs.size() == 2);
831  MachineInstrBuilder MIB = BuildMI(MBB, InsertBefore, DL,
832  TII->get(LoadStoreOpcode));
833  if (IsLoad) {
834  MIB.addReg(Regs[0].first, RegState::Define)
835  .addReg(Regs[1].first, RegState::Define);
836  } else {
837  MIB.addReg(Regs[0].first, getKillRegState(Regs[0].second))
838  .addReg(Regs[1].first, getKillRegState(Regs[1].second));
839  }
840  MIB.addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
841  return MIB.getInstr();
842 }
843 
844 /// Call MergeOps and update MemOps and merges accordingly on success.
845 MachineInstr *ARMLoadStoreOpt::MergeOpsUpdate(const MergeCandidate &Cand) {
846  const MachineInstr *First = Cand.Instrs.front();
847  unsigned Opcode = First->getOpcode();
848  bool IsLoad = isLoadSingle(Opcode);
850  SmallVector<unsigned, 4> ImpDefs;
851  DenseSet<unsigned> KilledRegs;
852  DenseSet<unsigned> UsedRegs;
853  // Determine list of registers and list of implicit super-register defs.
854  for (const MachineInstr *MI : Cand.Instrs) {
855  const MachineOperand &MO = getLoadStoreRegOp(*MI);
856  unsigned Reg = MO.getReg();
857  bool IsKill = MO.isKill();
858  if (IsKill)
859  KilledRegs.insert(Reg);
860  Regs.push_back(std::make_pair(Reg, IsKill));
861  UsedRegs.insert(Reg);
862 
863  if (IsLoad) {
864  // Collect any implicit defs of super-registers, after merging we can't
865  // be sure anymore that we properly preserved these live ranges and must
866  // removed these implicit operands.
867  for (const MachineOperand &MO : MI->implicit_operands()) {
868  if (!MO.isReg() || !MO.isDef() || MO.isDead())
869  continue;
870  assert(MO.isImplicit());
871  unsigned DefReg = MO.getReg();
872 
873  if (is_contained(ImpDefs, DefReg))
874  continue;
875  // We can ignore cases where the super-reg is read and written.
876  if (MI->readsRegister(DefReg))
877  continue;
878  ImpDefs.push_back(DefReg);
879  }
880  }
881  }
882 
883  // Attempt the merge.
884  using iterator = MachineBasicBlock::iterator;
885 
886  MachineInstr *LatestMI = Cand.Instrs[Cand.LatestMIIdx];
887  iterator InsertBefore = std::next(iterator(LatestMI));
888  MachineBasicBlock &MBB = *LatestMI->getParent();
889  unsigned Offset = getMemoryOpOffset(*First);
890  unsigned Base = getLoadStoreBaseOp(*First).getReg();
891  bool BaseKill = LatestMI->killsRegister(Base);
892  unsigned PredReg = 0;
893  ARMCC::CondCodes Pred = getInstrPredicate(*First, PredReg);
894  DebugLoc DL = First->getDebugLoc();
895  MachineInstr *Merged = nullptr;
896  if (Cand.CanMergeToLSDouble)
897  Merged = CreateLoadStoreDouble(MBB, InsertBefore, Offset, Base, BaseKill,
898  Opcode, Pred, PredReg, DL, Regs);
899  if (!Merged && Cand.CanMergeToLSMulti)
900  Merged = CreateLoadStoreMulti(MBB, InsertBefore, Offset, Base, BaseKill,
901  Opcode, Pred, PredReg, DL, Regs);
902  if (!Merged)
903  return nullptr;
904 
905  // Determine earliest instruction that will get removed. We then keep an
906  // iterator just above it so the following erases don't invalidated it.
907  iterator EarliestI(Cand.Instrs[Cand.EarliestMIIdx]);
908  bool EarliestAtBegin = false;
909  if (EarliestI == MBB.begin()) {
910  EarliestAtBegin = true;
911  } else {
912  EarliestI = std::prev(EarliestI);
913  }
914 
915  // Remove instructions which have been merged.
916  for (MachineInstr *MI : Cand.Instrs)
917  MBB.erase(MI);
918 
919  // Determine range between the earliest removed instruction and the new one.
920  if (EarliestAtBegin)
921  EarliestI = MBB.begin();
922  else
923  EarliestI = std::next(EarliestI);
924  auto FixupRange = make_range(EarliestI, iterator(Merged));
925 
926  if (isLoadSingle(Opcode)) {
927  // If the previous loads defined a super-reg, then we have to mark earlier
928  // operands undef; Replicate the super-reg def on the merged instruction.
929  for (MachineInstr &MI : FixupRange) {
930  for (unsigned &ImpDefReg : ImpDefs) {
931  for (MachineOperand &MO : MI.implicit_operands()) {
932  if (!MO.isReg() || MO.getReg() != ImpDefReg)
933  continue;
934  if (MO.readsReg())
935  MO.setIsUndef();
936  else if (MO.isDef())
937  ImpDefReg = 0;
938  }
939  }
940  }
941 
942  MachineInstrBuilder MIB(*Merged->getParent()->getParent(), Merged);
943  for (unsigned ImpDef : ImpDefs)
944  MIB.addReg(ImpDef, RegState::ImplicitDefine);
945  } else {
946  // Remove kill flags: We are possibly storing the values later now.
947  assert(isi32Store(Opcode) || Opcode == ARM::VSTRS || Opcode == ARM::VSTRD);
948  for (MachineInstr &MI : FixupRange) {
949  for (MachineOperand &MO : MI.uses()) {
950  if (!MO.isReg() || !MO.isKill())
951  continue;
952  if (UsedRegs.count(MO.getReg()))
953  MO.setIsKill(false);
954  }
955  }
956  assert(ImpDefs.empty());
957  }
958 
959  return Merged;
960 }
961 
962 static bool isValidLSDoubleOffset(int Offset) {
963  unsigned Value = abs(Offset);
964  // t2LDRDi8/t2STRDi8 supports an 8 bit immediate which is internally
965  // multiplied by 4.
966  return (Value % 4) == 0 && Value < 1024;
967 }
968 
969 /// Return true for loads/stores that can be combined to a double/multi
970 /// operation without increasing the requirements for alignment.
972  const MachineInstr &MI) {
973  // vldr/vstr trap on misaligned pointers anyway, forming vldm makes no
974  // difference.
975  unsigned Opcode = MI.getOpcode();
976  if (!isi32Load(Opcode) && !isi32Store(Opcode))
977  return true;
978 
979  // Stack pointer alignment is out of the programmers control so we can trust
980  // SP-relative loads/stores.
981  if (getLoadStoreBaseOp(MI).getReg() == ARM::SP &&
983  return true;
984  return false;
985 }
986 
987 /// Find candidates for load/store multiple merge in list of MemOpQueueEntries.
988 void ARMLoadStoreOpt::FormCandidates(const MemOpQueue &MemOps) {
989  const MachineInstr *FirstMI = MemOps[0].MI;
990  unsigned Opcode = FirstMI->getOpcode();
991  bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
992  unsigned Size = getLSMultipleTransferSize(FirstMI);
993 
994  unsigned SIndex = 0;
995  unsigned EIndex = MemOps.size();
996  do {
997  // Look at the first instruction.
998  const MachineInstr *MI = MemOps[SIndex].MI;
999  int Offset = MemOps[SIndex].Offset;
1000  const MachineOperand &PMO = getLoadStoreRegOp(*MI);
1001  unsigned PReg = PMO.getReg();
1002  unsigned PRegNum = PMO.isUndef() ? std::numeric_limits<unsigned>::max()
1003  : TRI->getEncodingValue(PReg);
1004  unsigned Latest = SIndex;
1005  unsigned Earliest = SIndex;
1006  unsigned Count = 1;
1007  bool CanMergeToLSDouble =
1008  STI->isThumb2() && isNotVFP && isValidLSDoubleOffset(Offset);
1009  // ARM errata 602117: LDRD with base in list may result in incorrect base
1010  // register when interrupted or faulted.
1011  if (STI->isCortexM3() && isi32Load(Opcode) &&
1012  PReg == getLoadStoreBaseOp(*MI).getReg())
1013  CanMergeToLSDouble = false;
1014 
1015  bool CanMergeToLSMulti = true;
1016  // On swift vldm/vstm starting with an odd register number as that needs
1017  // more uops than single vldrs.
1018  if (STI->hasSlowOddRegister() && !isNotVFP && (PRegNum % 2) == 1)
1019  CanMergeToLSMulti = false;
1020 
1021  // LDRD/STRD do not allow SP/PC. LDM/STM do not support it or have it
1022  // deprecated; LDM to PC is fine but cannot happen here.
1023  if (PReg == ARM::SP || PReg == ARM::PC)
1024  CanMergeToLSMulti = CanMergeToLSDouble = false;
1025 
1026  // Should we be conservative?
1028  CanMergeToLSMulti = CanMergeToLSDouble = false;
1029 
1030  // vldm / vstm limit are 32 for S variants, 16 for D variants.
1031  unsigned Limit;
1032  switch (Opcode) {
1033  default:
1034  Limit = UINT_MAX;
1035  break;
1036  case ARM::VLDRD:
1037  case ARM::VSTRD:
1038  Limit = 16;
1039  break;
1040  }
1041 
1042  // Merge following instructions where possible.
1043  for (unsigned I = SIndex+1; I < EIndex; ++I, ++Count) {
1044  int NewOffset = MemOps[I].Offset;
1045  if (NewOffset != Offset + (int)Size)
1046  break;
1047  const MachineOperand &MO = getLoadStoreRegOp(*MemOps[I].MI);
1048  unsigned Reg = MO.getReg();
1049  if (Reg == ARM::SP || Reg == ARM::PC)
1050  break;
1051  if (Count == Limit)
1052  break;
1053 
1054  // See if the current load/store may be part of a multi load/store.
1055  unsigned RegNum = MO.isUndef() ? std::numeric_limits<unsigned>::max()
1056  : TRI->getEncodingValue(Reg);
1057  bool PartOfLSMulti = CanMergeToLSMulti;
1058  if (PartOfLSMulti) {
1059  // Register numbers must be in ascending order.
1060  if (RegNum <= PRegNum)
1061  PartOfLSMulti = false;
1062  // For VFP / NEON load/store multiples, the registers must be
1063  // consecutive and within the limit on the number of registers per
1064  // instruction.
1065  else if (!isNotVFP && RegNum != PRegNum+1)
1066  PartOfLSMulti = false;
1067  }
1068  // See if the current load/store may be part of a double load/store.
1069  bool PartOfLSDouble = CanMergeToLSDouble && Count <= 1;
1070 
1071  if (!PartOfLSMulti && !PartOfLSDouble)
1072  break;
1073  CanMergeToLSMulti &= PartOfLSMulti;
1074  CanMergeToLSDouble &= PartOfLSDouble;
1075  // Track MemOp with latest and earliest position (Positions are
1076  // counted in reverse).
1077  unsigned Position = MemOps[I].Position;
1078  if (Position < MemOps[Latest].Position)
1079  Latest = I;
1080  else if (Position > MemOps[Earliest].Position)
1081  Earliest = I;
1082  // Prepare for next MemOp.
1083  Offset += Size;
1084  PRegNum = RegNum;
1085  }
1086 
1087  // Form a candidate from the Ops collected so far.
1088  MergeCandidate *Candidate = new(Allocator.Allocate()) MergeCandidate;
1089  for (unsigned C = SIndex, CE = SIndex + Count; C < CE; ++C)
1090  Candidate->Instrs.push_back(MemOps[C].MI);
1091  Candidate->LatestMIIdx = Latest - SIndex;
1092  Candidate->EarliestMIIdx = Earliest - SIndex;
1093  Candidate->InsertPos = MemOps[Latest].Position;
1094  if (Count == 1)
1095  CanMergeToLSMulti = CanMergeToLSDouble = false;
1096  Candidate->CanMergeToLSMulti = CanMergeToLSMulti;
1097  Candidate->CanMergeToLSDouble = CanMergeToLSDouble;
1098  Candidates.push_back(Candidate);
1099  // Continue after the chain.
1100  SIndex += Count;
1101  } while (SIndex < EIndex);
1102 }
1103 
1104 static unsigned getUpdatingLSMultipleOpcode(unsigned Opc,
1106  switch (Opc) {
1107  default: llvm_unreachable("Unhandled opcode!");
1108  case ARM::LDMIA:
1109  case ARM::LDMDA:
1110  case ARM::LDMDB:
1111  case ARM::LDMIB:
1112  switch (Mode) {
1113  default: llvm_unreachable("Unhandled submode!");
1114  case ARM_AM::ia: return ARM::LDMIA_UPD;
1115  case ARM_AM::ib: return ARM::LDMIB_UPD;
1116  case ARM_AM::da: return ARM::LDMDA_UPD;
1117  case ARM_AM::db: return ARM::LDMDB_UPD;
1118  }
1119  case ARM::STMIA:
1120  case ARM::STMDA:
1121  case ARM::STMDB:
1122  case ARM::STMIB:
1123  switch (Mode) {
1124  default: llvm_unreachable("Unhandled submode!");
1125  case ARM_AM::ia: return ARM::STMIA_UPD;
1126  case ARM_AM::ib: return ARM::STMIB_UPD;
1127  case ARM_AM::da: return ARM::STMDA_UPD;
1128  case ARM_AM::db: return ARM::STMDB_UPD;
1129  }
1130  case ARM::t2LDMIA:
1131  case ARM::t2LDMDB:
1132  switch (Mode) {
1133  default: llvm_unreachable("Unhandled submode!");
1134  case ARM_AM::ia: return ARM::t2LDMIA_UPD;
1135  case ARM_AM::db: return ARM::t2LDMDB_UPD;
1136  }
1137  case ARM::t2STMIA:
1138  case ARM::t2STMDB:
1139  switch (Mode) {
1140  default: llvm_unreachable("Unhandled submode!");
1141  case ARM_AM::ia: return ARM::t2STMIA_UPD;
1142  case ARM_AM::db: return ARM::t2STMDB_UPD;
1143  }
1144  case ARM::VLDMSIA:
1145  switch (Mode) {
1146  default: llvm_unreachable("Unhandled submode!");
1147  case ARM_AM::ia: return ARM::VLDMSIA_UPD;
1148  case ARM_AM::db: return ARM::VLDMSDB_UPD;
1149  }
1150  case ARM::VLDMDIA:
1151  switch (Mode) {
1152  default: llvm_unreachable("Unhandled submode!");
1153  case ARM_AM::ia: return ARM::VLDMDIA_UPD;
1154  case ARM_AM::db: return ARM::VLDMDDB_UPD;
1155  }
1156  case ARM::VSTMSIA:
1157  switch (Mode) {
1158  default: llvm_unreachable("Unhandled submode!");
1159  case ARM_AM::ia: return ARM::VSTMSIA_UPD;
1160  case ARM_AM::db: return ARM::VSTMSDB_UPD;
1161  }
1162  case ARM::VSTMDIA:
1163  switch (Mode) {
1164  default: llvm_unreachable("Unhandled submode!");
1165  case ARM_AM::ia: return ARM::VSTMDIA_UPD;
1166  case ARM_AM::db: return ARM::VSTMDDB_UPD;
1167  }
1168  }
1169 }
1170 
1171 /// Check if the given instruction increments or decrements a register and
1172 /// return the amount it is incremented/decremented. Returns 0 if the CPSR flags
1173 /// generated by the instruction are possibly read as well.
1174 static int isIncrementOrDecrement(const MachineInstr &MI, unsigned Reg,
1175  ARMCC::CondCodes Pred, unsigned PredReg) {
1176  bool CheckCPSRDef;
1177  int Scale;
1178  switch (MI.getOpcode()) {
1179  case ARM::tADDi8: Scale = 4; CheckCPSRDef = true; break;
1180  case ARM::tSUBi8: Scale = -4; CheckCPSRDef = true; break;
1181  case ARM::t2SUBri:
1182  case ARM::SUBri: Scale = -1; CheckCPSRDef = true; break;
1183  case ARM::t2ADDri:
1184  case ARM::ADDri: Scale = 1; CheckCPSRDef = true; break;
1185  case ARM::tADDspi: Scale = 4; CheckCPSRDef = false; break;
1186  case ARM::tSUBspi: Scale = -4; CheckCPSRDef = false; break;
1187  default: return 0;
1188  }
1189 
1190  unsigned MIPredReg;
1191  if (MI.getOperand(0).getReg() != Reg ||
1192  MI.getOperand(1).getReg() != Reg ||
1193  getInstrPredicate(MI, MIPredReg) != Pred ||
1194  MIPredReg != PredReg)
1195  return 0;
1196 
1197  if (CheckCPSRDef && definesCPSR(MI))
1198  return 0;
1199  return MI.getOperand(2).getImm() * Scale;
1200 }
1201 
1202 /// Searches for an increment or decrement of \p Reg before \p MBBI.
1205  ARMCC::CondCodes Pred, unsigned PredReg, int &Offset) {
1206  Offset = 0;
1207  MachineBasicBlock &MBB = *MBBI->getParent();
1208  MachineBasicBlock::iterator BeginMBBI = MBB.begin();
1209  MachineBasicBlock::iterator EndMBBI = MBB.end();
1210  if (MBBI == BeginMBBI)
1211  return EndMBBI;
1212 
1213  // Skip debug values.
1214  MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
1215  while (PrevMBBI->isDebugInstr() && PrevMBBI != BeginMBBI)
1216  --PrevMBBI;
1217 
1218  Offset = isIncrementOrDecrement(*PrevMBBI, Reg, Pred, PredReg);
1219  return Offset == 0 ? EndMBBI : PrevMBBI;
1220 }
1221 
1222 /// Searches for a increment or decrement of \p Reg after \p MBBI.
1225  ARMCC::CondCodes Pred, unsigned PredReg, int &Offset) {
1226  Offset = 0;
1227  MachineBasicBlock &MBB = *MBBI->getParent();
1228  MachineBasicBlock::iterator EndMBBI = MBB.end();
1229  MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
1230  // Skip debug values.
1231  while (NextMBBI != EndMBBI && NextMBBI->isDebugInstr())
1232  ++NextMBBI;
1233  if (NextMBBI == EndMBBI)
1234  return EndMBBI;
1235 
1236  Offset = isIncrementOrDecrement(*NextMBBI, Reg, Pred, PredReg);
1237  return Offset == 0 ? EndMBBI : NextMBBI;
1238 }
1239 
1240 /// Fold proceeding/trailing inc/dec of base register into the
1241 /// LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
1242 ///
1243 /// stmia rn, <ra, rb, rc>
1244 /// rn := rn + 4 * 3;
1245 /// =>
1246 /// stmia rn!, <ra, rb, rc>
1247 ///
1248 /// rn := rn - 4 * 3;
1249 /// ldmia rn, <ra, rb, rc>
1250 /// =>
1251 /// ldmdb rn!, <ra, rb, rc>
1252 bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineInstr *MI) {
1253  // Thumb1 is already using updating loads/stores.
1254  if (isThumb1) return false;
1255 
1256  const MachineOperand &BaseOP = MI->getOperand(0);
1257  unsigned Base = BaseOP.getReg();
1258  bool BaseKill = BaseOP.isKill();
1259  unsigned PredReg = 0;
1260  ARMCC::CondCodes Pred = getInstrPredicate(*MI, PredReg);
1261  unsigned Opcode = MI->getOpcode();
1262  DebugLoc DL = MI->getDebugLoc();
1263 
1264  // Can't use an updating ld/st if the base register is also a dest
1265  // register. e.g. ldmdb r0!, {r0, r1, r2}. The behavior is undefined.
1266  for (unsigned i = 2, e = MI->getNumOperands(); i != e; ++i)
1267  if (MI->getOperand(i).getReg() == Base)
1268  return false;
1269 
1270  int Bytes = getLSMultipleTransferSize(MI);
1271  MachineBasicBlock &MBB = *MI->getParent();
1272  MachineBasicBlock::iterator MBBI(MI);
1273  int Offset;
1274  MachineBasicBlock::iterator MergeInstr
1275  = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
1277  if (Mode == ARM_AM::ia && Offset == -Bytes) {
1278  Mode = ARM_AM::db;
1279  } else if (Mode == ARM_AM::ib && Offset == -Bytes) {
1280  Mode = ARM_AM::da;
1281  } else {
1282  MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
1283  if (((Mode != ARM_AM::ia && Mode != ARM_AM::ib) || Offset != Bytes) &&
1284  ((Mode != ARM_AM::da && Mode != ARM_AM::db) || Offset != -Bytes)) {
1285 
1286  // We couldn't find an inc/dec to merge. But if the base is dead, we
1287  // can still change to a writeback form as that will save us 2 bytes
1288  // of code size. It can create WAW hazards though, so only do it if
1289  // we're minimizing code size.
1290  if (!MBB.getParent()->getFunction().optForMinSize() || !BaseKill)
1291  return false;
1292 
1293  bool HighRegsUsed = false;
1294  for (unsigned i = 2, e = MI->getNumOperands(); i != e; ++i)
1295  if (MI->getOperand(i).getReg() >= ARM::R8) {
1296  HighRegsUsed = true;
1297  break;
1298  }
1299 
1300  if (!HighRegsUsed)
1301  MergeInstr = MBB.end();
1302  else
1303  return false;
1304  }
1305  }
1306  if (MergeInstr != MBB.end())
1307  MBB.erase(MergeInstr);
1308 
1309  unsigned NewOpc = getUpdatingLSMultipleOpcode(Opcode, Mode);
1310  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
1311  .addReg(Base, getDefRegState(true)) // WB base register
1312  .addReg(Base, getKillRegState(BaseKill))
1313  .addImm(Pred).addReg(PredReg);
1314 
1315  // Transfer the rest of operands.
1316  for (unsigned OpNum = 3, e = MI->getNumOperands(); OpNum != e; ++OpNum)
1317  MIB.add(MI->getOperand(OpNum));
1318 
1319  // Transfer memoperands.
1320  MIB.setMemRefs(MI->memoperands());
1321 
1322  MBB.erase(MBBI);
1323  return true;
1324 }
1325 
1326 static unsigned getPreIndexedLoadStoreOpcode(unsigned Opc,
1328  switch (Opc) {
1329  case ARM::LDRi12:
1330  return ARM::LDR_PRE_IMM;
1331  case ARM::STRi12:
1332  return ARM::STR_PRE_IMM;
1333  case ARM::VLDRS:
1334  return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
1335  case ARM::VLDRD:
1336  return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
1337  case ARM::VSTRS:
1338  return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
1339  case ARM::VSTRD:
1340  return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
1341  case ARM::t2LDRi8:
1342  case ARM::t2LDRi12:
1343  return ARM::t2LDR_PRE;
1344  case ARM::t2STRi8:
1345  case ARM::t2STRi12:
1346  return ARM::t2STR_PRE;
1347  default: llvm_unreachable("Unhandled opcode!");
1348  }
1349 }
1350 
1351 static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc,
1353  switch (Opc) {
1354  case ARM::LDRi12:
1355  return ARM::LDR_POST_IMM;
1356  case ARM::STRi12:
1357  return ARM::STR_POST_IMM;
1358  case ARM::VLDRS:
1359  return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
1360  case ARM::VLDRD:
1361  return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
1362  case ARM::VSTRS:
1363  return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
1364  case ARM::VSTRD:
1365  return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
1366  case ARM::t2LDRi8:
1367  case ARM::t2LDRi12:
1368  return ARM::t2LDR_POST;
1369  case ARM::t2STRi8:
1370  case ARM::t2STRi12:
1371  return ARM::t2STR_POST;
1372  default: llvm_unreachable("Unhandled opcode!");
1373  }
1374 }
1375 
1376 /// Fold proceeding/trailing inc/dec of base register into the
1377 /// LDR/STR/FLD{D|S}/FST{D|S} op when possible:
1378 bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
1379  // Thumb1 doesn't have updating LDR/STR.
1380  // FIXME: Use LDM/STM with single register instead.
1381  if (isThumb1) return false;
1382 
1383  unsigned Base = getLoadStoreBaseOp(*MI).getReg();
1384  bool BaseKill = getLoadStoreBaseOp(*MI).isKill();
1385  unsigned Opcode = MI->getOpcode();
1386  DebugLoc DL = MI->getDebugLoc();
1387  bool isAM5 = (Opcode == ARM::VLDRD || Opcode == ARM::VLDRS ||
1388  Opcode == ARM::VSTRD || Opcode == ARM::VSTRS);
1389  bool isAM2 = (Opcode == ARM::LDRi12 || Opcode == ARM::STRi12);
1390  if (isi32Load(Opcode) || isi32Store(Opcode))
1391  if (MI->getOperand(2).getImm() != 0)
1392  return false;
1393  if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0)
1394  return false;
1395 
1396  // Can't do the merge if the destination register is the same as the would-be
1397  // writeback register.
1398  if (MI->getOperand(0).getReg() == Base)
1399  return false;
1400 
1401  unsigned PredReg = 0;
1402  ARMCC::CondCodes Pred = getInstrPredicate(*MI, PredReg);
1403  int Bytes = getLSMultipleTransferSize(MI);
1404  MachineBasicBlock &MBB = *MI->getParent();
1405  MachineBasicBlock::iterator MBBI(MI);
1406  int Offset;
1407  MachineBasicBlock::iterator MergeInstr
1408  = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
1409  unsigned NewOpc;
1410  if (!isAM5 && Offset == Bytes) {
1411  NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
1412  } else if (Offset == -Bytes) {
1413  NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
1414  } else {
1415  MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
1416  if (Offset == Bytes) {
1417  NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
1418  } else if (!isAM5 && Offset == -Bytes) {
1419  NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
1420  } else
1421  return false;
1422  }
1423  MBB.erase(MergeInstr);
1424 
1425  ARM_AM::AddrOpc AddSub = Offset < 0 ? ARM_AM::sub : ARM_AM::add;
1426 
1427  bool isLd = isLoadSingle(Opcode);
1428  if (isAM5) {
1429  // VLDM[SD]_UPD, VSTM[SD]_UPD
1430  // (There are no base-updating versions of VLDR/VSTR instructions, but the
1431  // updating load/store-multiple instructions can be used with only one
1432  // register.)
1433  MachineOperand &MO = MI->getOperand(0);
1434  BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
1435  .addReg(Base, getDefRegState(true)) // WB base register
1436  .addReg(Base, getKillRegState(isLd ? BaseKill : false))
1437  .addImm(Pred).addReg(PredReg)
1438  .addReg(MO.getReg(), (isLd ? getDefRegState(true) :
1439  getKillRegState(MO.isKill())));
1440  } else if (isLd) {
1441  if (isAM2) {
1442  // LDR_PRE, LDR_POST
1443  if (NewOpc == ARM::LDR_PRE_IMM || NewOpc == ARM::LDRB_PRE_IMM) {
1444  BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
1445  .addReg(Base, RegState::Define)
1446  .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
1447  } else {
1448  int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
1449  BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
1450  .addReg(Base, RegState::Define)
1451  .addReg(Base)
1452  .addReg(0)
1453  .addImm(Imm)
1454  .add(predOps(Pred, PredReg));
1455  }
1456  } else {
1457  // t2LDR_PRE, t2LDR_POST
1458  BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
1459  .addReg(Base, RegState::Define)
1460  .addReg(Base)
1461  .addImm(Offset)
1462  .add(predOps(Pred, PredReg));
1463  }
1464  } else {
1465  MachineOperand &MO = MI->getOperand(0);
1466  // FIXME: post-indexed stores use am2offset_imm, which still encodes
1467  // the vestigal zero-reg offset register. When that's fixed, this clause
1468  // can be removed entirely.
1469  if (isAM2 && NewOpc == ARM::STR_POST_IMM) {
1470  int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
1471  // STR_PRE, STR_POST
1472  BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
1473  .addReg(MO.getReg(), getKillRegState(MO.isKill()))
1474  .addReg(Base)
1475  .addReg(0)
1476  .addImm(Imm)
1477  .add(predOps(Pred, PredReg));
1478  } else {
1479  // t2STR_PRE, t2STR_POST
1480  BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
1481  .addReg(MO.getReg(), getKillRegState(MO.isKill()))
1482  .addReg(Base)
1483  .addImm(Offset)
1484  .add(predOps(Pred, PredReg));
1485  }
1486  }
1487  MBB.erase(MBBI);
1488 
1489  return true;
1490 }
1491 
1492 bool ARMLoadStoreOpt::MergeBaseUpdateLSDouble(MachineInstr &MI) const {
1493  unsigned Opcode = MI.getOpcode();
1494  assert((Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) &&
1495  "Must have t2STRDi8 or t2LDRDi8");
1496  if (MI.getOperand(3).getImm() != 0)
1497  return false;
1498 
1499  // Behaviour for writeback is undefined if base register is the same as one
1500  // of the others.
1501  const MachineOperand &BaseOp = MI.getOperand(2);
1502  unsigned Base = BaseOp.getReg();
1503  const MachineOperand &Reg0Op = MI.getOperand(0);
1504  const MachineOperand &Reg1Op = MI.getOperand(1);
1505  if (Reg0Op.getReg() == Base || Reg1Op.getReg() == Base)
1506  return false;
1507 
1508  unsigned PredReg;
1509  ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
1510  MachineBasicBlock::iterator MBBI(MI);
1511  MachineBasicBlock &MBB = *MI.getParent();
1512  int Offset;
1513  MachineBasicBlock::iterator MergeInstr = findIncDecBefore(MBBI, Base, Pred,
1514  PredReg, Offset);
1515  unsigned NewOpc;
1516  if (Offset == 8 || Offset == -8) {
1517  NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_PRE : ARM::t2STRD_PRE;
1518  } else {
1519  MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
1520  if (Offset == 8 || Offset == -8) {
1521  NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_POST : ARM::t2STRD_POST;
1522  } else
1523  return false;
1524  }
1525  MBB.erase(MergeInstr);
1526 
1527  DebugLoc DL = MI.getDebugLoc();
1528  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
1529  if (NewOpc == ARM::t2LDRD_PRE || NewOpc == ARM::t2LDRD_POST) {
1530  MIB.add(Reg0Op).add(Reg1Op).addReg(BaseOp.getReg(), RegState::Define);
1531  } else {
1532  assert(NewOpc == ARM::t2STRD_PRE || NewOpc == ARM::t2STRD_POST);
1533  MIB.addReg(BaseOp.getReg(), RegState::Define).add(Reg0Op).add(Reg1Op);
1534  }
1535  MIB.addReg(BaseOp.getReg(), RegState::Kill)
1536  .addImm(Offset).addImm(Pred).addReg(PredReg);
1537  assert(TII->get(Opcode).getNumOperands() == 6 &&
1538  TII->get(NewOpc).getNumOperands() == 7 &&
1539  "Unexpected number of operands in Opcode specification.");
1540 
1541  // Transfer implicit operands.
1542  for (const MachineOperand &MO : MI.implicit_operands())
1543  MIB.add(MO);
1544  MIB.setMemRefs(MI.memoperands());
1545 
1546  MBB.erase(MBBI);
1547  return true;
1548 }
1549 
1550 /// Returns true if instruction is a memory operation that this pass is capable
1551 /// of operating on.
1552 static bool isMemoryOp(const MachineInstr &MI) {
1553  unsigned Opcode = MI.getOpcode();
1554  switch (Opcode) {
1555  case ARM::VLDRS:
1556  case ARM::VSTRS:
1557  case ARM::VLDRD:
1558  case ARM::VSTRD:
1559  case ARM::LDRi12:
1560  case ARM::STRi12:
1561  case ARM::tLDRi:
1562  case ARM::tSTRi:
1563  case ARM::tLDRspi:
1564  case ARM::tSTRspi:
1565  case ARM::t2LDRi8:
1566  case ARM::t2LDRi12:
1567  case ARM::t2STRi8:
1568  case ARM::t2STRi12:
1569  break;
1570  default:
1571  return false;
1572  }
1573  if (!MI.getOperand(1).isReg())
1574  return false;
1575 
1576  // When no memory operands are present, conservatively assume unaligned,
1577  // volatile, unfoldable.
1578  if (!MI.hasOneMemOperand())
1579  return false;
1580 
1581  const MachineMemOperand &MMO = **MI.memoperands_begin();
1582 
1583  // Don't touch volatile memory accesses - we may be changing their order.
1584  if (MMO.isVolatile())
1585  return false;
1586 
1587  // Unaligned ldr/str is emulated by some kernels, but unaligned ldm/stm is
1588  // not.
1589  if (MMO.getAlignment() < 4)
1590  return false;
1591 
1592  // str <undef> could probably be eliminated entirely, but for now we just want
1593  // to avoid making a mess of it.
1594  // FIXME: Use str <undef> as a wildcard to enable better stm folding.
1595  if (MI.getOperand(0).isReg() && MI.getOperand(0).isUndef())
1596  return false;
1597 
1598  // Likewise don't mess with references to undefined addresses.
1599  if (MI.getOperand(1).isUndef())
1600  return false;
1601 
1602  return true;
1603 }
1604 
1606  MachineBasicBlock::iterator &MBBI, int Offset,
1607  bool isDef, unsigned NewOpc, unsigned Reg,
1608  bool RegDeadKill, bool RegUndef, unsigned BaseReg,
1609  bool BaseKill, bool BaseUndef, ARMCC::CondCodes Pred,
1610  unsigned PredReg, const TargetInstrInfo *TII) {
1611  if (isDef) {
1612  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
1613  TII->get(NewOpc))
1614  .addReg(Reg, getDefRegState(true) | getDeadRegState(RegDeadKill))
1615  .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef));
1616  MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
1617  } else {
1618  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
1619  TII->get(NewOpc))
1620  .addReg(Reg, getKillRegState(RegDeadKill) | getUndefRegState(RegUndef))
1621  .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef));
1622  MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
1623  }
1624 }
1625 
1626 bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
1628  MachineInstr *MI = &*MBBI;
1629  unsigned Opcode = MI->getOpcode();
1630  // FIXME: Code/comments below check Opcode == t2STRDi8, but this check returns
1631  // if we see this opcode.
1632  if (Opcode != ARM::LDRD && Opcode != ARM::STRD && Opcode != ARM::t2LDRDi8)
1633  return false;
1634 
1635  const MachineOperand &BaseOp = MI->getOperand(2);
1636  unsigned BaseReg = BaseOp.getReg();
1637  unsigned EvenReg = MI->getOperand(0).getReg();
1638  unsigned OddReg = MI->getOperand(1).getReg();
1639  unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
1640  unsigned OddRegNum = TRI->getDwarfRegNum(OddReg, false);
1641 
1642  // ARM errata 602117: LDRD with base in list may result in incorrect base
1643  // register when interrupted or faulted.
1644  bool Errata602117 = EvenReg == BaseReg &&
1645  (Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8) && STI->isCortexM3();
1646  // ARM LDRD/STRD needs consecutive registers.
1647  bool NonConsecutiveRegs = (Opcode == ARM::LDRD || Opcode == ARM::STRD) &&
1648  (EvenRegNum % 2 != 0 || EvenRegNum + 1 != OddRegNum);
1649 
1650  if (!Errata602117 && !NonConsecutiveRegs)
1651  return false;
1652 
1653  bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
1654  bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
1655  bool EvenDeadKill = isLd ?
1656  MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
1657  bool EvenUndef = MI->getOperand(0).isUndef();
1658  bool OddDeadKill = isLd ?
1659  MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
1660  bool OddUndef = MI->getOperand(1).isUndef();
1661  bool BaseKill = BaseOp.isKill();
1662  bool BaseUndef = BaseOp.isUndef();
1663  assert((isT2 || MI->getOperand(3).getReg() == ARM::NoRegister) &&
1664  "register offset not handled below");
1665  int OffImm = getMemoryOpOffset(*MI);
1666  unsigned PredReg = 0;
1667  ARMCC::CondCodes Pred = getInstrPredicate(*MI, PredReg);
1668 
1669  if (OddRegNum > EvenRegNum && OffImm == 0) {
1670  // Ascending register numbers and no offset. It's safe to change it to a
1671  // ldm or stm.
1672  unsigned NewOpc = (isLd)
1673  ? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
1674  : (isT2 ? ARM::t2STMIA : ARM::STMIA);
1675  if (isLd) {
1676  BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
1677  .addReg(BaseReg, getKillRegState(BaseKill))
1678  .addImm(Pred).addReg(PredReg)
1679  .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
1680  .addReg(OddReg, getDefRegState(isLd) | getDeadRegState(OddDeadKill));
1681  ++NumLDRD2LDM;
1682  } else {
1683  BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
1684  .addReg(BaseReg, getKillRegState(BaseKill))
1685  .addImm(Pred).addReg(PredReg)
1686  .addReg(EvenReg,
1687  getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
1688  .addReg(OddReg,
1689  getKillRegState(OddDeadKill) | getUndefRegState(OddUndef));
1690  ++NumSTRD2STM;
1691  }
1692  } else {
1693  // Split into two instructions.
1694  unsigned NewOpc = (isLd)
1695  ? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
1696  : (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
1697  // Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
1698  // so adjust and use t2LDRi12 here for that.
1699  unsigned NewOpc2 = (isLd)
1700  ? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
1701  : (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
1702  // If this is a load, make sure the first load does not clobber the base
1703  // register before the second load reads it.
1704  if (isLd && TRI->regsOverlap(EvenReg, BaseReg)) {
1705  assert(!TRI->regsOverlap(OddReg, BaseReg));
1706  InsertLDR_STR(MBB, MBBI, OffImm + 4, isLd, NewOpc2, OddReg, OddDeadKill,
1707  false, BaseReg, false, BaseUndef, Pred, PredReg, TII);
1708  InsertLDR_STR(MBB, MBBI, OffImm, isLd, NewOpc, EvenReg, EvenDeadKill,
1709  false, BaseReg, BaseKill, BaseUndef, Pred, PredReg, TII);
1710  } else {
1711  if (OddReg == EvenReg && EvenDeadKill) {
1712  // If the two source operands are the same, the kill marker is
1713  // probably on the first one. e.g.
1714  // t2STRDi8 killed %r5, %r5, killed %r9, 0, 14, %reg0
1715  EvenDeadKill = false;
1716  OddDeadKill = true;
1717  }
1718  // Never kill the base register in the first instruction.
1719  if (EvenReg == BaseReg)
1720  EvenDeadKill = false;
1721  InsertLDR_STR(MBB, MBBI, OffImm, isLd, NewOpc, EvenReg, EvenDeadKill,
1722  EvenUndef, BaseReg, false, BaseUndef, Pred, PredReg, TII);
1723  InsertLDR_STR(MBB, MBBI, OffImm + 4, isLd, NewOpc2, OddReg, OddDeadKill,
1724  OddUndef, BaseReg, BaseKill, BaseUndef, Pred, PredReg, TII);
1725  }
1726  if (isLd)
1727  ++NumLDRD2LDR;
1728  else
1729  ++NumSTRD2STR;
1730  }
1731 
1732  MBBI = MBB.erase(MBBI);
1733  return true;
1734 }
1735 
1736 /// An optimization pass to turn multiple LDR / STR ops of the same base and
1737 /// incrementing offset into LDM / STM ops.
1738 bool ARMLoadStoreOpt::LoadStoreMultipleOpti(MachineBasicBlock &MBB) {
1739  MemOpQueue MemOps;
1740  unsigned CurrBase = 0;
1741  unsigned CurrOpc = ~0u;
1742  ARMCC::CondCodes CurrPred = ARMCC::AL;
1743  unsigned Position = 0;
1744  assert(Candidates.size() == 0);
1745  assert(MergeBaseCandidates.size() == 0);
1746  LiveRegsValid = false;
1747 
1748  for (MachineBasicBlock::iterator I = MBB.end(), MBBI; I != MBB.begin();
1749  I = MBBI) {
1750  // The instruction in front of the iterator is the one we look at.
1751  MBBI = std::prev(I);
1752  if (FixInvalidRegPairOp(MBB, MBBI))
1753  continue;
1754  ++Position;
1755 
1756  if (isMemoryOp(*MBBI)) {
1757  unsigned Opcode = MBBI->getOpcode();
1758  const MachineOperand &MO = MBBI->getOperand(0);
1759  unsigned Reg = MO.getReg();
1760  unsigned Base = getLoadStoreBaseOp(*MBBI).getReg();
1761  unsigned PredReg = 0;
1762  ARMCC::CondCodes Pred = getInstrPredicate(*MBBI, PredReg);
1763  int Offset = getMemoryOpOffset(*MBBI);
1764  if (CurrBase == 0) {
1765  // Start of a new chain.
1766  CurrBase = Base;
1767  CurrOpc = Opcode;
1768  CurrPred = Pred;
1769  MemOps.push_back(MemOpQueueEntry(*MBBI, Offset, Position));
1770  continue;
1771  }
1772  // Note: No need to match PredReg in the next if.
1773  if (CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred) {
1774  // Watch out for:
1775  // r4 := ldr [r0, #8]
1776  // r4 := ldr [r0, #4]
1777  // or
1778  // r0 := ldr [r0]
1779  // If a load overrides the base register or a register loaded by
1780  // another load in our chain, we cannot take this instruction.
1781  bool Overlap = false;
1782  if (isLoadSingle(Opcode)) {
1783  Overlap = (Base == Reg);
1784  if (!Overlap) {
1785  for (const MemOpQueueEntry &E : MemOps) {
1786  if (TRI->regsOverlap(Reg, E.MI->getOperand(0).getReg())) {
1787  Overlap = true;
1788  break;
1789  }
1790  }
1791  }
1792  }
1793 
1794  if (!Overlap) {
1795  // Check offset and sort memory operation into the current chain.
1796  if (Offset > MemOps.back().Offset) {
1797  MemOps.push_back(MemOpQueueEntry(*MBBI, Offset, Position));
1798  continue;
1799  } else {
1800  MemOpQueue::iterator MI, ME;
1801  for (MI = MemOps.begin(), ME = MemOps.end(); MI != ME; ++MI) {
1802  if (Offset < MI->Offset) {
1803  // Found a place to insert.
1804  break;
1805  }
1806  if (Offset == MI->Offset) {
1807  // Collision, abort.
1808  MI = ME;
1809  break;
1810  }
1811  }
1812  if (MI != MemOps.end()) {
1813  MemOps.insert(MI, MemOpQueueEntry(*MBBI, Offset, Position));
1814  continue;
1815  }
1816  }
1817  }
1818  }
1819 
1820  // Don't advance the iterator; The op will start a new chain next.
1821  MBBI = I;
1822  --Position;
1823  // Fallthrough to look into existing chain.
1824  } else if (MBBI->isDebugInstr()) {
1825  continue;
1826  } else if (MBBI->getOpcode() == ARM::t2LDRDi8 ||
1827  MBBI->getOpcode() == ARM::t2STRDi8) {
1828  // ARMPreAllocLoadStoreOpt has already formed some LDRD/STRD instructions
1829  // remember them because we may still be able to merge add/sub into them.
1830  MergeBaseCandidates.push_back(&*MBBI);
1831  }
1832 
1833  // If we are here then the chain is broken; Extract candidates for a merge.
1834  if (MemOps.size() > 0) {
1835  FormCandidates(MemOps);
1836  // Reset for the next chain.
1837  CurrBase = 0;
1838  CurrOpc = ~0u;
1839  CurrPred = ARMCC::AL;
1840  MemOps.clear();
1841  }
1842  }
1843  if (MemOps.size() > 0)
1844  FormCandidates(MemOps);
1845 
1846  // Sort candidates so they get processed from end to begin of the basic
1847  // block later; This is necessary for liveness calculation.
1848  auto LessThan = [](const MergeCandidate* M0, const MergeCandidate *M1) {
1849  return M0->InsertPos < M1->InsertPos;
1850  };
1851  llvm::sort(Candidates, LessThan);
1852 
1853  // Go through list of candidates and merge.
1854  bool Changed = false;
1855  for (const MergeCandidate *Candidate : Candidates) {
1856  if (Candidate->CanMergeToLSMulti || Candidate->CanMergeToLSDouble) {
1857  MachineInstr *Merged = MergeOpsUpdate(*Candidate);
1858  // Merge preceding/trailing base inc/dec into the merged op.
1859  if (Merged) {
1860  Changed = true;
1861  unsigned Opcode = Merged->getOpcode();
1862  if (Opcode == ARM::t2STRDi8 || Opcode == ARM::t2LDRDi8)
1863  MergeBaseUpdateLSDouble(*Merged);
1864  else
1865  MergeBaseUpdateLSMultiple(Merged);
1866  } else {
1867  for (MachineInstr *MI : Candidate->Instrs) {
1868  if (MergeBaseUpdateLoadStore(MI))
1869  Changed = true;
1870  }
1871  }
1872  } else {
1873  assert(Candidate->Instrs.size() == 1);
1874  if (MergeBaseUpdateLoadStore(Candidate->Instrs.front()))
1875  Changed = true;
1876  }
1877  }
1878  Candidates.clear();
1879  // Try to fold add/sub into the LDRD/STRD formed by ARMPreAllocLoadStoreOpt.
1880  for (MachineInstr *MI : MergeBaseCandidates)
1881  MergeBaseUpdateLSDouble(*MI);
1882  MergeBaseCandidates.clear();
1883 
1884  return Changed;
1885 }
1886 
1887 /// If this is a exit BB, try merging the return ops ("bx lr" and "mov pc, lr")
1888 /// into the preceding stack restore so it directly restore the value of LR
1889 /// into pc.
1890 /// ldmfd sp!, {..., lr}
1891 /// bx lr
1892 /// or
1893 /// ldmfd sp!, {..., lr}
1894 /// mov pc, lr
1895 /// =>
1896 /// ldmfd sp!, {..., pc}
1897 bool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) {
1898  // Thumb1 LDM doesn't allow high registers.
1899  if (isThumb1) return false;
1900  if (MBB.empty()) return false;
1901 
1903  if (MBBI != MBB.begin() && MBBI != MBB.end() &&
1904  (MBBI->getOpcode() == ARM::BX_RET ||
1905  MBBI->getOpcode() == ARM::tBX_RET ||
1906  MBBI->getOpcode() == ARM::MOVPCLR)) {
1907  MachineBasicBlock::iterator PrevI = std::prev(MBBI);
1908  // Ignore any debug instructions.
1909  while (PrevI->isDebugInstr() && PrevI != MBB.begin())
1910  --PrevI;
1911  MachineInstr &PrevMI = *PrevI;
1912  unsigned Opcode = PrevMI.getOpcode();
1913  if (Opcode == ARM::LDMIA_UPD || Opcode == ARM::LDMDA_UPD ||
1914  Opcode == ARM::LDMDB_UPD || Opcode == ARM::LDMIB_UPD ||
1915  Opcode == ARM::t2LDMIA_UPD || Opcode == ARM::t2LDMDB_UPD) {
1916  MachineOperand &MO = PrevMI.getOperand(PrevMI.getNumOperands() - 1);
1917  if (MO.getReg() != ARM::LR)
1918  return false;
1919  unsigned NewOpc = (isThumb2 ? ARM::t2LDMIA_RET : ARM::LDMIA_RET);
1920  assert(((isThumb2 && Opcode == ARM::t2LDMIA_UPD) ||
1921  Opcode == ARM::LDMIA_UPD) && "Unsupported multiple load-return!");
1922  PrevMI.setDesc(TII->get(NewOpc));
1923  MO.setReg(ARM::PC);
1924  PrevMI.copyImplicitOps(*MBB.getParent(), *MBBI);
1925  MBB.erase(MBBI);
1926  // We now restore LR into PC so it is not live-out of the return block
1927  // anymore: Clear the CSI Restored bit.
1928  MachineFrameInfo &MFI = MBB.getParent()->getFrameInfo();
1929  // CSI should be fixed after PrologEpilog Insertion
1930  assert(MFI.isCalleeSavedInfoValid() && "CSI should be valid");
1931  for (CalleeSavedInfo &Info : MFI.getCalleeSavedInfo()) {
1932  if (Info.getReg() == ARM::LR) {
1933  Info.setRestored(false);
1934  break;
1935  }
1936  }
1937  return true;
1938  }
1939  }
1940  return false;
1941 }
1942 
1943 bool ARMLoadStoreOpt::CombineMovBx(MachineBasicBlock &MBB) {
1945  if (MBBI == MBB.begin() || MBBI == MBB.end() ||
1946  MBBI->getOpcode() != ARM::tBX_RET)
1947  return false;
1948 
1949  MachineBasicBlock::iterator Prev = MBBI;
1950  --Prev;
1951  if (Prev->getOpcode() != ARM::tMOVr || !Prev->definesRegister(ARM::LR))
1952  return false;
1953 
1954  for (auto Use : Prev->uses())
1955  if (Use.isKill()) {
1956  assert(STI->hasV4TOps());
1957  BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(ARM::tBX))
1958  .addReg(Use.getReg(), RegState::Kill)
1959  .add(predOps(ARMCC::AL))
1960  .copyImplicitOps(*MBBI);
1961  MBB.erase(MBBI);
1962  MBB.erase(Prev);
1963  return true;
1964  }
1965 
1966  llvm_unreachable("tMOVr doesn't kill a reg before tBX_RET?");
1967 }
1968 
1969 bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
1970  if (skipFunction(Fn.getFunction()))
1971  return false;
1972 
1973  MF = &Fn;
1974  STI = &static_cast<const ARMSubtarget &>(Fn.getSubtarget());
1975  TL = STI->getTargetLowering();
1976  AFI = Fn.getInfo<ARMFunctionInfo>();
1977  TII = STI->getInstrInfo();
1978  TRI = STI->getRegisterInfo();
1979 
1980  RegClassInfoValid = false;
1981  isThumb2 = AFI->isThumb2Function();
1982  isThumb1 = AFI->isThumbFunction() && !isThumb2;
1983 
1984  bool Modified = false;
1985  for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
1986  ++MFI) {
1987  MachineBasicBlock &MBB = *MFI;
1988  Modified |= LoadStoreMultipleOpti(MBB);
1989  if (STI->hasV5TOps())
1990  Modified |= MergeReturnIntoLDM(MBB);
1991  if (isThumb1)
1992  Modified |= CombineMovBx(MBB);
1993  }
1994 
1995  Allocator.DestroyAll();
1996  return Modified;
1997 }
1998 
1999 #define ARM_PREALLOC_LOAD_STORE_OPT_NAME \
2000  "ARM pre- register allocation load / store optimization pass"
2001 
2002 namespace {
2003 
2004  /// Pre- register allocation pass that move load / stores from consecutive
2005  /// locations close to make it more likely they will be combined later.
2006  struct ARMPreAllocLoadStoreOpt : public MachineFunctionPass{
2007  static char ID;
2008 
2009  AliasAnalysis *AA;
2010  const DataLayout *TD;
2011  const TargetInstrInfo *TII;
2012  const TargetRegisterInfo *TRI;
2013  const ARMSubtarget *STI;
2015  MachineFunction *MF;
2016 
2017  ARMPreAllocLoadStoreOpt() : MachineFunctionPass(ID) {}
2018 
2019  bool runOnMachineFunction(MachineFunction &Fn) override;
2020 
2021  StringRef getPassName() const override {
2023  }
2024 
2025  void getAnalysisUsage(AnalysisUsage &AU) const override {
2028  }
2029 
2030  private:
2031  bool CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1, DebugLoc &dl,
2032  unsigned &NewOpc, unsigned &EvenReg,
2033  unsigned &OddReg, unsigned &BaseReg,
2034  int &Offset,
2035  unsigned &PredReg, ARMCC::CondCodes &Pred,
2036  bool &isT2);
2037  bool RescheduleOps(MachineBasicBlock *MBB,
2039  unsigned Base, bool isLd,
2041  bool RescheduleLoadStoreInstrs(MachineBasicBlock *MBB);
2042  };
2043 
2044 } // end anonymous namespace
2045 
2047 
2048 INITIALIZE_PASS(ARMPreAllocLoadStoreOpt, "arm-prera-ldst-opt",
2049  ARM_PREALLOC_LOAD_STORE_OPT_NAME, false, false)
2050 
2051 bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
2052  if (AssumeMisalignedLoadStores || skipFunction(Fn.getFunction()))
2053  return false;
2054 
2055  TD = &Fn.getDataLayout();
2056  STI = &static_cast<const ARMSubtarget &>(Fn.getSubtarget());
2057  TII = STI->getInstrInfo();
2058  TRI = STI->getRegisterInfo();
2059  MRI = &Fn.getRegInfo();
2060  MF = &Fn;
2061  AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2062 
2063  bool Modified = false;
2064  for (MachineBasicBlock &MFI : Fn)
2065  Modified |= RescheduleLoadStoreInstrs(&MFI);
2066 
2067  return Modified;
2068 }
2069 
2070 static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base,
2074  SmallSet<unsigned, 4> &MemRegs,
2075  const TargetRegisterInfo *TRI,
2076  AliasAnalysis *AA) {
2077  // Are there stores / loads / calls between them?
2078  SmallSet<unsigned, 4> AddedRegPressure;
2079  while (++I != E) {
2080  if (I->isDebugInstr() || MemOps.count(&*I))
2081  continue;
2082  if (I->isCall() || I->isTerminator() || I->hasUnmodeledSideEffects())
2083  return false;
2084  if (I->mayStore() || (!isLd && I->mayLoad()))
2085  for (MachineInstr *MemOp : MemOps)
2086  if (I->mayAlias(AA, *MemOp, /*UseTBAA*/ false))
2087  return false;
2088  for (unsigned j = 0, NumOps = I->getNumOperands(); j != NumOps; ++j) {
2089  MachineOperand &MO = I->getOperand(j);
2090  if (!MO.isReg())
2091  continue;
2092  unsigned Reg = MO.getReg();
2093  if (MO.isDef() && TRI->regsOverlap(Reg, Base))
2094  return false;
2095  if (Reg != Base && !MemRegs.count(Reg))
2096  AddedRegPressure.insert(Reg);
2097  }
2098  }
2099 
2100  // Estimate register pressure increase due to the transformation.
2101  if (MemRegs.size() <= 4)
2102  // Ok if we are moving small number of instructions.
2103  return true;
2104  return AddedRegPressure.size() <= MemRegs.size() * 2;
2105 }
2106 
2107 bool
2108 ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1,
2109  DebugLoc &dl, unsigned &NewOpc,
2110  unsigned &FirstReg,
2111  unsigned &SecondReg,
2112  unsigned &BaseReg, int &Offset,
2113  unsigned &PredReg,
2114  ARMCC::CondCodes &Pred,
2115  bool &isT2) {
2116  // Make sure we're allowed to generate LDRD/STRD.
2117  if (!STI->hasV5TEOps())
2118  return false;
2119 
2120  // FIXME: VLDRS / VSTRS -> VLDRD / VSTRD
2121  unsigned Scale = 1;
2122  unsigned Opcode = Op0->getOpcode();
2123  if (Opcode == ARM::LDRi12) {
2124  NewOpc = ARM::LDRD;
2125  } else if (Opcode == ARM::STRi12) {
2126  NewOpc = ARM::STRD;
2127  } else if (Opcode == ARM::t2LDRi8 || Opcode == ARM::t2LDRi12) {
2128  NewOpc = ARM::t2LDRDi8;
2129  Scale = 4;
2130  isT2 = true;
2131  } else if (Opcode == ARM::t2STRi8 || Opcode == ARM::t2STRi12) {
2132  NewOpc = ARM::t2STRDi8;
2133  Scale = 4;
2134  isT2 = true;
2135  } else {
2136  return false;
2137  }
2138 
2139  // Make sure the base address satisfies i64 ld / st alignment requirement.
2140  // At the moment, we ignore the memoryoperand's value.
2141  // If we want to use AliasAnalysis, we should check it accordingly.
2142  if (!Op0->hasOneMemOperand() ||
2143  (*Op0->memoperands_begin())->isVolatile())
2144  return false;
2145 
2146  unsigned Align = (*Op0->memoperands_begin())->getAlignment();
2147  const Function &Func = MF->getFunction();
2148  unsigned ReqAlign = STI->hasV6Ops()
2149  ? TD->getABITypeAlignment(Type::getInt64Ty(Func.getContext()))
2150  : 8; // Pre-v6 need 8-byte align
2151  if (Align < ReqAlign)
2152  return false;
2153 
2154  // Then make sure the immediate offset fits.
2155  int OffImm = getMemoryOpOffset(*Op0);
2156  if (isT2) {
2157  int Limit = (1 << 8) * Scale;
2158  if (OffImm >= Limit || (OffImm <= -Limit) || (OffImm & (Scale-1)))
2159  return false;
2160  Offset = OffImm;
2161  } else {
2162  ARM_AM::AddrOpc AddSub = ARM_AM::add;
2163  if (OffImm < 0) {
2164  AddSub = ARM_AM::sub;
2165  OffImm = - OffImm;
2166  }
2167  int Limit = (1 << 8) * Scale;
2168  if (OffImm >= Limit || (OffImm & (Scale-1)))
2169  return false;
2170  Offset = ARM_AM::getAM3Opc(AddSub, OffImm);
2171  }
2172  FirstReg = Op0->getOperand(0).getReg();
2173  SecondReg = Op1->getOperand(0).getReg();
2174  if (FirstReg == SecondReg)
2175  return false;
2176  BaseReg = Op0->getOperand(1).getReg();
2177  Pred = getInstrPredicate(*Op0, PredReg);
2178  dl = Op0->getDebugLoc();
2179  return true;
2180 }
2181 
2182 bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
2184  unsigned Base, bool isLd,
2185  DenseMap<MachineInstr*, unsigned> &MI2LocMap) {
2186  bool RetVal = false;
2187 
2188  // Sort by offset (in reverse order).
2189  llvm::sort(Ops, [](const MachineInstr *LHS, const MachineInstr *RHS) {
2190  int LOffset = getMemoryOpOffset(*LHS);
2191  int ROffset = getMemoryOpOffset(*RHS);
2192  assert(LHS == RHS || LOffset != ROffset);
2193  return LOffset > ROffset;
2194  });
2195 
2196  // The loads / stores of the same base are in order. Scan them from first to
2197  // last and check for the following:
2198  // 1. Any def of base.
2199  // 2. Any gaps.
2200  while (Ops.size() > 1) {
2201  unsigned FirstLoc = ~0U;
2202  unsigned LastLoc = 0;
2203  MachineInstr *FirstOp = nullptr;
2204  MachineInstr *LastOp = nullptr;
2205  int LastOffset = 0;
2206  unsigned LastOpcode = 0;
2207  unsigned LastBytes = 0;
2208  unsigned NumMove = 0;
2209  for (int i = Ops.size() - 1; i >= 0; --i) {
2210  // Make sure each operation has the same kind.
2211  MachineInstr *Op = Ops[i];
2212  unsigned LSMOpcode
2214  if (LastOpcode && LSMOpcode != LastOpcode)
2215  break;
2216 
2217  // Check that we have a continuous set of offsets.
2218  int Offset = getMemoryOpOffset(*Op);
2219  unsigned Bytes = getLSMultipleTransferSize(Op);
2220  if (LastBytes) {
2221  if (Bytes != LastBytes || Offset != (LastOffset + (int)Bytes))
2222  break;
2223  }
2224 
2225  // Don't try to reschedule too many instructions.
2226  if (NumMove == 8) // FIXME: Tune this limit.
2227  break;
2228 
2229  // Found a mergable instruction; save information about it.
2230  ++NumMove;
2231  LastOffset = Offset;
2232  LastBytes = Bytes;
2233  LastOpcode = LSMOpcode;
2234 
2235  unsigned Loc = MI2LocMap[Op];
2236  if (Loc <= FirstLoc) {
2237  FirstLoc = Loc;
2238  FirstOp = Op;
2239  }
2240  if (Loc >= LastLoc) {
2241  LastLoc = Loc;
2242  LastOp = Op;
2243  }
2244  }
2245 
2246  if (NumMove <= 1)
2247  Ops.pop_back();
2248  else {
2250  SmallSet<unsigned, 4> MemRegs;
2251  for (size_t i = Ops.size() - NumMove, e = Ops.size(); i != e; ++i) {
2252  MemOps.insert(Ops[i]);
2253  MemRegs.insert(Ops[i]->getOperand(0).getReg());
2254  }
2255 
2256  // Be conservative, if the instructions are too far apart, don't
2257  // move them. We want to limit the increase of register pressure.
2258  bool DoMove = (LastLoc - FirstLoc) <= NumMove*4; // FIXME: Tune this.
2259  if (DoMove)
2260  DoMove = IsSafeAndProfitableToMove(isLd, Base, FirstOp, LastOp,
2261  MemOps, MemRegs, TRI, AA);
2262  if (!DoMove) {
2263  for (unsigned i = 0; i != NumMove; ++i)
2264  Ops.pop_back();
2265  } else {
2266  // This is the new location for the loads / stores.
2267  MachineBasicBlock::iterator InsertPos = isLd ? FirstOp : LastOp;
2268  while (InsertPos != MBB->end() &&
2269  (MemOps.count(&*InsertPos) || InsertPos->isDebugInstr()))
2270  ++InsertPos;
2271 
2272  // If we are moving a pair of loads / stores, see if it makes sense
2273  // to try to allocate a pair of registers that can form register pairs.
2274  MachineInstr *Op0 = Ops.back();
2275  MachineInstr *Op1 = Ops[Ops.size()-2];
2276  unsigned FirstReg = 0, SecondReg = 0;
2277  unsigned BaseReg = 0, PredReg = 0;
2278  ARMCC::CondCodes Pred = ARMCC::AL;
2279  bool isT2 = false;
2280  unsigned NewOpc = 0;
2281  int Offset = 0;
2282  DebugLoc dl;
2283  if (NumMove == 2 && CanFormLdStDWord(Op0, Op1, dl, NewOpc,
2284  FirstReg, SecondReg, BaseReg,
2285  Offset, PredReg, Pred, isT2)) {
2286  Ops.pop_back();
2287  Ops.pop_back();
2288 
2289  const MCInstrDesc &MCID = TII->get(NewOpc);
2290  const TargetRegisterClass *TRC = TII->getRegClass(MCID, 0, TRI, *MF);
2291  MRI->constrainRegClass(FirstReg, TRC);
2292  MRI->constrainRegClass(SecondReg, TRC);
2293 
2294  // Form the pair instruction.
2295  if (isLd) {
2296  MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
2297  .addReg(FirstReg, RegState::Define)
2298  .addReg(SecondReg, RegState::Define)
2299  .addReg(BaseReg);
2300  // FIXME: We're converting from LDRi12 to an insn that still
2301  // uses addrmode2, so we need an explicit offset reg. It should
2302  // always by reg0 since we're transforming LDRi12s.
2303  if (!isT2)
2304  MIB.addReg(0);
2305  MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
2306  MIB.cloneMergedMemRefs({Op0, Op1});
2307  LLVM_DEBUG(dbgs() << "Formed " << *MIB << "\n");
2308  ++NumLDRDFormed;
2309  } else {
2310  MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
2311  .addReg(FirstReg)
2312  .addReg(SecondReg)
2313  .addReg(BaseReg);
2314  // FIXME: We're converting from LDRi12 to an insn that still
2315  // uses addrmode2, so we need an explicit offset reg. It should
2316  // always by reg0 since we're transforming STRi12s.
2317  if (!isT2)
2318  MIB.addReg(0);
2319  MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
2320  MIB.cloneMergedMemRefs({Op0, Op1});
2321  LLVM_DEBUG(dbgs() << "Formed " << *MIB << "\n");
2322  ++NumSTRDFormed;
2323  }
2324  MBB->erase(Op0);
2325  MBB->erase(Op1);
2326 
2327  if (!isT2) {
2328  // Add register allocation hints to form register pairs.
2329  MRI->setRegAllocationHint(FirstReg, ARMRI::RegPairEven, SecondReg);
2330  MRI->setRegAllocationHint(SecondReg, ARMRI::RegPairOdd, FirstReg);
2331  }
2332  } else {
2333  for (unsigned i = 0; i != NumMove; ++i) {
2334  MachineInstr *Op = Ops.back();
2335  Ops.pop_back();
2336  MBB->splice(InsertPos, MBB, Op);
2337  }
2338  }
2339 
2340  NumLdStMoved += NumMove;
2341  RetVal = true;
2342  }
2343  }
2344  }
2345 
2346  return RetVal;
2347 }
2348 
2349 bool
2350 ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
2351  bool RetVal = false;
2352 
2356  SmallVector<unsigned, 4> LdBases;
2357  SmallVector<unsigned, 4> StBases;
2358 
2359  unsigned Loc = 0;
2360  MachineBasicBlock::iterator MBBI = MBB->begin();
2362  while (MBBI != E) {
2363  for (; MBBI != E; ++MBBI) {
2364  MachineInstr &MI = *MBBI;
2365  if (MI.isCall() || MI.isTerminator()) {
2366  // Stop at barriers.
2367  ++MBBI;
2368  break;
2369  }
2370 
2371  if (!MI.isDebugInstr())
2372  MI2LocMap[&MI] = ++Loc;
2373 
2374  if (!isMemoryOp(MI))
2375  continue;
2376  unsigned PredReg = 0;
2377  if (getInstrPredicate(MI, PredReg) != ARMCC::AL)
2378  continue;
2379 
2380  int Opc = MI.getOpcode();
2381  bool isLd = isLoadSingle(Opc);
2382  unsigned Base = MI.getOperand(1).getReg();
2383  int Offset = getMemoryOpOffset(MI);
2384 
2385  bool StopHere = false;
2386  if (isLd) {
2388  Base2LdsMap.find(Base);
2389  if (BI != Base2LdsMap.end()) {
2390  for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
2391  if (Offset == getMemoryOpOffset(*BI->second[i])) {
2392  StopHere = true;
2393  break;
2394  }
2395  }
2396  if (!StopHere)
2397  BI->second.push_back(&MI);
2398  } else {
2399  Base2LdsMap[Base].push_back(&MI);
2400  LdBases.push_back(Base);
2401  }
2402  } else {
2404  Base2StsMap.find(Base);
2405  if (BI != Base2StsMap.end()) {
2406  for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
2407  if (Offset == getMemoryOpOffset(*BI->second[i])) {
2408  StopHere = true;
2409  break;
2410  }
2411  }
2412  if (!StopHere)
2413  BI->second.push_back(&MI);
2414  } else {
2415  Base2StsMap[Base].push_back(&MI);
2416  StBases.push_back(Base);
2417  }
2418  }
2419 
2420  if (StopHere) {
2421  // Found a duplicate (a base+offset combination that's seen earlier).
2422  // Backtrack.
2423  --Loc;
2424  break;
2425  }
2426  }
2427 
2428  // Re-schedule loads.
2429  for (unsigned i = 0, e = LdBases.size(); i != e; ++i) {
2430  unsigned Base = LdBases[i];
2431  SmallVectorImpl<MachineInstr *> &Lds = Base2LdsMap[Base];
2432  if (Lds.size() > 1)
2433  RetVal |= RescheduleOps(MBB, Lds, Base, true, MI2LocMap);
2434  }
2435 
2436  // Re-schedule stores.
2437  for (unsigned i = 0, e = StBases.size(); i != e; ++i) {
2438  unsigned Base = StBases[i];
2439  SmallVectorImpl<MachineInstr *> &Sts = Base2StsMap[Base];
2440  if (Sts.size() > 1)
2441  RetVal |= RescheduleOps(MBB, Sts, Base, false, MI2LocMap);
2442  }
2443 
2444  if (MBBI != E) {
2445  Base2LdsMap.clear();
2446  Base2StsMap.clear();
2447  LdBases.clear();
2448  StBases.clear();
2449  }
2450  }
2451 
2452  return RetVal;
2453 }
2454 
2455 /// Returns an instance of the load / store optimization pass.
2457  if (PreAlloc)
2458  return new ARMPreAllocLoadStoreOpt();
2459  return new ARMLoadStoreOpt();
2460 }
const Function & getFunction() const
Definition: Function.h:134
uint64_t CallInst * C
static unsigned getPreIndexedLoadStoreOpcode(unsigned Opc, ARM_AM::AddrOpc Mode)
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand *> MMOs) const
const MachineInstrBuilder & add(const MachineOperand &MO) const
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
bool isCalleeSavedInfoValid() const
Has the callee saved info been calculated yet?
static bool isT1i32Load(unsigned Opc)
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
bool isCall(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:633
SI Whole Quad Mode
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
iterator_range< mop_iterator > uses()
Returns a range that includes all operands that are register uses.
Definition: MachineInstr.h:492
LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI, unsigned Reg, const_iterator Before, unsigned Neighborhood=10) const
Return whether (physical) register Reg has been defined and not killed as of just before Before...
static unsigned getImmScale(unsigned Opc)
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:383
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
unsigned getReg() const
getReg - Returns the register number.
const ARMTargetLowering * getTargetLowering() const override
Definition: ARMSubtarget.h:495
unsigned Reg
unsigned second
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:34
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:177
static int isIncrementOrDecrement(const MachineInstr &MI, unsigned Reg, ARMCC::CondCodes Pred, unsigned PredReg)
Check if the given instruction increments or decrements a register and return the amount it is increm...
This file defines the MallocAllocator and BumpPtrAllocator interfaces.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
AnalysisUsage & addRequired()
static uint32_t getAlignment(const MCSectionCOFF &Sec)
A description of a memory reference used in the backend.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:210
const HexagonInstrInfo * TII
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:412
static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base, MachineBasicBlock::iterator I, MachineBasicBlock::iterator E, SmallPtrSetImpl< MachineInstr *> &MemOps, SmallSet< unsigned, 4 > &MemRegs, const TargetRegisterInfo *TRI, AliasAnalysis *AA)
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
Definition: MachineInstr.h:649
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:409
Position
Position to insert a new instruction relative to an existing instruction.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:406
void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI)
Copy implicit register operands from specified instruction to this instruction.
static std::array< MachineOperand, 2 > predOps(ARMCC::CondCodes Pred, unsigned PredReg=0)
Get the operands corresponding to the given Pred value.
static ARM_AM::AMSubMode getLoadStoreMultipleSubMode(unsigned Opcode)
unsigned char getAM3Offset(unsigned AM3Opc)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
iterator getLastNonDebugInstr()
Returns an iterator to the last non-debug instruction in the basic block, or end().
unsigned getUndefRegState(bool B)
unsigned getKillRegState(bool B)
static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc, ARM_AM::AddrOpc Mode)
TargetInstrInfo - Interface to description of machine instruction set.
#define ARM_PREALLOC_LOAD_STORE_OPT_NAME
unsigned getDeadRegState(bool B)
unsigned getDefRegState(bool B)
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
MachineInstrBundleIterator< MachineInstr > iterator
unsigned char getAM5Offset(unsigned AM5Opc)
AddrOpc getAM3Op(unsigned AM3Opc)
unsigned const MachineRegisterInfo * MRI
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:516
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static bool isT2i32Load(unsigned Opc)
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:188
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:371
size_type size() const
Definition: SmallSet.h:160
Register is known to be fully dead.
Represent the analysis usage information of a pass.
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
Definition: MachineInstr.h:549
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
void setImm(int64_t immVal)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:181
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:194
ARMCC::CondCodes getInstrPredicate(const MachineInstr &MI, unsigned &PredReg)
getInstrPredicate - If instruction is predicated, returns its predicate condition, otherwise returns AL.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
size_t size() const
Definition: SmallVector.h:53
static bool isMemoryOp(const MachineInstr &MI)
Returns true if instruction is a memory operation that this pass is capable of operating on...
bool isDebugInstr() const
Definition: MachineInstr.h:999
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint64_t getAlignment() const
Return the minimum known alignment in bytes of the actual memory reference.
static cl::opt< bool > AssumeMisalignedLoadStores("arm-assume-misaligned-load-store", cl::Hidden, cl::init(false), cl::desc("Be more conservative in ARM load/store opt"))
This switch disables formation of double/multi instructions that could potentially lead to (new) alig...
FunctionPass * createARMLoadStoreOptimizationPass(bool PreAlloc=false)
Returns an instance of the load / store optimization pass.
unsigned first
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:972
static bool mayCombineMisaligned(const TargetSubtargetInfo &STI, const MachineInstr &MI)
Return true for loads/stores that can be combined to a double/multi operation without increasing the ...
static MachineBasicBlock::iterator findIncDecBefore(MachineBasicBlock::iterator MBBI, unsigned Reg, ARMCC::CondCodes Pred, unsigned PredReg, int &Offset)
Searches for an increment or decrement of Reg before MBBI.
static bool isi32Store(unsigned Opc)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
static void InsertLDR_STR(MachineBasicBlock &MBB, MachineBasicBlock::iterator &MBBI, int Offset, bool isDef, unsigned NewOpc, unsigned Reg, bool RegDeadKill, bool RegUndef, unsigned BaseReg, bool BaseKill, bool BaseUndef, ARMCC::CondCodes Pred, unsigned PredReg, const TargetInstrInfo *TII)
Basic Register Allocator
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:418
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
static MachineOperand t1CondCodeOp(bool isDead=false)
Get the operand corresponding to the conditional code result for Thumb1.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:534
bool regsOverlap(unsigned regA, unsigned regB) const
Returns true if the two registers are equal or alias each other.
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly. ...
INITIALIZE_PASS(ARMLoadStoreOpt, "arm-ldst-opt", ARM_LOAD_STORE_OPT_NAME, false, false) static bool definesCPSR(const MachineInstr &MI)
A BumpPtrAllocator that allows only elements of a specific type to be allocated.
Definition: Allocator.h:415
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
#define ARM_LOAD_STORE_OPT_NAME
unsigned getTransientStackAlignment() const
getTransientStackAlignment - This method returns the number of bytes to which the stack pointer must ...
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr reads the specified register.
iterator_range< mop_iterator > implicit_operands()
Definition: MachineInstr.h:473
static const MachineOperand & getLoadStoreBaseOp(const MachineInstr &MI)
unsigned succ_size() const
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:254
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
MachineFunctionProperties & set(Property P)
TargetSubtargetInfo - Generic base class for all target subtargets.
static bool isARMLowRegister(unsigned Reg)
isARMLowRegister - Returns true if the register is a low register (r0-r7).
Definition: ARMBaseInfo.h:161
Representation of each machine instruction.
Definition: MachineInstr.h:64
static bool isT2i32Store(unsigned Opc)
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
bool killsRegister(unsigned Reg, const TargetRegisterInfo *TRI=nullptr) const
Return true if the MachineInstr kills the specified register.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
static bool isValidLSDoubleOffset(int Offset)
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
static MachineOperand condCodeOp(unsigned CCReg=0)
Get the operand corresponding to the conditional code result.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
Definition: LivePhysRegs.h:49
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:56
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1213
bool optForMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:595
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr *> OtherMIs) const
virtual const TargetFrameLowering * getFrameLowering() const
uint32_t Size
Definition: Profile.cpp:47
static bool isLoadSingle(unsigned Opc)
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:92
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static unsigned getUpdatingLSMultipleOpcode(unsigned Opc, ARM_AM::AMSubMode Mode)
static bool isi32Load(unsigned Opc)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static unsigned getLSMultipleTransferSize(const MachineInstr *MI)
aarch64 promote const
LLVM Value Representation.
Definition: Value.h:73
static int getLoadStoreMultipleOpcode(unsigned Opcode, ARM_AM::AMSubMode Mode)
static const MachineOperand & getLoadStoreRegOp(const MachineInstr &MI)
static MachineBasicBlock::iterator findIncDecAfter(MachineBasicBlock::iterator MBBI, unsigned Reg, ARMCC::CondCodes Pred, unsigned PredReg, int &Offset)
Searches for a increment or decrement of Reg after MBBI.
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
AddrOpc getAM5Op(unsigned AM5Opc)
static bool isVolatile(Instruction *Inst)
static bool isT1i32Store(unsigned Opc)
static int getMemoryOpOffset(const MachineInstr &MI)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
#define LLVM_DEBUG(X)
Definition: Debug.h:123
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:414
static bool ContainsReg(const ArrayRef< std::pair< unsigned, bool >> &Regs, unsigned Reg)
Properties which a MachineFunction may have at a given point in time.
This file describes how to lower LLVM code to machine code.
bool isImplicit() const
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:165
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:1101