LLVM  3.7.0
ARMConstantIslandPass.cpp
Go to the documentation of this file.
1 //===-- ARMConstantIslandPass.cpp - ARM constant islands ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file contains a pass that splits the constant pool up into 'islands'
11 // which are scattered through-out the function. This is required due to the
12 // limited pc-relative displacements that ARM has.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "ARM.h"
17 #include "ARMMachineFunctionInfo.h"
19 #include "Thumb2InstrInfo.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/Statistic.h"
28 #include "llvm/IR/DataLayout.h"
30 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/Format.h"
35 #include <algorithm>
36 using namespace llvm;
37 
38 #define DEBUG_TYPE "arm-cp-islands"
39 
40 STATISTIC(NumCPEs, "Number of constpool entries");
41 STATISTIC(NumSplit, "Number of uncond branches inserted");
42 STATISTIC(NumCBrFixed, "Number of cond branches fixed");
43 STATISTIC(NumUBrFixed, "Number of uncond branches fixed");
44 STATISTIC(NumTBs, "Number of table branches generated");
45 STATISTIC(NumT2CPShrunk, "Number of Thumb2 constantpool instructions shrunk");
46 STATISTIC(NumT2BrShrunk, "Number of Thumb2 immediate branches shrunk");
47 STATISTIC(NumCBZ, "Number of CBZ / CBNZ formed");
48 STATISTIC(NumJTMoved, "Number of jump table destination blocks moved");
49 STATISTIC(NumJTInserted, "Number of jump table intermediate blocks inserted");
50 
51 
52 static cl::opt<bool>
53 AdjustJumpTableBlocks("arm-adjust-jump-tables", cl::Hidden, cl::init(true),
54  cl::desc("Adjust basic block layout to better use TB[BH]"));
55 
56 /// UnknownPadding - Return the worst case padding that could result from
57 /// unknown offset bits. This does not include alignment padding caused by
58 /// known offset bits.
59 ///
60 /// @param LogAlign log2(alignment)
61 /// @param KnownBits Number of known low offset bits.
62 static inline unsigned UnknownPadding(unsigned LogAlign, unsigned KnownBits) {
63  if (KnownBits < LogAlign)
64  return (1u << LogAlign) - (1u << KnownBits);
65  return 0;
66 }
67 
68 namespace {
69  /// ARMConstantIslands - Due to limited PC-relative displacements, ARM
70  /// requires constant pool entries to be scattered among the instructions
71  /// inside a function. To do this, it completely ignores the normal LLVM
72  /// constant pool; instead, it places constants wherever it feels like with
73  /// special instructions.
74  ///
75  /// The terminology used in this pass includes:
76  /// Islands - Clumps of constants placed in the function.
77  /// Water - Potential places where an island could be formed.
78  /// CPE - A constant pool entry that has been placed somewhere, which
79  /// tracks a list of users.
80  class ARMConstantIslands : public MachineFunctionPass {
81  /// BasicBlockInfo - Information about the offset and size of a single
82  /// basic block.
83  struct BasicBlockInfo {
84  /// Offset - Distance from the beginning of the function to the beginning
85  /// of this basic block.
86  ///
87  /// Offsets are computed assuming worst case padding before an aligned
88  /// block. This means that subtracting basic block offsets always gives a
89  /// conservative estimate of the real distance which may be smaller.
90  ///
91  /// Because worst case padding is used, the computed offset of an aligned
92  /// block may not actually be aligned.
93  unsigned Offset;
94 
95  /// Size - Size of the basic block in bytes. If the block contains
96  /// inline assembly, this is a worst case estimate.
97  ///
98  /// The size does not include any alignment padding whether from the
99  /// beginning of the block, or from an aligned jump table at the end.
100  unsigned Size;
101 
102  /// KnownBits - The number of low bits in Offset that are known to be
103  /// exact. The remaining bits of Offset are an upper bound.
104  uint8_t KnownBits;
105 
106  /// Unalign - When non-zero, the block contains instructions (inline asm)
107  /// of unknown size. The real size may be smaller than Size bytes by a
108  /// multiple of 1 << Unalign.
109  uint8_t Unalign;
110 
111  /// PostAlign - When non-zero, the block terminator contains a .align
112  /// directive, so the end of the block is aligned to 1 << PostAlign
113  /// bytes.
114  uint8_t PostAlign;
115 
116  BasicBlockInfo() : Offset(0), Size(0), KnownBits(0), Unalign(0),
117  PostAlign(0) {}
118 
119  /// Compute the number of known offset bits internally to this block.
120  /// This number should be used to predict worst case padding when
121  /// splitting the block.
122  unsigned internalKnownBits() const {
123  unsigned Bits = Unalign ? Unalign : KnownBits;
124  // If the block size isn't a multiple of the known bits, assume the
125  // worst case padding.
126  if (Size & ((1u << Bits) - 1))
127  Bits = countTrailingZeros(Size);
128  return Bits;
129  }
130 
131  /// Compute the offset immediately following this block. If LogAlign is
132  /// specified, return the offset the successor block will get if it has
133  /// this alignment.
134  unsigned postOffset(unsigned LogAlign = 0) const {
135  unsigned PO = Offset + Size;
136  unsigned LA = std::max(unsigned(PostAlign), LogAlign);
137  if (!LA)
138  return PO;
139  // Add alignment padding from the terminator.
140  return PO + UnknownPadding(LA, internalKnownBits());
141  }
142 
143  /// Compute the number of known low bits of postOffset. If this block
144  /// contains inline asm, the number of known bits drops to the
145  /// instruction alignment. An aligned terminator may increase the number
146  /// of know bits.
147  /// If LogAlign is given, also consider the alignment of the next block.
148  unsigned postKnownBits(unsigned LogAlign = 0) const {
149  return std::max(std::max(unsigned(PostAlign), LogAlign),
150  internalKnownBits());
151  }
152  };
153 
154  std::vector<BasicBlockInfo> BBInfo;
155 
156  /// WaterList - A sorted list of basic blocks where islands could be placed
157  /// (i.e. blocks that don't fall through to the following block, due
158  /// to a return, unreachable, or unconditional branch).
159  std::vector<MachineBasicBlock*> WaterList;
160 
161  /// NewWaterList - The subset of WaterList that was created since the
162  /// previous iteration by inserting unconditional branches.
163  SmallSet<MachineBasicBlock*, 4> NewWaterList;
164 
165  typedef std::vector<MachineBasicBlock*>::iterator water_iterator;
166 
167  /// CPUser - One user of a constant pool, keeping the machine instruction
168  /// pointer, the constant pool being referenced, and the max displacement
169  /// allowed from the instruction to the CP. The HighWaterMark records the
170  /// highest basic block where a new CPEntry can be placed. To ensure this
171  /// pass terminates, the CP entries are initially placed at the end of the
172  /// function and then move monotonically to lower addresses. The
173  /// exception to this rule is when the current CP entry for a particular
174  /// CPUser is out of range, but there is another CP entry for the same
175  /// constant value in range. We want to use the existing in-range CP
176  /// entry, but if it later moves out of range, the search for new water
177  /// should resume where it left off. The HighWaterMark is used to record
178  /// that point.
179  struct CPUser {
180  MachineInstr *MI;
181  MachineInstr *CPEMI;
182  MachineBasicBlock *HighWaterMark;
183  unsigned MaxDisp;
184  bool NegOk;
185  bool IsSoImm;
186  bool KnownAlignment;
187  CPUser(MachineInstr *mi, MachineInstr *cpemi, unsigned maxdisp,
188  bool neg, bool soimm)
189  : MI(mi), CPEMI(cpemi), MaxDisp(maxdisp), NegOk(neg), IsSoImm(soimm),
190  KnownAlignment(false) {
191  HighWaterMark = CPEMI->getParent();
192  }
193  /// getMaxDisp - Returns the maximum displacement supported by MI.
194  /// Correct for unknown alignment.
195  /// Conservatively subtract 2 bytes to handle weird alignment effects.
196  unsigned getMaxDisp() const {
197  return (KnownAlignment ? MaxDisp : MaxDisp - 2) - 2;
198  }
199  };
200 
201  /// CPUsers - Keep track of all of the machine instructions that use various
202  /// constant pools and their max displacement.
203  std::vector<CPUser> CPUsers;
204 
205  /// CPEntry - One per constant pool entry, keeping the machine instruction
206  /// pointer, the constpool index, and the number of CPUser's which
207  /// reference this entry.
208  struct CPEntry {
209  MachineInstr *CPEMI;
210  unsigned CPI;
211  unsigned RefCount;
212  CPEntry(MachineInstr *cpemi, unsigned cpi, unsigned rc = 0)
213  : CPEMI(cpemi), CPI(cpi), RefCount(rc) {}
214  };
215 
216  /// CPEntries - Keep track of all of the constant pool entry machine
217  /// instructions. For each original constpool index (i.e. those that existed
218  /// upon entry to this pass), it keeps a vector of entries. Original
219  /// elements are cloned as we go along; the clones are put in the vector of
220  /// the original element, but have distinct CPIs.
221  ///
222  /// The first half of CPEntries contains generic constants, the second half
223  /// contains jump tables. Use getCombinedIndex on a generic CPEMI to look up
224  /// which vector it will be in here.
225  std::vector<std::vector<CPEntry> > CPEntries;
226 
227  /// Maps a JT index to the offset in CPEntries containing copies of that
228  /// table. The equivalent map for a CONSTPOOL_ENTRY is the identity.
229  DenseMap<int, int> JumpTableEntryIndices;
230 
231  /// Maps a JT index to the LEA that actually uses the index to calculate its
232  /// base address.
233  DenseMap<int, int> JumpTableUserIndices;
234 
235  /// ImmBranch - One per immediate branch, keeping the machine instruction
236  /// pointer, conditional or unconditional, the max displacement,
237  /// and (if isCond is true) the corresponding unconditional branch
238  /// opcode.
239  struct ImmBranch {
240  MachineInstr *MI;
241  unsigned MaxDisp : 31;
242  bool isCond : 1;
243  unsigned UncondBr;
244  ImmBranch(MachineInstr *mi, unsigned maxdisp, bool cond, unsigned ubr)
245  : MI(mi), MaxDisp(maxdisp), isCond(cond), UncondBr(ubr) {}
246  };
247 
248  /// ImmBranches - Keep track of all the immediate branch instructions.
249  ///
250  std::vector<ImmBranch> ImmBranches;
251 
252  /// PushPopMIs - Keep track of all the Thumb push / pop instructions.
253  ///
255 
256  /// T2JumpTables - Keep track of all the Thumb2 jumptable instructions.
257  SmallVector<MachineInstr*, 4> T2JumpTables;
258 
259  /// HasFarJump - True if any far jump instruction has been emitted during
260  /// the branch fix up pass.
261  bool HasFarJump;
262 
263  MachineFunction *MF;
264  MachineConstantPool *MCP;
265  const ARMBaseInstrInfo *TII;
266  const ARMSubtarget *STI;
267  ARMFunctionInfo *AFI;
268  bool isThumb;
269  bool isThumb1;
270  bool isThumb2;
271  public:
272  static char ID;
273  ARMConstantIslands() : MachineFunctionPass(ID) {}
274 
275  bool runOnMachineFunction(MachineFunction &MF) override;
276 
277  const char *getPassName() const override {
278  return "ARM constant island placement and branch shortening pass";
279  }
280 
281  private:
282  void doInitialConstPlacement(std::vector<MachineInstr *> &CPEMIs);
283  void doInitialJumpTablePlacement(std::vector<MachineInstr *> &CPEMIs);
285  CPEntry *findConstPoolEntry(unsigned CPI, const MachineInstr *CPEMI);
286  unsigned getCPELogAlign(const MachineInstr *CPEMI);
287  void scanFunctionJumpTables();
288  void initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs);
289  MachineBasicBlock *splitBlockBeforeInstr(MachineInstr *MI);
290  void updateForInsertedWaterBlock(MachineBasicBlock *NewBB);
291  void adjustBBOffsetsAfter(MachineBasicBlock *BB);
292  bool decrementCPEReferenceCount(unsigned CPI, MachineInstr* CPEMI);
293  unsigned getCombinedIndex(const MachineInstr *CPEMI);
294  int findInRangeCPEntry(CPUser& U, unsigned UserOffset);
295  bool findAvailableWater(CPUser&U, unsigned UserOffset,
296  water_iterator &WaterIter);
297  void createNewWater(unsigned CPUserIndex, unsigned UserOffset,
298  MachineBasicBlock *&NewMBB);
299  bool handleConstantPoolUser(unsigned CPUserIndex);
300  void removeDeadCPEMI(MachineInstr *CPEMI);
301  bool removeUnusedCPEntries();
302  bool isCPEntryInRange(MachineInstr *MI, unsigned UserOffset,
303  MachineInstr *CPEMI, unsigned Disp, bool NegOk,
304  bool DoDump = false);
305  bool isWaterInRange(unsigned UserOffset, MachineBasicBlock *Water,
306  CPUser &U, unsigned &Growth);
307  bool isBBInRange(MachineInstr *MI, MachineBasicBlock *BB, unsigned Disp);
308  bool fixupImmediateBr(ImmBranch &Br);
309  bool fixupConditionalBr(ImmBranch &Br);
310  bool fixupUnconditionalBr(ImmBranch &Br);
311  bool undoLRSpillRestore();
312  bool mayOptimizeThumb2Instruction(const MachineInstr *MI) const;
313  bool optimizeThumb2Instructions();
314  bool optimizeThumb2Branches();
315  bool reorderThumb2JumpTables();
316  bool preserveBaseRegister(MachineInstr *JumpMI, MachineInstr *LEAMI,
317  unsigned &DeadSize, bool &CanDeleteLEA,
318  bool &BaseRegKill);
319  bool optimizeThumb2JumpTables();
320  MachineBasicBlock *adjustJTTargetBlockForward(MachineBasicBlock *BB,
321  MachineBasicBlock *JTBB);
322 
323  void computeBlockSize(MachineBasicBlock *MBB);
324  unsigned getOffsetOf(MachineInstr *MI) const;
325  unsigned getUserOffset(CPUser&) const;
326  void dumpBBs();
327  void verify();
328 
329  bool isOffsetInRange(unsigned UserOffset, unsigned TrialOffset,
330  unsigned Disp, bool NegativeOK, bool IsSoImm = false);
331  bool isOffsetInRange(unsigned UserOffset, unsigned TrialOffset,
332  const CPUser &U) {
333  return isOffsetInRange(UserOffset, TrialOffset,
334  U.getMaxDisp(), U.NegOk, U.IsSoImm);
335  }
336  };
337  char ARMConstantIslands::ID = 0;
338 }
339 
340 /// verify - check BBOffsets, BBSizes, alignment of islands
342 #ifndef NDEBUG
343  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
344  MBBI != E; ++MBBI) {
345  MachineBasicBlock *MBB = MBBI;
346  unsigned MBBId = MBB->getNumber();
347  assert(!MBBId || BBInfo[MBBId - 1].postOffset() <= BBInfo[MBBId].Offset);
348  }
349  DEBUG(dbgs() << "Verifying " << CPUsers.size() << " CP users.\n");
350  for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) {
351  CPUser &U = CPUsers[i];
352  unsigned UserOffset = getUserOffset(U);
353  // Verify offset using the real max displacement without the safety
354  // adjustment.
355  if (isCPEntryInRange(U.MI, UserOffset, U.CPEMI, U.getMaxDisp()+2, U.NegOk,
356  /* DoDump = */ true)) {
357  DEBUG(dbgs() << "OK\n");
358  continue;
359  }
360  DEBUG(dbgs() << "Out of range.\n");
361  dumpBBs();
362  DEBUG(MF->dump());
363  llvm_unreachable("Constant pool entry out of range!");
364  }
365 #endif
366 }
367 
368 /// print block size and offset information - debugging
369 void ARMConstantIslands::dumpBBs() {
370  DEBUG({
371  for (unsigned J = 0, E = BBInfo.size(); J !=E; ++J) {
372  const BasicBlockInfo &BBI = BBInfo[J];
373  dbgs() << format("%08x BB#%u\t", BBI.Offset, J)
374  << " kb=" << unsigned(BBI.KnownBits)
375  << " ua=" << unsigned(BBI.Unalign)
376  << " pa=" << unsigned(BBI.PostAlign)
377  << format(" size=%#x\n", BBInfo[J].Size);
378  }
379  });
380 }
381 
382 /// createARMConstantIslandPass - returns an instance of the constpool
383 /// island pass.
385  return new ARMConstantIslands();
386 }
387 
388 bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
389  MF = &mf;
390  MCP = mf.getConstantPool();
391 
392  DEBUG(dbgs() << "***** ARMConstantIslands: "
393  << MCP->getConstants().size() << " CP entries, aligned to "
394  << MCP->getConstantPoolAlignment() << " bytes *****\n");
395 
396  STI = &static_cast<const ARMSubtarget &>(MF->getSubtarget());
397  TII = STI->getInstrInfo();
398  AFI = MF->getInfo<ARMFunctionInfo>();
399 
400  isThumb = AFI->isThumbFunction();
401  isThumb1 = AFI->isThumb1OnlyFunction();
402  isThumb2 = AFI->isThumb2Function();
403 
404  HasFarJump = false;
405 
406  // This pass invalidates liveness information when it splits basic blocks.
407  MF->getRegInfo().invalidateLiveness();
408 
409  // Renumber all of the machine basic blocks in the function, guaranteeing that
410  // the numbers agree with the position of the block in the function.
411  MF->RenumberBlocks();
412 
413  // Try to reorder and otherwise adjust the block layout to make good use
414  // of the TB[BH] instructions.
415  bool MadeChange = false;
416  if (isThumb2 && AdjustJumpTableBlocks) {
417  scanFunctionJumpTables();
418  MadeChange |= reorderThumb2JumpTables();
419  // Data is out of date, so clear it. It'll be re-computed later.
420  T2JumpTables.clear();
421  // Blocks may have shifted around. Keep the numbering up to date.
422  MF->RenumberBlocks();
423  }
424 
425  // Perform the initial placement of the constant pool entries. To start with,
426  // we put them all at the end of the function.
427  std::vector<MachineInstr*> CPEMIs;
428  if (!MCP->isEmpty())
429  doInitialConstPlacement(CPEMIs);
430 
431  if (MF->getJumpTableInfo())
432  doInitialJumpTablePlacement(CPEMIs);
433 
434  /// The next UID to take is the first unused one.
435  AFI->initPICLabelUId(CPEMIs.size());
436 
437  // Do the initial scan of the function, building up information about the
438  // sizes of each block, the location of all the water, and finding all of the
439  // constant pool users.
440  initializeFunctionInfo(CPEMIs);
441  CPEMIs.clear();
442  DEBUG(dumpBBs());
443 
444  // Functions with jump tables need an alignment of 4 because they use the ADR
445  // instruction, which aligns the PC to 4 bytes before adding an offset.
446  if (!T2JumpTables.empty())
447  MF->ensureAlignment(2);
448 
449  /// Remove dead constant pool entries.
450  MadeChange |= removeUnusedCPEntries();
451 
452  // Iteratively place constant pool entries and fix up branches until there
453  // is no change.
454  unsigned NoCPIters = 0, NoBRIters = 0;
455  while (true) {
456  DEBUG(dbgs() << "Beginning CP iteration #" << NoCPIters << '\n');
457  bool CPChange = false;
458  for (unsigned i = 0, e = CPUsers.size(); i != e; ++i)
459  CPChange |= handleConstantPoolUser(i);
460  if (CPChange && ++NoCPIters > 30)
461  report_fatal_error("Constant Island pass failed to converge!");
462  DEBUG(dumpBBs());
463 
464  // Clear NewWaterList now. If we split a block for branches, it should
465  // appear as "new water" for the next iteration of constant pool placement.
466  NewWaterList.clear();
467 
468  DEBUG(dbgs() << "Beginning BR iteration #" << NoBRIters << '\n');
469  bool BRChange = false;
470  for (unsigned i = 0, e = ImmBranches.size(); i != e; ++i)
471  BRChange |= fixupImmediateBr(ImmBranches[i]);
472  if (BRChange && ++NoBRIters > 30)
473  report_fatal_error("Branch Fix Up pass failed to converge!");
474  DEBUG(dumpBBs());
475 
476  if (!CPChange && !BRChange)
477  break;
478  MadeChange = true;
479  }
480 
481  // Shrink 32-bit Thumb2 branch, load, and store instructions.
482  if (isThumb2 && !STI->prefers32BitThumb())
483  MadeChange |= optimizeThumb2Instructions();
484 
485  // After a while, this might be made debug-only, but it is not expensive.
486  verify();
487 
488  // If LR has been forced spilled and no far jump (i.e. BL) has been issued,
489  // undo the spill / restore of LR if possible.
490  if (isThumb && !HasFarJump && AFI->isLRSpilledForFarJump())
491  MadeChange |= undoLRSpillRestore();
492 
493  // Save the mapping between original and cloned constpool entries.
494  for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) {
495  for (unsigned j = 0, je = CPEntries[i].size(); j != je; ++j) {
496  const CPEntry & CPE = CPEntries[i][j];
497  if (CPE.CPEMI && CPE.CPEMI->getOperand(1).isCPI())
498  AFI->recordCPEClone(i, CPE.CPI);
499  }
500  }
501 
502  DEBUG(dbgs() << '\n'; dumpBBs());
503 
504  BBInfo.clear();
505  WaterList.clear();
506  CPUsers.clear();
507  CPEntries.clear();
508  JumpTableEntryIndices.clear();
509  JumpTableUserIndices.clear();
510  ImmBranches.clear();
511  PushPopMIs.clear();
512  T2JumpTables.clear();
513 
514  return MadeChange;
515 }
516 
517 /// \brief Perform the initial placement of the regular constant pool entries.
518 /// To start with, we put them all at the end of the function.
519 void
520 ARMConstantIslands::doInitialConstPlacement(std::vector<MachineInstr*> &CPEMIs) {
521  // Create the basic block to hold the CPE's.
522  MachineBasicBlock *BB = MF->CreateMachineBasicBlock();
523  MF->push_back(BB);
524 
525  // MachineConstantPool measures alignment in bytes. We measure in log2(bytes).
526  unsigned MaxAlign = Log2_32(MCP->getConstantPoolAlignment());
527 
528  // Mark the basic block as required by the const-pool.
529  BB->setAlignment(MaxAlign);
530 
531  // The function needs to be as aligned as the basic blocks. The linker may
532  // move functions around based on their alignment.
533  MF->ensureAlignment(BB->getAlignment());
534 
535  // Order the entries in BB by descending alignment. That ensures correct
536  // alignment of all entries as long as BB is sufficiently aligned. Keep
537  // track of the insertion point for each alignment. We are going to bucket
538  // sort the entries as they are created.
539  SmallVector<MachineBasicBlock::iterator, 8> InsPoint(MaxAlign + 1, BB->end());
540 
541  // Add all of the constants from the constant pool to the end block, use an
542  // identity mapping of CPI's to CPE's.
543  const std::vector<MachineConstantPoolEntry> &CPs = MCP->getConstants();
544 
545  const DataLayout &TD = *MF->getTarget().getDataLayout();
546  for (unsigned i = 0, e = CPs.size(); i != e; ++i) {
547  unsigned Size = TD.getTypeAllocSize(CPs[i].getType());
548  assert(Size >= 4 && "Too small constant pool entry");
549  unsigned Align = CPs[i].getAlignment();
550  assert(isPowerOf2_32(Align) && "Invalid alignment");
551  // Verify that all constant pool entries are a multiple of their alignment.
552  // If not, we would have to pad them out so that instructions stay aligned.
553  assert((Size % Align) == 0 && "CP Entry not multiple of 4 bytes!");
554 
555  // Insert CONSTPOOL_ENTRY before entries with a smaller alignment.
556  unsigned LogAlign = Log2_32(Align);
557  MachineBasicBlock::iterator InsAt = InsPoint[LogAlign];
558  MachineInstr *CPEMI =
559  BuildMI(*BB, InsAt, DebugLoc(), TII->get(ARM::CONSTPOOL_ENTRY))
560  .addImm(i).addConstantPoolIndex(i).addImm(Size);
561  CPEMIs.push_back(CPEMI);
562 
563  // Ensure that future entries with higher alignment get inserted before
564  // CPEMI. This is bucket sort with iterators.
565  for (unsigned a = LogAlign + 1; a <= MaxAlign; ++a)
566  if (InsPoint[a] == InsAt)
567  InsPoint[a] = CPEMI;
568 
569  // Add a new CPEntry, but no corresponding CPUser yet.
570  CPEntries.emplace_back(1, CPEntry(CPEMI, i));
571  ++NumCPEs;
572  DEBUG(dbgs() << "Moved CPI#" << i << " to end of function, size = "
573  << Size << ", align = " << Align <<'\n');
574  }
575  DEBUG(BB->dump());
576 }
577 
578 /// \brief Do initial placement of the jump tables. Because Thumb2's TBB and TBH
579 /// instructions can be made more efficient if the jump table immediately
580 /// follows the instruction, it's best to place them immediately next to their
581 /// jumps to begin with. In almost all cases they'll never be moved from that
582 /// position.
583 void ARMConstantIslands::doInitialJumpTablePlacement(
584  std::vector<MachineInstr *> &CPEMIs) {
585  unsigned i = CPEntries.size();
586  auto MJTI = MF->getJumpTableInfo();
587  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
588 
589  MachineBasicBlock *LastCorrectlyNumberedBB = nullptr;
590  for (MachineBasicBlock &MBB : *MF) {
591  auto MI = MBB.getLastNonDebugInstr();
592 
593  unsigned JTOpcode;
594  switch (MI->getOpcode()) {
595  default:
596  continue;
597  case ARM::BR_JTadd:
598  case ARM::BR_JTr:
599  case ARM::tBR_JTr:
600  case ARM::BR_JTm:
601  JTOpcode = ARM::JUMPTABLE_ADDRS;
602  break;
603  case ARM::t2BR_JT:
604  JTOpcode = ARM::JUMPTABLE_INSTS;
605  break;
606  case ARM::t2TBB_JT:
607  JTOpcode = ARM::JUMPTABLE_TBB;
608  break;
609  case ARM::t2TBH_JT:
610  JTOpcode = ARM::JUMPTABLE_TBH;
611  break;
612  }
613 
614  unsigned NumOps = MI->getDesc().getNumOperands();
615  MachineOperand JTOp =
616  MI->getOperand(NumOps - (MI->isPredicable() ? 2 : 1));
617  unsigned JTI = JTOp.getIndex();
618  unsigned Size = JT[JTI].MBBs.size() * sizeof(uint32_t);
619  MachineBasicBlock *JumpTableBB = MF->CreateMachineBasicBlock();
620  MF->insert(std::next(MachineFunction::iterator(MBB)), JumpTableBB);
621  MachineInstr *CPEMI = BuildMI(*JumpTableBB, JumpTableBB->begin(),
622  DebugLoc(), TII->get(JTOpcode))
623  .addImm(i++)
624  .addJumpTableIndex(JTI)
625  .addImm(Size);
626  CPEMIs.push_back(CPEMI);
627  CPEntries.emplace_back(1, CPEntry(CPEMI, JTI));
628  JumpTableEntryIndices.insert(std::make_pair(JTI, CPEntries.size() - 1));
629  if (!LastCorrectlyNumberedBB)
630  LastCorrectlyNumberedBB = &MBB;
631  }
632 
633  // If we did anything then we need to renumber the subsequent blocks.
634  if (LastCorrectlyNumberedBB)
635  MF->RenumberBlocks(LastCorrectlyNumberedBB);
636 }
637 
638 /// BBHasFallthrough - Return true if the specified basic block can fallthrough
639 /// into the block immediately after it.
641  // Get the next machine basic block in the function.
642  MachineFunction::iterator MBBI = MBB;
643  // Can't fall off end of function.
644  if (std::next(MBBI) == MBB->getParent()->end())
645  return false;
646 
647  MachineBasicBlock *NextBB = std::next(MBBI);
648  if (std::find(MBB->succ_begin(), MBB->succ_end(), NextBB) == MBB->succ_end())
649  return false;
650 
651  // Try to analyze the end of the block. A potential fallthrough may already
652  // have an unconditional branch for whatever reason.
653  MachineBasicBlock *TBB, *FBB;
655  bool TooDifficult = TII->AnalyzeBranch(*MBB, TBB, FBB, Cond);
656  return TooDifficult || FBB == nullptr;
657 }
658 
659 /// findConstPoolEntry - Given the constpool index and CONSTPOOL_ENTRY MI,
660 /// look up the corresponding CPEntry.
661 ARMConstantIslands::CPEntry
662 *ARMConstantIslands::findConstPoolEntry(unsigned CPI,
663  const MachineInstr *CPEMI) {
664  std::vector<CPEntry> &CPEs = CPEntries[CPI];
665  // Number of entries per constpool index should be small, just do a
666  // linear search.
667  for (unsigned i = 0, e = CPEs.size(); i != e; ++i) {
668  if (CPEs[i].CPEMI == CPEMI)
669  return &CPEs[i];
670  }
671  return nullptr;
672 }
673 
674 /// getCPELogAlign - Returns the required alignment of the constant pool entry
675 /// represented by CPEMI. Alignment is measured in log2(bytes) units.
676 unsigned ARMConstantIslands::getCPELogAlign(const MachineInstr *CPEMI) {
677  switch (CPEMI->getOpcode()) {
678  case ARM::CONSTPOOL_ENTRY:
679  break;
680  case ARM::JUMPTABLE_TBB:
681  return 0;
682  case ARM::JUMPTABLE_TBH:
683  case ARM::JUMPTABLE_INSTS:
684  return 1;
685  case ARM::JUMPTABLE_ADDRS:
686  return 2;
687  default:
688  llvm_unreachable("unknown constpool entry kind");
689  }
690 
691  unsigned CPI = getCombinedIndex(CPEMI);
692  assert(CPI < MCP->getConstants().size() && "Invalid constant pool index.");
693  unsigned Align = MCP->getConstants()[CPI].getAlignment();
694  assert(isPowerOf2_32(Align) && "Invalid CPE alignment");
695  return Log2_32(Align);
696 }
697 
698 /// scanFunctionJumpTables - Do a scan of the function, building up
699 /// information about the sizes of each block and the locations of all
700 /// the jump tables.
701 void ARMConstantIslands::scanFunctionJumpTables() {
702  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
703  MBBI != E; ++MBBI) {
704  MachineBasicBlock &MBB = *MBBI;
705 
706  for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
707  I != E; ++I)
708  if (I->isBranch() && I->getOpcode() == ARM::t2BR_JT)
709  T2JumpTables.push_back(I);
710  }
711 }
712 
713 /// initializeFunctionInfo - Do the initial scan of the function, building up
714 /// information about the sizes of each block, the location of all the water,
715 /// and finding all of the constant pool users.
716 void ARMConstantIslands::
717 initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
718  BBInfo.clear();
719  BBInfo.resize(MF->getNumBlockIDs());
720 
721  // First thing, compute the size of all basic blocks, and see if the function
722  // has any inline assembly in it. If so, we have to be conservative about
723  // alignment assumptions, as we don't know for sure the size of any
724  // instructions in the inline assembly.
725  for (MachineFunction::iterator I = MF->begin(), E = MF->end(); I != E; ++I)
726  computeBlockSize(I);
727 
728  // The known bits of the entry block offset are determined by the function
729  // alignment.
730  BBInfo.front().KnownBits = MF->getAlignment();
731 
732  // Compute block offsets and known bits.
733  adjustBBOffsetsAfter(MF->begin());
734 
735  // Now go back through the instructions and build up our data structures.
736  for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end();
737  MBBI != E; ++MBBI) {
738  MachineBasicBlock &MBB = *MBBI;
739 
740  // If this block doesn't fall through into the next MBB, then this is
741  // 'water' that a constant pool island could be placed.
742  if (!BBHasFallthrough(&MBB))
743  WaterList.push_back(&MBB);
744 
745  for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
746  I != E; ++I) {
747  if (I->isDebugValue())
748  continue;
749 
750  unsigned Opc = I->getOpcode();
751  if (I->isBranch()) {
752  bool isCond = false;
753  unsigned Bits = 0;
754  unsigned Scale = 1;
755  int UOpc = Opc;
756  switch (Opc) {
757  default:
758  continue; // Ignore other JT branches
759  case ARM::t2BR_JT:
760  T2JumpTables.push_back(I);
761  continue; // Does not get an entry in ImmBranches
762  case ARM::Bcc:
763  isCond = true;
764  UOpc = ARM::B;
765  // Fallthrough
766  case ARM::B:
767  Bits = 24;
768  Scale = 4;
769  break;
770  case ARM::tBcc:
771  isCond = true;
772  UOpc = ARM::tB;
773  Bits = 8;
774  Scale = 2;
775  break;
776  case ARM::tB:
777  Bits = 11;
778  Scale = 2;
779  break;
780  case ARM::t2Bcc:
781  isCond = true;
782  UOpc = ARM::t2B;
783  Bits = 20;
784  Scale = 2;
785  break;
786  case ARM::t2B:
787  Bits = 24;
788  Scale = 2;
789  break;
790  }
791 
792  // Record this immediate branch.
793  unsigned MaxOffs = ((1 << (Bits-1))-1) * Scale;
794  ImmBranches.push_back(ImmBranch(I, MaxOffs, isCond, UOpc));
795  }
796 
797  if (Opc == ARM::tPUSH || Opc == ARM::tPOP_RET)
798  PushPopMIs.push_back(I);
799 
800  if (Opc == ARM::CONSTPOOL_ENTRY || Opc == ARM::JUMPTABLE_ADDRS ||
801  Opc == ARM::JUMPTABLE_INSTS || Opc == ARM::JUMPTABLE_TBB ||
802  Opc == ARM::JUMPTABLE_TBH)
803  continue;
804 
805  // Scan the instructions for constant pool operands.
806  for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op)
807  if (I->getOperand(op).isCPI() || I->getOperand(op).isJTI()) {
808  // We found one. The addressing mode tells us the max displacement
809  // from the PC that this instruction permits.
810 
811  // Basic size info comes from the TSFlags field.
812  unsigned Bits = 0;
813  unsigned Scale = 1;
814  bool NegOk = false;
815  bool IsSoImm = false;
816 
817  switch (Opc) {
818  default:
819  llvm_unreachable("Unknown addressing mode for CP reference!");
820 
821  // Taking the address of a CP entry.
822  case ARM::LEApcrel:
823  case ARM::LEApcrelJT:
824  // This takes a SoImm, which is 8 bit immediate rotated. We'll
825  // pretend the maximum offset is 255 * 4. Since each instruction
826  // 4 byte wide, this is always correct. We'll check for other
827  // displacements that fits in a SoImm as well.
828  Bits = 8;
829  Scale = 4;
830  NegOk = true;
831  IsSoImm = true;
832  break;
833  case ARM::t2LEApcrel:
834  case ARM::t2LEApcrelJT:
835  Bits = 12;
836  NegOk = true;
837  break;
838  case ARM::tLEApcrel:
839  case ARM::tLEApcrelJT:
840  Bits = 8;
841  Scale = 4;
842  break;
843 
844  case ARM::LDRBi12:
845  case ARM::LDRi12:
846  case ARM::LDRcp:
847  case ARM::t2LDRpci:
848  Bits = 12; // +-offset_12
849  NegOk = true;
850  break;
851 
852  case ARM::tLDRpci:
853  Bits = 8;
854  Scale = 4; // +(offset_8*4)
855  break;
856 
857  case ARM::VLDRD:
858  case ARM::VLDRS:
859  Bits = 8;
860  Scale = 4; // +-(offset_8*4)
861  NegOk = true;
862  break;
863  }
864 
865  // Remember that this is a user of a CP entry.
866  unsigned CPI = I->getOperand(op).getIndex();
867  if (I->getOperand(op).isJTI()) {
868  JumpTableUserIndices.insert(std::make_pair(CPI, CPUsers.size()));
869  CPI = JumpTableEntryIndices[CPI];
870  }
871 
872  MachineInstr *CPEMI = CPEMIs[CPI];
873  unsigned MaxOffs = ((1 << Bits)-1) * Scale;
874  CPUsers.push_back(CPUser(I, CPEMI, MaxOffs, NegOk, IsSoImm));
875 
876  // Increment corresponding CPEntry reference count.
877  CPEntry *CPE = findConstPoolEntry(CPI, CPEMI);
878  assert(CPE && "Cannot find a corresponding CPEntry!");
879  CPE->RefCount++;
880 
881  // Instructions can only use one CP entry, don't bother scanning the
882  // rest of the operands.
883  break;
884  }
885  }
886  }
887 }
888 
889 /// computeBlockSize - Compute the size and some alignment information for MBB.
890 /// This function updates BBInfo directly.
891 void ARMConstantIslands::computeBlockSize(MachineBasicBlock *MBB) {
892  BasicBlockInfo &BBI = BBInfo[MBB->getNumber()];
893  BBI.Size = 0;
894  BBI.Unalign = 0;
895  BBI.PostAlign = 0;
896 
897  for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
898  ++I) {
899  BBI.Size += TII->GetInstSizeInBytes(I);
900  // For inline asm, GetInstSizeInBytes returns a conservative estimate.
901  // The actual size may be smaller, but still a multiple of the instr size.
902  if (I->isInlineAsm())
903  BBI.Unalign = isThumb ? 1 : 2;
904  // Also consider instructions that may be shrunk later.
905  else if (isThumb && mayOptimizeThumb2Instruction(I))
906  BBI.Unalign = 1;
907  }
908 
909  // tBR_JTr contains a .align 2 directive.
910  if (!MBB->empty() && MBB->back().getOpcode() == ARM::tBR_JTr) {
911  BBI.PostAlign = 2;
912  MBB->getParent()->ensureAlignment(2);
913  }
914 }
915 
916 /// getOffsetOf - Return the current offset of the specified machine instruction
917 /// from the start of the function. This offset changes as stuff is moved
918 /// around inside the function.
919 unsigned ARMConstantIslands::getOffsetOf(MachineInstr *MI) const {
920  MachineBasicBlock *MBB = MI->getParent();
921 
922  // The offset is composed of two things: the sum of the sizes of all MBB's
923  // before this instruction's block, and the offset from the start of the block
924  // it is in.
925  unsigned Offset = BBInfo[MBB->getNumber()].Offset;
926 
927  // Sum instructions before MI in MBB.
928  for (MachineBasicBlock::iterator I = MBB->begin(); &*I != MI; ++I) {
929  assert(I != MBB->end() && "Didn't find MI in its own basic block?");
930  Offset += TII->GetInstSizeInBytes(I);
931  }
932  return Offset;
933 }
934 
935 /// CompareMBBNumbers - Little predicate function to sort the WaterList by MBB
936 /// ID.
937 static bool CompareMBBNumbers(const MachineBasicBlock *LHS,
938  const MachineBasicBlock *RHS) {
939  return LHS->getNumber() < RHS->getNumber();
940 }
941 
942 /// updateForInsertedWaterBlock - When a block is newly inserted into the
943 /// machine function, it upsets all of the block numbers. Renumber the blocks
944 /// and update the arrays that parallel this numbering.
945 void ARMConstantIslands::updateForInsertedWaterBlock(MachineBasicBlock *NewBB) {
946  // Renumber the MBB's to keep them consecutive.
947  NewBB->getParent()->RenumberBlocks(NewBB);
948 
949  // Insert an entry into BBInfo to align it properly with the (newly
950  // renumbered) block numbers.
951  BBInfo.insert(BBInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
952 
953  // Next, update WaterList. Specifically, we need to add NewMBB as having
954  // available water after it.
955  water_iterator IP =
956  std::lower_bound(WaterList.begin(), WaterList.end(), NewBB,
958  WaterList.insert(IP, NewBB);
959 }
960 
961 
962 /// Split the basic block containing MI into two blocks, which are joined by
963 /// an unconditional branch. Update data structures and renumber blocks to
964 /// account for this change and returns the newly created block.
965 MachineBasicBlock *ARMConstantIslands::splitBlockBeforeInstr(MachineInstr *MI) {
966  MachineBasicBlock *OrigBB = MI->getParent();
967 
968  // Create a new MBB for the code after the OrigBB.
969  MachineBasicBlock *NewBB =
970  MF->CreateMachineBasicBlock(OrigBB->getBasicBlock());
971  MachineFunction::iterator MBBI = OrigBB; ++MBBI;
972  MF->insert(MBBI, NewBB);
973 
974  // Splice the instructions starting with MI over to NewBB.
975  NewBB->splice(NewBB->end(), OrigBB, MI, OrigBB->end());
976 
977  // Add an unconditional branch from OrigBB to NewBB.
978  // Note the new unconditional branch is not being recorded.
979  // There doesn't seem to be meaningful DebugInfo available; this doesn't
980  // correspond to anything in the source.
981  unsigned Opc = isThumb ? (isThumb2 ? ARM::t2B : ARM::tB) : ARM::B;
982  if (!isThumb)
983  BuildMI(OrigBB, DebugLoc(), TII->get(Opc)).addMBB(NewBB);
984  else
985  BuildMI(OrigBB, DebugLoc(), TII->get(Opc)).addMBB(NewBB)
986  .addImm(ARMCC::AL).addReg(0);
987  ++NumSplit;
988 
989  // Update the CFG. All succs of OrigBB are now succs of NewBB.
990  NewBB->transferSuccessors(OrigBB);
991 
992  // OrigBB branches to NewBB.
993  OrigBB->addSuccessor(NewBB);
994 
995  // Update internal data structures to account for the newly inserted MBB.
996  // This is almost the same as updateForInsertedWaterBlock, except that
997  // the Water goes after OrigBB, not NewBB.
998  MF->RenumberBlocks(NewBB);
999 
1000  // Insert an entry into BBInfo to align it properly with the (newly
1001  // renumbered) block numbers.
1002  BBInfo.insert(BBInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
1003 
1004  // Next, update WaterList. Specifically, we need to add OrigMBB as having
1005  // available water after it (but not if it's already there, which happens
1006  // when splitting before a conditional branch that is followed by an
1007  // unconditional branch - in that case we want to insert NewBB).
1008  water_iterator IP =
1009  std::lower_bound(WaterList.begin(), WaterList.end(), OrigBB,
1011  MachineBasicBlock* WaterBB = *IP;
1012  if (WaterBB == OrigBB)
1013  WaterList.insert(std::next(IP), NewBB);
1014  else
1015  WaterList.insert(IP, OrigBB);
1016  NewWaterList.insert(OrigBB);
1017 
1018  // Figure out how large the OrigBB is. As the first half of the original
1019  // block, it cannot contain a tablejump. The size includes
1020  // the new jump we added. (It should be possible to do this without
1021  // recounting everything, but it's very confusing, and this is rarely
1022  // executed.)
1023  computeBlockSize(OrigBB);
1024 
1025  // Figure out how large the NewMBB is. As the second half of the original
1026  // block, it may contain a tablejump.
1027  computeBlockSize(NewBB);
1028 
1029  // All BBOffsets following these blocks must be modified.
1030  adjustBBOffsetsAfter(OrigBB);
1031 
1032  return NewBB;
1033 }
1034 
1035 /// getUserOffset - Compute the offset of U.MI as seen by the hardware
1036 /// displacement computation. Update U.KnownAlignment to match its current
1037 /// basic block location.
1038 unsigned ARMConstantIslands::getUserOffset(CPUser &U) const {
1039  unsigned UserOffset = getOffsetOf(U.MI);
1040  const BasicBlockInfo &BBI = BBInfo[U.MI->getParent()->getNumber()];
1041  unsigned KnownBits = BBI.internalKnownBits();
1042 
1043  // The value read from PC is offset from the actual instruction address.
1044  UserOffset += (isThumb ? 4 : 8);
1045 
1046  // Because of inline assembly, we may not know the alignment (mod 4) of U.MI.
1047  // Make sure U.getMaxDisp() returns a constrained range.
1048  U.KnownAlignment = (KnownBits >= 2);
1049 
1050  // On Thumb, offsets==2 mod 4 are rounded down by the hardware for
1051  // purposes of the displacement computation; compensate for that here.
1052  // For unknown alignments, getMaxDisp() constrains the range instead.
1053  if (isThumb && U.KnownAlignment)
1054  UserOffset &= ~3u;
1055 
1056  return UserOffset;
1057 }
1058 
1059 /// isOffsetInRange - Checks whether UserOffset (the location of a constant pool
1060 /// reference) is within MaxDisp of TrialOffset (a proposed location of a
1061 /// constant pool entry).
1062 /// UserOffset is computed by getUserOffset above to include PC adjustments. If
1063 /// the mod 4 alignment of UserOffset is not known, the uncertainty must be
1064 /// subtracted from MaxDisp instead. CPUser::getMaxDisp() does that.
1065 bool ARMConstantIslands::isOffsetInRange(unsigned UserOffset,
1066  unsigned TrialOffset, unsigned MaxDisp,
1067  bool NegativeOK, bool IsSoImm) {
1068  if (UserOffset <= TrialOffset) {
1069  // User before the Trial.
1070  if (TrialOffset - UserOffset <= MaxDisp)
1071  return true;
1072  // FIXME: Make use full range of soimm values.
1073  } else if (NegativeOK) {
1074  if (UserOffset - TrialOffset <= MaxDisp)
1075  return true;
1076  // FIXME: Make use full range of soimm values.
1077  }
1078  return false;
1079 }
1080 
1081 /// isWaterInRange - Returns true if a CPE placed after the specified
1082 /// Water (a basic block) will be in range for the specific MI.
1083 ///
1084 /// Compute how much the function will grow by inserting a CPE after Water.
1085 bool ARMConstantIslands::isWaterInRange(unsigned UserOffset,
1086  MachineBasicBlock* Water, CPUser &U,
1087  unsigned &Growth) {
1088  unsigned CPELogAlign = getCPELogAlign(U.CPEMI);
1089  unsigned CPEOffset = BBInfo[Water->getNumber()].postOffset(CPELogAlign);
1090  unsigned NextBlockOffset, NextBlockAlignment;
1091  MachineFunction::const_iterator NextBlock = Water;
1092  if (++NextBlock == MF->end()) {
1093  NextBlockOffset = BBInfo[Water->getNumber()].postOffset();
1094  NextBlockAlignment = 0;
1095  } else {
1096  NextBlockOffset = BBInfo[NextBlock->getNumber()].Offset;
1097  NextBlockAlignment = NextBlock->getAlignment();
1098  }
1099  unsigned Size = U.CPEMI->getOperand(2).getImm();
1100  unsigned CPEEnd = CPEOffset + Size;
1101 
1102  // The CPE may be able to hide in the alignment padding before the next
1103  // block. It may also cause more padding to be required if it is more aligned
1104  // that the next block.
1105  if (CPEEnd > NextBlockOffset) {
1106  Growth = CPEEnd - NextBlockOffset;
1107  // Compute the padding that would go at the end of the CPE to align the next
1108  // block.
1109  Growth += OffsetToAlignment(CPEEnd, 1u << NextBlockAlignment);
1110 
1111  // If the CPE is to be inserted before the instruction, that will raise
1112  // the offset of the instruction. Also account for unknown alignment padding
1113  // in blocks between CPE and the user.
1114  if (CPEOffset < UserOffset)
1115  UserOffset += Growth + UnknownPadding(MF->getAlignment(), CPELogAlign);
1116  } else
1117  // CPE fits in existing padding.
1118  Growth = 0;
1119 
1120  return isOffsetInRange(UserOffset, CPEOffset, U);
1121 }
1122 
1123 /// isCPEntryInRange - Returns true if the distance between specific MI and
1124 /// specific ConstPool entry instruction can fit in MI's displacement field.
1125 bool ARMConstantIslands::isCPEntryInRange(MachineInstr *MI, unsigned UserOffset,
1126  MachineInstr *CPEMI, unsigned MaxDisp,
1127  bool NegOk, bool DoDump) {
1128  unsigned CPEOffset = getOffsetOf(CPEMI);
1129 
1130  if (DoDump) {
1131  DEBUG({
1132  unsigned Block = MI->getParent()->getNumber();
1133  const BasicBlockInfo &BBI = BBInfo[Block];
1134  dbgs() << "User of CPE#" << CPEMI->getOperand(0).getImm()
1135  << " max delta=" << MaxDisp
1136  << format(" insn address=%#x", UserOffset)
1137  << " in BB#" << Block << ": "
1138  << format("%#x-%x\t", BBI.Offset, BBI.postOffset()) << *MI
1139  << format("CPE address=%#x offset=%+d: ", CPEOffset,
1140  int(CPEOffset-UserOffset));
1141  });
1142  }
1143 
1144  return isOffsetInRange(UserOffset, CPEOffset, MaxDisp, NegOk);
1145 }
1146 
1147 #ifndef NDEBUG
1148 /// BBIsJumpedOver - Return true of the specified basic block's only predecessor
1149 /// unconditionally branches to its only successor.
1151  if (MBB->pred_size() != 1 || MBB->succ_size() != 1)
1152  return false;
1153 
1154  MachineBasicBlock *Succ = *MBB->succ_begin();
1155  MachineBasicBlock *Pred = *MBB->pred_begin();
1156  MachineInstr *PredMI = &Pred->back();
1157  if (PredMI->getOpcode() == ARM::B || PredMI->getOpcode() == ARM::tB
1158  || PredMI->getOpcode() == ARM::t2B)
1159  return PredMI->getOperand(0).getMBB() == Succ;
1160  return false;
1161 }
1162 #endif // NDEBUG
1163 
1164 void ARMConstantIslands::adjustBBOffsetsAfter(MachineBasicBlock *BB) {
1165  unsigned BBNum = BB->getNumber();
1166  for(unsigned i = BBNum + 1, e = MF->getNumBlockIDs(); i < e; ++i) {
1167  // Get the offset and known bits at the end of the layout predecessor.
1168  // Include the alignment of the current block.
1169  unsigned LogAlign = MF->getBlockNumbered(i)->getAlignment();
1170  unsigned Offset = BBInfo[i - 1].postOffset(LogAlign);
1171  unsigned KnownBits = BBInfo[i - 1].postKnownBits(LogAlign);
1172 
1173  // This is where block i begins. Stop if the offset is already correct,
1174  // and we have updated 2 blocks. This is the maximum number of blocks
1175  // changed before calling this function.
1176  if (i > BBNum + 2 &&
1177  BBInfo[i].Offset == Offset &&
1178  BBInfo[i].KnownBits == KnownBits)
1179  break;
1180 
1181  BBInfo[i].Offset = Offset;
1182  BBInfo[i].KnownBits = KnownBits;
1183  }
1184 }
1185 
1186 /// decrementCPEReferenceCount - find the constant pool entry with index CPI
1187 /// and instruction CPEMI, and decrement its refcount. If the refcount
1188 /// becomes 0 remove the entry and instruction. Returns true if we removed
1189 /// the entry, false if we didn't.
1190 
1191 bool ARMConstantIslands::decrementCPEReferenceCount(unsigned CPI,
1192  MachineInstr *CPEMI) {
1193  // Find the old entry. Eliminate it if it is no longer used.
1194  CPEntry *CPE = findConstPoolEntry(CPI, CPEMI);
1195  assert(CPE && "Unexpected!");
1196  if (--CPE->RefCount == 0) {
1197  removeDeadCPEMI(CPEMI);
1198  CPE->CPEMI = nullptr;
1199  --NumCPEs;
1200  return true;
1201  }
1202  return false;
1203 }
1204 
1205 unsigned ARMConstantIslands::getCombinedIndex(const MachineInstr *CPEMI) {
1206  if (CPEMI->getOperand(1).isCPI())
1207  return CPEMI->getOperand(1).getIndex();
1208 
1209  return JumpTableEntryIndices[CPEMI->getOperand(1).getIndex()];
1210 }
1211 
1212 /// LookForCPEntryInRange - see if the currently referenced CPE is in range;
1213 /// if not, see if an in-range clone of the CPE is in range, and if so,
1214 /// change the data structures so the user references the clone. Returns:
1215 /// 0 = no existing entry found
1216 /// 1 = entry found, and there were no code insertions or deletions
1217 /// 2 = entry found, and there were code insertions or deletions
1218 int ARMConstantIslands::findInRangeCPEntry(CPUser& U, unsigned UserOffset)
1219 {
1220  MachineInstr *UserMI = U.MI;
1221  MachineInstr *CPEMI = U.CPEMI;
1222 
1223  // Check to see if the CPE is already in-range.
1224  if (isCPEntryInRange(UserMI, UserOffset, CPEMI, U.getMaxDisp(), U.NegOk,
1225  true)) {
1226  DEBUG(dbgs() << "In range\n");
1227  return 1;
1228  }
1229 
1230  // No. Look for previously created clones of the CPE that are in range.
1231  unsigned CPI = getCombinedIndex(CPEMI);
1232  std::vector<CPEntry> &CPEs = CPEntries[CPI];
1233  for (unsigned i = 0, e = CPEs.size(); i != e; ++i) {
1234  // We already tried this one
1235  if (CPEs[i].CPEMI == CPEMI)
1236  continue;
1237  // Removing CPEs can leave empty entries, skip
1238  if (CPEs[i].CPEMI == nullptr)
1239  continue;
1240  if (isCPEntryInRange(UserMI, UserOffset, CPEs[i].CPEMI, U.getMaxDisp(),
1241  U.NegOk)) {
1242  DEBUG(dbgs() << "Replacing CPE#" << CPI << " with CPE#"
1243  << CPEs[i].CPI << "\n");
1244  // Point the CPUser node to the replacement
1245  U.CPEMI = CPEs[i].CPEMI;
1246  // Change the CPI in the instruction operand to refer to the clone.
1247  for (unsigned j = 0, e = UserMI->getNumOperands(); j != e; ++j)
1248  if (UserMI->getOperand(j).isCPI()) {
1249  UserMI->getOperand(j).setIndex(CPEs[i].CPI);
1250  break;
1251  }
1252  // Adjust the refcount of the clone...
1253  CPEs[i].RefCount++;
1254  // ...and the original. If we didn't remove the old entry, none of the
1255  // addresses changed, so we don't need another pass.
1256  return decrementCPEReferenceCount(CPI, CPEMI) ? 2 : 1;
1257  }
1258  }
1259  return 0;
1260 }
1261 
1262 /// getUnconditionalBrDisp - Returns the maximum displacement that can fit in
1263 /// the specific unconditional branch instruction.
1264 static inline unsigned getUnconditionalBrDisp(int Opc) {
1265  switch (Opc) {
1266  case ARM::tB:
1267  return ((1<<10)-1)*2;
1268  case ARM::t2B:
1269  return ((1<<23)-1)*2;
1270  default:
1271  break;
1272  }
1273 
1274  return ((1<<23)-1)*4;
1275 }
1276 
1277 /// findAvailableWater - Look for an existing entry in the WaterList in which
1278 /// we can place the CPE referenced from U so it's within range of U's MI.
1279 /// Returns true if found, false if not. If it returns true, WaterIter
1280 /// is set to the WaterList entry. For Thumb, prefer water that will not
1281 /// introduce padding to water that will. To ensure that this pass
1282 /// terminates, the CPE location for a particular CPUser is only allowed to
1283 /// move to a lower address, so search backward from the end of the list and
1284 /// prefer the first water that is in range.
1285 bool ARMConstantIslands::findAvailableWater(CPUser &U, unsigned UserOffset,
1286  water_iterator &WaterIter) {
1287  if (WaterList.empty())
1288  return false;
1289 
1290  unsigned BestGrowth = ~0u;
1291  for (water_iterator IP = std::prev(WaterList.end()), B = WaterList.begin();;
1292  --IP) {
1293  MachineBasicBlock* WaterBB = *IP;
1294  // Check if water is in range and is either at a lower address than the
1295  // current "high water mark" or a new water block that was created since
1296  // the previous iteration by inserting an unconditional branch. In the
1297  // latter case, we want to allow resetting the high water mark back to
1298  // this new water since we haven't seen it before. Inserting branches
1299  // should be relatively uncommon and when it does happen, we want to be
1300  // sure to take advantage of it for all the CPEs near that block, so that
1301  // we don't insert more branches than necessary.
1302  unsigned Growth;
1303  if (isWaterInRange(UserOffset, WaterBB, U, Growth) &&
1304  (WaterBB->getNumber() < U.HighWaterMark->getNumber() ||
1305  NewWaterList.count(WaterBB) || WaterBB == U.MI->getParent()) &&
1306  Growth < BestGrowth) {
1307  // This is the least amount of required padding seen so far.
1308  BestGrowth = Growth;
1309  WaterIter = IP;
1310  DEBUG(dbgs() << "Found water after BB#" << WaterBB->getNumber()
1311  << " Growth=" << Growth << '\n');
1312 
1313  // Keep looking unless it is perfect.
1314  if (BestGrowth == 0)
1315  return true;
1316  }
1317  if (IP == B)
1318  break;
1319  }
1320  return BestGrowth != ~0u;
1321 }
1322 
1323 /// createNewWater - No existing WaterList entry will work for
1324 /// CPUsers[CPUserIndex], so create a place to put the CPE. The end of the
1325 /// block is used if in range, and the conditional branch munged so control
1326 /// flow is correct. Otherwise the block is split to create a hole with an
1327 /// unconditional branch around it. In either case NewMBB is set to a
1328 /// block following which the new island can be inserted (the WaterList
1329 /// is not adjusted).
1330 void ARMConstantIslands::createNewWater(unsigned CPUserIndex,
1331  unsigned UserOffset,
1332  MachineBasicBlock *&NewMBB) {
1333  CPUser &U = CPUsers[CPUserIndex];
1334  MachineInstr *UserMI = U.MI;
1335  MachineInstr *CPEMI = U.CPEMI;
1336  unsigned CPELogAlign = getCPELogAlign(CPEMI);
1337  MachineBasicBlock *UserMBB = UserMI->getParent();
1338  const BasicBlockInfo &UserBBI = BBInfo[UserMBB->getNumber()];
1339 
1340  // If the block does not end in an unconditional branch already, and if the
1341  // end of the block is within range, make new water there. (The addition
1342  // below is for the unconditional branch we will be adding: 4 bytes on ARM +
1343  // Thumb2, 2 on Thumb1.
1344  if (BBHasFallthrough(UserMBB)) {
1345  // Size of branch to insert.
1346  unsigned Delta = isThumb1 ? 2 : 4;
1347  // Compute the offset where the CPE will begin.
1348  unsigned CPEOffset = UserBBI.postOffset(CPELogAlign) + Delta;
1349 
1350  if (isOffsetInRange(UserOffset, CPEOffset, U)) {
1351  DEBUG(dbgs() << "Split at end of BB#" << UserMBB->getNumber()
1352  << format(", expected CPE offset %#x\n", CPEOffset));
1353  NewMBB = std::next(MachineFunction::iterator(UserMBB));
1354  // Add an unconditional branch from UserMBB to fallthrough block. Record
1355  // it for branch lengthening; this new branch will not get out of range,
1356  // but if the preceding conditional branch is out of range, the targets
1357  // will be exchanged, and the altered branch may be out of range, so the
1358  // machinery has to know about it.
1359  int UncondBr = isThumb ? ((isThumb2) ? ARM::t2B : ARM::tB) : ARM::B;
1360  if (!isThumb)
1361  BuildMI(UserMBB, DebugLoc(), TII->get(UncondBr)).addMBB(NewMBB);
1362  else
1363  BuildMI(UserMBB, DebugLoc(), TII->get(UncondBr)).addMBB(NewMBB)
1364  .addImm(ARMCC::AL).addReg(0);
1365  unsigned MaxDisp = getUnconditionalBrDisp(UncondBr);
1366  ImmBranches.push_back(ImmBranch(&UserMBB->back(),
1367  MaxDisp, false, UncondBr));
1368  computeBlockSize(UserMBB);
1369  adjustBBOffsetsAfter(UserMBB);
1370  return;
1371  }
1372  }
1373 
1374  // What a big block. Find a place within the block to split it. This is a
1375  // little tricky on Thumb1 since instructions are 2 bytes and constant pool
1376  // entries are 4 bytes: if instruction I references island CPE, and
1377  // instruction I+1 references CPE', it will not work well to put CPE as far
1378  // forward as possible, since then CPE' cannot immediately follow it (that
1379  // location is 2 bytes farther away from I+1 than CPE was from I) and we'd
1380  // need to create a new island. So, we make a first guess, then walk through
1381  // the instructions between the one currently being looked at and the
1382  // possible insertion point, and make sure any other instructions that
1383  // reference CPEs will be able to use the same island area; if not, we back
1384  // up the insertion point.
1385 
1386  // Try to split the block so it's fully aligned. Compute the latest split
1387  // point where we can add a 4-byte branch instruction, and then align to
1388  // LogAlign which is the largest possible alignment in the function.
1389  unsigned LogAlign = MF->getAlignment();
1390  assert(LogAlign >= CPELogAlign && "Over-aligned constant pool entry");
1391  unsigned KnownBits = UserBBI.internalKnownBits();
1392  unsigned UPad = UnknownPadding(LogAlign, KnownBits);
1393  unsigned BaseInsertOffset = UserOffset + U.getMaxDisp() - UPad;
1394  DEBUG(dbgs() << format("Split in middle of big block before %#x",
1395  BaseInsertOffset));
1396 
1397  // The 4 in the following is for the unconditional branch we'll be inserting
1398  // (allows for long branch on Thumb1). Alignment of the island is handled
1399  // inside isOffsetInRange.
1400  BaseInsertOffset -= 4;
1401 
1402  DEBUG(dbgs() << format(", adjusted to %#x", BaseInsertOffset)
1403  << " la=" << LogAlign
1404  << " kb=" << KnownBits
1405  << " up=" << UPad << '\n');
1406 
1407  // This could point off the end of the block if we've already got constant
1408  // pool entries following this block; only the last one is in the water list.
1409  // Back past any possible branches (allow for a conditional and a maximally
1410  // long unconditional).
1411  if (BaseInsertOffset + 8 >= UserBBI.postOffset()) {
1412  // Ensure BaseInsertOffset is larger than the offset of the instruction
1413  // following UserMI so that the loop which searches for the split point
1414  // iterates at least once.
1415  BaseInsertOffset =
1416  std::max(UserBBI.postOffset() - UPad - 8,
1417  UserOffset + TII->GetInstSizeInBytes(UserMI) + 1);
1418  DEBUG(dbgs() << format("Move inside block: %#x\n", BaseInsertOffset));
1419  }
1420  unsigned EndInsertOffset = BaseInsertOffset + 4 + UPad +
1421  CPEMI->getOperand(2).getImm();
1422  MachineBasicBlock::iterator MI = UserMI;
1423  ++MI;
1424  unsigned CPUIndex = CPUserIndex+1;
1425  unsigned NumCPUsers = CPUsers.size();
1426  MachineInstr *LastIT = nullptr;
1427  for (unsigned Offset = UserOffset+TII->GetInstSizeInBytes(UserMI);
1428  Offset < BaseInsertOffset;
1429  Offset += TII->GetInstSizeInBytes(MI), MI = std::next(MI)) {
1430  assert(MI != UserMBB->end() && "Fell off end of block");
1431  if (CPUIndex < NumCPUsers && CPUsers[CPUIndex].MI == MI) {
1432  CPUser &U = CPUsers[CPUIndex];
1433  if (!isOffsetInRange(Offset, EndInsertOffset, U)) {
1434  // Shift intertion point by one unit of alignment so it is within reach.
1435  BaseInsertOffset -= 1u << LogAlign;
1436  EndInsertOffset -= 1u << LogAlign;
1437  }
1438  // This is overly conservative, as we don't account for CPEMIs being
1439  // reused within the block, but it doesn't matter much. Also assume CPEs
1440  // are added in order with alignment padding. We may eventually be able
1441  // to pack the aligned CPEs better.
1442  EndInsertOffset += U.CPEMI->getOperand(2).getImm();
1443  CPUIndex++;
1444  }
1445 
1446  // Remember the last IT instruction.
1447  if (MI->getOpcode() == ARM::t2IT)
1448  LastIT = MI;
1449  }
1450 
1451  --MI;
1452 
1453  // Avoid splitting an IT block.
1454  if (LastIT) {
1455  unsigned PredReg = 0;
1456  ARMCC::CondCodes CC = getITInstrPredicate(MI, PredReg);
1457  if (CC != ARMCC::AL)
1458  MI = LastIT;
1459  }
1460 
1461  // We really must not split an IT block.
1462  DEBUG(unsigned PredReg;
1463  assert(!isThumb || getITInstrPredicate(MI, PredReg) == ARMCC::AL));
1464 
1465  NewMBB = splitBlockBeforeInstr(MI);
1466 }
1467 
1468 /// handleConstantPoolUser - Analyze the specified user, checking to see if it
1469 /// is out-of-range. If so, pick up the constant pool value and move it some
1470 /// place in-range. Return true if we changed any addresses (thus must run
1471 /// another pass of branch lengthening), false otherwise.
1472 bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex) {
1473  CPUser &U = CPUsers[CPUserIndex];
1474  MachineInstr *UserMI = U.MI;
1475  MachineInstr *CPEMI = U.CPEMI;
1476  unsigned CPI = getCombinedIndex(CPEMI);
1477  unsigned Size = CPEMI->getOperand(2).getImm();
1478  // Compute this only once, it's expensive.
1479  unsigned UserOffset = getUserOffset(U);
1480 
1481  // See if the current entry is within range, or there is a clone of it
1482  // in range.
1483  int result = findInRangeCPEntry(U, UserOffset);
1484  if (result==1) return false;
1485  else if (result==2) return true;
1486 
1487  // No existing clone of this CPE is within range.
1488  // We will be generating a new clone. Get a UID for it.
1489  unsigned ID = AFI->createPICLabelUId();
1490 
1491  // Look for water where we can place this CPE.
1492  MachineBasicBlock *NewIsland = MF->CreateMachineBasicBlock();
1493  MachineBasicBlock *NewMBB;
1494  water_iterator IP;
1495  if (findAvailableWater(U, UserOffset, IP)) {
1496  DEBUG(dbgs() << "Found water in range\n");
1497  MachineBasicBlock *WaterBB = *IP;
1498 
1499  // If the original WaterList entry was "new water" on this iteration,
1500  // propagate that to the new island. This is just keeping NewWaterList
1501  // updated to match the WaterList, which will be updated below.
1502  if (NewWaterList.erase(WaterBB))
1503  NewWaterList.insert(NewIsland);
1504 
1505  // The new CPE goes before the following block (NewMBB).
1506  NewMBB = std::next(MachineFunction::iterator(WaterBB));
1507 
1508  } else {
1509  // No water found.
1510  DEBUG(dbgs() << "No water found\n");
1511  createNewWater(CPUserIndex, UserOffset, NewMBB);
1512 
1513  // splitBlockBeforeInstr adds to WaterList, which is important when it is
1514  // called while handling branches so that the water will be seen on the
1515  // next iteration for constant pools, but in this context, we don't want
1516  // it. Check for this so it will be removed from the WaterList.
1517  // Also remove any entry from NewWaterList.
1518  MachineBasicBlock *WaterBB = std::prev(MachineFunction::iterator(NewMBB));
1519  IP = std::find(WaterList.begin(), WaterList.end(), WaterBB);
1520  if (IP != WaterList.end())
1521  NewWaterList.erase(WaterBB);
1522 
1523  // We are adding new water. Update NewWaterList.
1524  NewWaterList.insert(NewIsland);
1525  }
1526 
1527  // Remove the original WaterList entry; we want subsequent insertions in
1528  // this vicinity to go after the one we're about to insert. This
1529  // considerably reduces the number of times we have to move the same CPE
1530  // more than once and is also important to ensure the algorithm terminates.
1531  if (IP != WaterList.end())
1532  WaterList.erase(IP);
1533 
1534  // Okay, we know we can put an island before NewMBB now, do it!
1535  MF->insert(NewMBB, NewIsland);
1536 
1537  // Update internal data structures to account for the newly inserted MBB.
1538  updateForInsertedWaterBlock(NewIsland);
1539 
1540  // Now that we have an island to add the CPE to, clone the original CPE and
1541  // add it to the island.
1542  U.HighWaterMark = NewIsland;
1543  U.CPEMI = BuildMI(NewIsland, DebugLoc(), CPEMI->getDesc())
1544  .addImm(ID).addOperand(CPEMI->getOperand(1)).addImm(Size);
1545  CPEntries[CPI].push_back(CPEntry(U.CPEMI, ID, 1));
1546  ++NumCPEs;
1547 
1548  // Decrement the old entry, and remove it if refcount becomes 0.
1549  decrementCPEReferenceCount(CPI, CPEMI);
1550 
1551  // Mark the basic block as aligned as required by the const-pool entry.
1552  NewIsland->setAlignment(getCPELogAlign(U.CPEMI));
1553 
1554  // Increase the size of the island block to account for the new entry.
1555  BBInfo[NewIsland->getNumber()].Size += Size;
1556  adjustBBOffsetsAfter(std::prev(MachineFunction::iterator(NewIsland)));
1557 
1558  // Finally, change the CPI in the instruction operand to be ID.
1559  for (unsigned i = 0, e = UserMI->getNumOperands(); i != e; ++i)
1560  if (UserMI->getOperand(i).isCPI()) {
1561  UserMI->getOperand(i).setIndex(ID);
1562  break;
1563  }
1564 
1565  DEBUG(dbgs() << " Moved CPE to #" << ID << " CPI=" << CPI
1566  << format(" offset=%#x\n", BBInfo[NewIsland->getNumber()].Offset));
1567 
1568  return true;
1569 }
1570 
1571 /// removeDeadCPEMI - Remove a dead constant pool entry instruction. Update
1572 /// sizes and offsets of impacted basic blocks.
1573 void ARMConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) {
1574  MachineBasicBlock *CPEBB = CPEMI->getParent();
1575  unsigned Size = CPEMI->getOperand(2).getImm();
1576  CPEMI->eraseFromParent();
1577  BBInfo[CPEBB->getNumber()].Size -= Size;
1578  // All succeeding offsets have the current size value added in, fix this.
1579  if (CPEBB->empty()) {
1580  BBInfo[CPEBB->getNumber()].Size = 0;
1581 
1582  // This block no longer needs to be aligned.
1583  CPEBB->setAlignment(0);
1584  } else
1585  // Entries are sorted by descending alignment, so realign from the front.
1586  CPEBB->setAlignment(getCPELogAlign(CPEBB->begin()));
1587 
1588  adjustBBOffsetsAfter(CPEBB);
1589  // An island has only one predecessor BB and one successor BB. Check if
1590  // this BB's predecessor jumps directly to this BB's successor. This
1591  // shouldn't happen currently.
1592  assert(!BBIsJumpedOver(CPEBB) && "How did this happen?");
1593  // FIXME: remove the empty blocks after all the work is done?
1594 }
1595 
1596 /// removeUnusedCPEntries - Remove constant pool entries whose refcounts
1597 /// are zero.
1598 bool ARMConstantIslands::removeUnusedCPEntries() {
1599  unsigned MadeChange = false;
1600  for (unsigned i = 0, e = CPEntries.size(); i != e; ++i) {
1601  std::vector<CPEntry> &CPEs = CPEntries[i];
1602  for (unsigned j = 0, ee = CPEs.size(); j != ee; ++j) {
1603  if (CPEs[j].RefCount == 0 && CPEs[j].CPEMI) {
1604  removeDeadCPEMI(CPEs[j].CPEMI);
1605  CPEs[j].CPEMI = nullptr;
1606  MadeChange = true;
1607  }
1608  }
1609  }
1610  return MadeChange;
1611 }
1612 
1613 /// isBBInRange - Returns true if the distance between specific MI and
1614 /// specific BB can fit in MI's displacement field.
1615 bool ARMConstantIslands::isBBInRange(MachineInstr *MI,MachineBasicBlock *DestBB,
1616  unsigned MaxDisp) {
1617  unsigned PCAdj = isThumb ? 4 : 8;
1618  unsigned BrOffset = getOffsetOf(MI) + PCAdj;
1619  unsigned DestOffset = BBInfo[DestBB->getNumber()].Offset;
1620 
1621  DEBUG(dbgs() << "Branch of destination BB#" << DestBB->getNumber()
1622  << " from BB#" << MI->getParent()->getNumber()
1623  << " max delta=" << MaxDisp
1624  << " from " << getOffsetOf(MI) << " to " << DestOffset
1625  << " offset " << int(DestOffset-BrOffset) << "\t" << *MI);
1626 
1627  if (BrOffset <= DestOffset) {
1628  // Branch before the Dest.
1629  if (DestOffset-BrOffset <= MaxDisp)
1630  return true;
1631  } else {
1632  if (BrOffset-DestOffset <= MaxDisp)
1633  return true;
1634  }
1635  return false;
1636 }
1637 
1638 /// fixupImmediateBr - Fix up an immediate branch whose destination is too far
1639 /// away to fit in its displacement field.
1640 bool ARMConstantIslands::fixupImmediateBr(ImmBranch &Br) {
1641  MachineInstr *MI = Br.MI;
1642  MachineBasicBlock *DestBB = MI->getOperand(0).getMBB();
1643 
1644  // Check to see if the DestBB is already in-range.
1645  if (isBBInRange(MI, DestBB, Br.MaxDisp))
1646  return false;
1647 
1648  if (!Br.isCond)
1649  return fixupUnconditionalBr(Br);
1650  return fixupConditionalBr(Br);
1651 }
1652 
1653 /// fixupUnconditionalBr - Fix up an unconditional branch whose destination is
1654 /// too far away to fit in its displacement field. If the LR register has been
1655 /// spilled in the epilogue, then we can use BL to implement a far jump.
1656 /// Otherwise, add an intermediate branch instruction to a branch.
1657 bool
1658 ARMConstantIslands::fixupUnconditionalBr(ImmBranch &Br) {
1659  MachineInstr *MI = Br.MI;
1660  MachineBasicBlock *MBB = MI->getParent();
1661  if (!isThumb1)
1662  llvm_unreachable("fixupUnconditionalBr is Thumb1 only!");
1663 
1664  // Use BL to implement far jump.
1665  Br.MaxDisp = (1 << 21) * 2;
1666  MI->setDesc(TII->get(ARM::tBfar));
1667  BBInfo[MBB->getNumber()].Size += 2;
1668  adjustBBOffsetsAfter(MBB);
1669  HasFarJump = true;
1670  ++NumUBrFixed;
1671 
1672  DEBUG(dbgs() << " Changed B to long jump " << *MI);
1673 
1674  return true;
1675 }
1676 
1677 /// fixupConditionalBr - Fix up a conditional branch whose destination is too
1678 /// far away to fit in its displacement field. It is converted to an inverse
1679 /// conditional branch + an unconditional branch to the destination.
1680 bool
1681 ARMConstantIslands::fixupConditionalBr(ImmBranch &Br) {
1682  MachineInstr *MI = Br.MI;
1683  MachineBasicBlock *DestBB = MI->getOperand(0).getMBB();
1684 
1685  // Add an unconditional branch to the destination and invert the branch
1686  // condition to jump over it:
1687  // blt L1
1688  // =>
1689  // bge L2
1690  // b L1
1691  // L2:
1693  CC = ARMCC::getOppositeCondition(CC);
1694  unsigned CCReg = MI->getOperand(2).getReg();
1695 
1696  // If the branch is at the end of its MBB and that has a fall-through block,
1697  // direct the updated conditional branch to the fall-through block. Otherwise,
1698  // split the MBB before the next instruction.
1699  MachineBasicBlock *MBB = MI->getParent();
1700  MachineInstr *BMI = &MBB->back();
1701  bool NeedSplit = (BMI != MI) || !BBHasFallthrough(MBB);
1702 
1703  ++NumCBrFixed;
1704  if (BMI != MI) {
1705  if (std::next(MachineBasicBlock::iterator(MI)) == std::prev(MBB->end()) &&
1706  BMI->getOpcode() == Br.UncondBr) {
1707  // Last MI in the BB is an unconditional branch. Can we simply invert the
1708  // condition and swap destinations:
1709  // beq L1
1710  // b L2
1711  // =>
1712  // bne L2
1713  // b L1
1714  MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB();
1715  if (isBBInRange(MI, NewDest, Br.MaxDisp)) {
1716  DEBUG(dbgs() << " Invert Bcc condition and swap its destination with "
1717  << *BMI);
1718  BMI->getOperand(0).setMBB(DestBB);
1719  MI->getOperand(0).setMBB(NewDest);
1720  MI->getOperand(1).setImm(CC);
1721  return true;
1722  }
1723  }
1724  }
1725 
1726  if (NeedSplit) {
1727  splitBlockBeforeInstr(MI);
1728  // No need for the branch to the next block. We're adding an unconditional
1729  // branch to the destination.
1730  int delta = TII->GetInstSizeInBytes(&MBB->back());
1731  BBInfo[MBB->getNumber()].Size -= delta;
1732  MBB->back().eraseFromParent();
1733  // BBInfo[SplitBB].Offset is wrong temporarily, fixed below
1734  }
1735  MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(MBB));
1736 
1737  DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber()
1738  << " also invert condition and change dest. to BB#"
1739  << NextBB->getNumber() << "\n");
1740 
1741  // Insert a new conditional branch and a new unconditional branch.
1742  // Also update the ImmBranch as well as adding a new entry for the new branch.
1743  BuildMI(MBB, DebugLoc(), TII->get(MI->getOpcode()))
1744  .addMBB(NextBB).addImm(CC).addReg(CCReg);
1745  Br.MI = &MBB->back();
1746  BBInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
1747  if (isThumb)
1748  BuildMI(MBB, DebugLoc(), TII->get(Br.UncondBr)).addMBB(DestBB)
1749  .addImm(ARMCC::AL).addReg(0);
1750  else
1751  BuildMI(MBB, DebugLoc(), TII->get(Br.UncondBr)).addMBB(DestBB);
1752  BBInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
1753  unsigned MaxDisp = getUnconditionalBrDisp(Br.UncondBr);
1754  ImmBranches.push_back(ImmBranch(&MBB->back(), MaxDisp, false, Br.UncondBr));
1755 
1756  // Remove the old conditional branch. It may or may not still be in MBB.
1757  BBInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI);
1758  MI->eraseFromParent();
1759  adjustBBOffsetsAfter(MBB);
1760  return true;
1761 }
1762 
1763 /// undoLRSpillRestore - Remove Thumb push / pop instructions that only spills
1764 /// LR / restores LR to pc. FIXME: This is done here because it's only possible
1765 /// to do this if tBfar is not used.
1766 bool ARMConstantIslands::undoLRSpillRestore() {
1767  bool MadeChange = false;
1768  for (unsigned i = 0, e = PushPopMIs.size(); i != e; ++i) {
1769  MachineInstr *MI = PushPopMIs[i];
1770  // First two operands are predicates.
1771  if (MI->getOpcode() == ARM::tPOP_RET &&
1772  MI->getOperand(2).getReg() == ARM::PC &&
1773  MI->getNumExplicitOperands() == 3) {
1774  // Create the new insn and copy the predicate from the old.
1775  BuildMI(MI->getParent(), MI->getDebugLoc(), TII->get(ARM::tBX_RET))
1776  .addOperand(MI->getOperand(0))
1777  .addOperand(MI->getOperand(1));
1778  MI->eraseFromParent();
1779  MadeChange = true;
1780  }
1781  }
1782  return MadeChange;
1783 }
1784 
1785 // mayOptimizeThumb2Instruction - Returns true if optimizeThumb2Instructions
1786 // below may shrink MI.
1787 bool
1788 ARMConstantIslands::mayOptimizeThumb2Instruction(const MachineInstr *MI) const {
1789  switch(MI->getOpcode()) {
1790  // optimizeThumb2Instructions.
1791  case ARM::t2LEApcrel:
1792  case ARM::t2LDRpci:
1793  // optimizeThumb2Branches.
1794  case ARM::t2B:
1795  case ARM::t2Bcc:
1796  case ARM::tBcc:
1797  // optimizeThumb2JumpTables.
1798  case ARM::t2BR_JT:
1799  return true;
1800  }
1801  return false;
1802 }
1803 
1804 bool ARMConstantIslands::optimizeThumb2Instructions() {
1805  bool MadeChange = false;
1806 
1807  // Shrink ADR and LDR from constantpool.
1808  for (unsigned i = 0, e = CPUsers.size(); i != e; ++i) {
1809  CPUser &U = CPUsers[i];
1810  unsigned Opcode = U.MI->getOpcode();
1811  unsigned NewOpc = 0;
1812  unsigned Scale = 1;
1813  unsigned Bits = 0;
1814  switch (Opcode) {
1815  default: break;
1816  case ARM::t2LEApcrel:
1817  if (isARMLowRegister(U.MI->getOperand(0).getReg())) {
1818  NewOpc = ARM::tLEApcrel;
1819  Bits = 8;
1820  Scale = 4;
1821  }
1822  break;
1823  case ARM::t2LDRpci:
1824  if (isARMLowRegister(U.MI->getOperand(0).getReg())) {
1825  NewOpc = ARM::tLDRpci;
1826  Bits = 8;
1827  Scale = 4;
1828  }
1829  break;
1830  }
1831 
1832  if (!NewOpc)
1833  continue;
1834 
1835  unsigned UserOffset = getUserOffset(U);
1836  unsigned MaxOffs = ((1 << Bits) - 1) * Scale;
1837 
1838  // Be conservative with inline asm.
1839  if (!U.KnownAlignment)
1840  MaxOffs -= 2;
1841 
1842  // FIXME: Check if offset is multiple of scale if scale is not 4.
1843  if (isCPEntryInRange(U.MI, UserOffset, U.CPEMI, MaxOffs, false, true)) {
1844  DEBUG(dbgs() << "Shrink: " << *U.MI);
1845  U.MI->setDesc(TII->get(NewOpc));
1846  MachineBasicBlock *MBB = U.MI->getParent();
1847  BBInfo[MBB->getNumber()].Size -= 2;
1848  adjustBBOffsetsAfter(MBB);
1849  ++NumT2CPShrunk;
1850  MadeChange = true;
1851  }
1852  }
1853 
1854  MadeChange |= optimizeThumb2Branches();
1855  MadeChange |= optimizeThumb2JumpTables();
1856  return MadeChange;
1857 }
1858 
1859 bool ARMConstantIslands::optimizeThumb2Branches() {
1860  bool MadeChange = false;
1861 
1862  // The order in which branches appear in ImmBranches is approximately their
1863  // order within the function body. By visiting later branches first, we reduce
1864  // the distance between earlier forward branches and their targets, making it
1865  // more likely that the cbn?z optimization, which can only apply to forward
1866  // branches, will succeed.
1867  for (unsigned i = ImmBranches.size(); i != 0; --i) {
1868  ImmBranch &Br = ImmBranches[i-1];
1869  unsigned Opcode = Br.MI->getOpcode();
1870  unsigned NewOpc = 0;
1871  unsigned Scale = 1;
1872  unsigned Bits = 0;
1873  switch (Opcode) {
1874  default: break;
1875  case ARM::t2B:
1876  NewOpc = ARM::tB;
1877  Bits = 11;
1878  Scale = 2;
1879  break;
1880  case ARM::t2Bcc: {
1881  NewOpc = ARM::tBcc;
1882  Bits = 8;
1883  Scale = 2;
1884  break;
1885  }
1886  }
1887  if (NewOpc) {
1888  unsigned MaxOffs = ((1 << (Bits-1))-1) * Scale;
1889  MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB();
1890  if (isBBInRange(Br.MI, DestBB, MaxOffs)) {
1891  DEBUG(dbgs() << "Shrink branch: " << *Br.MI);
1892  Br.MI->setDesc(TII->get(NewOpc));
1893  MachineBasicBlock *MBB = Br.MI->getParent();
1894  BBInfo[MBB->getNumber()].Size -= 2;
1895  adjustBBOffsetsAfter(MBB);
1896  ++NumT2BrShrunk;
1897  MadeChange = true;
1898  }
1899  }
1900 
1901  Opcode = Br.MI->getOpcode();
1902  if (Opcode != ARM::tBcc)
1903  continue;
1904 
1905  // If the conditional branch doesn't kill CPSR, then CPSR can be liveout
1906  // so this transformation is not safe.
1907  if (!Br.MI->killsRegister(ARM::CPSR))
1908  continue;
1909 
1910  NewOpc = 0;
1911  unsigned PredReg = 0;
1912  ARMCC::CondCodes Pred = getInstrPredicate(Br.MI, PredReg);
1913  if (Pred == ARMCC::EQ)
1914  NewOpc = ARM::tCBZ;
1915  else if (Pred == ARMCC::NE)
1916  NewOpc = ARM::tCBNZ;
1917  if (!NewOpc)
1918  continue;
1919  MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB();
1920  // Check if the distance is within 126. Subtract starting offset by 2
1921  // because the cmp will be eliminated.
1922  unsigned BrOffset = getOffsetOf(Br.MI) + 4 - 2;
1923  unsigned DestOffset = BBInfo[DestBB->getNumber()].Offset;
1924  if (BrOffset < DestOffset && (DestOffset - BrOffset) <= 126) {
1925  MachineBasicBlock::iterator CmpMI = Br.MI;
1926  if (CmpMI != Br.MI->getParent()->begin()) {
1927  --CmpMI;
1928  if (CmpMI->getOpcode() == ARM::tCMPi8) {
1929  unsigned Reg = CmpMI->getOperand(0).getReg();
1930  Pred = getInstrPredicate(CmpMI, PredReg);
1931  if (Pred == ARMCC::AL &&
1932  CmpMI->getOperand(1).getImm() == 0 &&
1933  isARMLowRegister(Reg)) {
1934  MachineBasicBlock *MBB = Br.MI->getParent();
1935  DEBUG(dbgs() << "Fold: " << *CmpMI << " and: " << *Br.MI);
1936  MachineInstr *NewBR =
1937  BuildMI(*MBB, CmpMI, Br.MI->getDebugLoc(), TII->get(NewOpc))
1938  .addReg(Reg).addMBB(DestBB,Br.MI->getOperand(0).getTargetFlags());
1939  CmpMI->eraseFromParent();
1940  Br.MI->eraseFromParent();
1941  Br.MI = NewBR;
1942  BBInfo[MBB->getNumber()].Size -= 2;
1943  adjustBBOffsetsAfter(MBB);
1944  ++NumCBZ;
1945  MadeChange = true;
1946  }
1947  }
1948  }
1949  }
1950  }
1951 
1952  return MadeChange;
1953 }
1954 
1955 static bool isSimpleIndexCalc(MachineInstr &I, unsigned EntryReg,
1956  unsigned BaseReg) {
1957  if (I.getOpcode() != ARM::t2ADDrs)
1958  return false;
1959 
1960  if (I.getOperand(0).getReg() != EntryReg)
1961  return false;
1962 
1963  if (I.getOperand(1).getReg() != BaseReg)
1964  return false;
1965 
1966  // FIXME: what about CC and IdxReg?
1967  return true;
1968 }
1969 
1970 /// \brief While trying to form a TBB/TBH instruction, we may (if the table
1971 /// doesn't immediately follow the BR_JT) need access to the start of the
1972 /// jump-table. We know one instruction that produces such a register; this
1973 /// function works out whether that definition can be preserved to the BR_JT,
1974 /// possibly by removing an intervening addition (which is usually needed to
1975 /// calculate the actual entry to jump to).
1976 bool ARMConstantIslands::preserveBaseRegister(MachineInstr *JumpMI,
1977  MachineInstr *LEAMI,
1978  unsigned &DeadSize,
1979  bool &CanDeleteLEA,
1980  bool &BaseRegKill) {
1981  if (JumpMI->getParent() != LEAMI->getParent())
1982  return false;
1983 
1984  // Now we hope that we have at least these instructions in the basic block:
1985  // BaseReg = t2LEA ...
1986  // [...]
1987  // EntryReg = t2ADDrs BaseReg, ...
1988  // [...]
1989  // t2BR_JT EntryReg
1990  //
1991  // We have to be very conservative about what we recognise here though. The
1992  // main perturbing factors to watch out for are:
1993  // + Spills at any point in the chain: not direct problems but we would
1994  // expect a blocking Def of the spilled register so in practice what we
1995  // can do is limited.
1996  // + EntryReg == BaseReg: this is the one situation we should allow a Def
1997  // of BaseReg, but only if the t2ADDrs can be removed.
1998  // + Some instruction other than t2ADDrs computing the entry. Not seen in
1999  // the wild, but we should be careful.
2000  unsigned EntryReg = JumpMI->getOperand(0).getReg();
2001  unsigned BaseReg = LEAMI->getOperand(0).getReg();
2002 
2003  CanDeleteLEA = true;
2004  BaseRegKill = false;
2005  MachineInstr *RemovableAdd = nullptr;
2007  for (++I; &*I != JumpMI; ++I) {
2008  if (isSimpleIndexCalc(*I, EntryReg, BaseReg)) {
2009  RemovableAdd = &*I;
2010  break;
2011  }
2012 
2013  for (unsigned K = 0, E = I->getNumOperands(); K != E; ++K) {
2014  const MachineOperand &MO = I->getOperand(K);
2015  if (!MO.isReg() || !MO.getReg())
2016  continue;
2017  if (MO.isDef() && MO.getReg() == BaseReg)
2018  return false;
2019  if (MO.isUse() && MO.getReg() == BaseReg) {
2020  BaseRegKill = BaseRegKill || MO.isKill();
2021  CanDeleteLEA = false;
2022  }
2023  }
2024  }
2025 
2026  if (!RemovableAdd)
2027  return true;
2028 
2029  // Check the add really is removable, and that nothing else in the block
2030  // clobbers BaseReg.
2031  for (++I; &*I != JumpMI; ++I) {
2032  for (unsigned K = 0, E = I->getNumOperands(); K != E; ++K) {
2033  const MachineOperand &MO = I->getOperand(K);
2034  if (!MO.isReg() || !MO.getReg())
2035  continue;
2036  if (MO.isDef() && MO.getReg() == BaseReg)
2037  return false;
2038  if (MO.isUse() && MO.getReg() == EntryReg)
2039  RemovableAdd = nullptr;
2040  }
2041  }
2042 
2043  if (RemovableAdd) {
2044  RemovableAdd->eraseFromParent();
2045  DeadSize += 4;
2046  } else if (BaseReg == EntryReg) {
2047  // The add wasn't removable, but clobbered the base for the TBB. So we can't
2048  // preserve it.
2049  return false;
2050  }
2051 
2052  // We reached the end of the block without seeing another definition of
2053  // BaseReg (except, possibly the t2ADDrs, which was removed). BaseReg can be
2054  // used in the TBB/TBH if necessary.
2055  return true;
2056 }
2057 
2058 /// \brief Returns whether CPEMI is the first instruction in the block
2059 /// immediately following JTMI (assumed to be a TBB or TBH terminator). If so,
2060 /// we can switch the first register to PC and usually remove the address
2061 /// calculation that preceeded it.
2062 static bool jumpTableFollowsTB(MachineInstr *JTMI, MachineInstr *CPEMI) {
2063  MachineFunction::iterator MBB = JTMI->getParent();
2064  MachineFunction *MF = MBB->getParent();
2065  ++MBB;
2066 
2067  return MBB != MF->end() && MBB->begin() != MBB->end() &&
2068  &*MBB->begin() == CPEMI;
2069 }
2070 
2071 /// optimizeThumb2JumpTables - Use tbb / tbh instructions to generate smaller
2072 /// jumptables when it's possible.
2073 bool ARMConstantIslands::optimizeThumb2JumpTables() {
2074  bool MadeChange = false;
2075 
2076  // FIXME: After the tables are shrunk, can we get rid some of the
2077  // constantpool tables?
2078  MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
2079  if (!MJTI) return false;
2080 
2081  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
2082  for (unsigned i = 0, e = T2JumpTables.size(); i != e; ++i) {
2083  MachineInstr *MI = T2JumpTables[i];
2084  const MCInstrDesc &MCID = MI->getDesc();
2085  unsigned NumOps = MCID.getNumOperands();
2086  unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 2 : 1);
2087  MachineOperand JTOP = MI->getOperand(JTOpIdx);
2088  unsigned JTI = JTOP.getIndex();
2089  assert(JTI < JT.size());
2090 
2091  bool ByteOk = true;
2092  bool HalfWordOk = true;
2093  unsigned JTOffset = getOffsetOf(MI) + 4;
2094  const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
2095  for (unsigned j = 0, ee = JTBBs.size(); j != ee; ++j) {
2096  MachineBasicBlock *MBB = JTBBs[j];
2097  unsigned DstOffset = BBInfo[MBB->getNumber()].Offset;
2098  // Negative offset is not ok. FIXME: We should change BB layout to make
2099  // sure all the branches are forward.
2100  if (ByteOk && (DstOffset - JTOffset) > ((1<<8)-1)*2)
2101  ByteOk = false;
2102  unsigned TBHLimit = ((1<<16)-1)*2;
2103  if (HalfWordOk && (DstOffset - JTOffset) > TBHLimit)
2104  HalfWordOk = false;
2105  if (!ByteOk && !HalfWordOk)
2106  break;
2107  }
2108 
2109  if (!ByteOk && !HalfWordOk)
2110  continue;
2111 
2112  MachineBasicBlock *MBB = MI->getParent();
2113  if (!MI->getOperand(0).isKill()) // FIXME: needed now?
2114  continue;
2115  unsigned IdxReg = MI->getOperand(1).getReg();
2116  bool IdxRegKill = MI->getOperand(1).isKill();
2117 
2118  CPUser &User = CPUsers[JumpTableUserIndices[JTI]];
2119  unsigned DeadSize = 0;
2120  bool CanDeleteLEA = false;
2121  bool BaseRegKill = false;
2122  bool PreservedBaseReg =
2123  preserveBaseRegister(MI, User.MI, DeadSize, CanDeleteLEA, BaseRegKill);
2124 
2125  if (!jumpTableFollowsTB(MI, User.CPEMI) && !PreservedBaseReg)
2126  continue;
2127 
2128  DEBUG(dbgs() << "Shrink JT: " << *MI);
2129  MachineInstr *CPEMI = User.CPEMI;
2130  unsigned Opc = ByteOk ? ARM::t2TBB_JT : ARM::t2TBH_JT;
2132  MachineInstr *NewJTMI =
2133  BuildMI(*MBB, MI_JT, MI->getDebugLoc(), TII->get(Opc))
2134  .addReg(User.MI->getOperand(0).getReg(),
2135  getKillRegState(BaseRegKill))
2136  .addReg(IdxReg, getKillRegState(IdxRegKill))
2137  .addJumpTableIndex(JTI, JTOP.getTargetFlags())
2138  .addImm(CPEMI->getOperand(0).getImm());
2139  DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": " << *NewJTMI);
2140 
2141  unsigned JTOpc = ByteOk ? ARM::JUMPTABLE_TBB : ARM::JUMPTABLE_TBH;
2142  CPEMI->setDesc(TII->get(JTOpc));
2143 
2144  if (jumpTableFollowsTB(MI, User.CPEMI)) {
2145  NewJTMI->getOperand(0).setReg(ARM::PC);
2146  NewJTMI->getOperand(0).setIsKill(false);
2147 
2148  if (CanDeleteLEA) {
2149  User.MI->eraseFromParent();
2150  DeadSize += 4;
2151 
2152  // The LEA was eliminated, the TBB instruction becomes the only new user
2153  // of the jump table.
2154  User.MI = NewJTMI;
2155  User.MaxDisp = 4;
2156  User.NegOk = false;
2157  User.IsSoImm = false;
2158  User.KnownAlignment = false;
2159  } else {
2160  // The LEA couldn't be eliminated, so we must add another CPUser to
2161  // record the TBB or TBH use.
2162  int CPEntryIdx = JumpTableEntryIndices[JTI];
2163  auto &CPEs = CPEntries[CPEntryIdx];
2164  auto Entry = std::find_if(CPEs.begin(), CPEs.end(), [&](CPEntry &E) {
2165  return E.CPEMI == User.CPEMI;
2166  });
2167  ++Entry->RefCount;
2168  CPUsers.emplace_back(CPUser(NewJTMI, User.CPEMI, 4, false, false));
2169  }
2170  }
2171 
2172  unsigned NewSize = TII->GetInstSizeInBytes(NewJTMI);
2173  unsigned OrigSize = TII->GetInstSizeInBytes(MI);
2174  MI->eraseFromParent();
2175 
2176  int Delta = OrigSize - NewSize + DeadSize;
2177  BBInfo[MBB->getNumber()].Size -= Delta;
2178  adjustBBOffsetsAfter(MBB);
2179 
2180  ++NumTBs;
2181  MadeChange = true;
2182  }
2183 
2184  return MadeChange;
2185 }
2186 
2187 /// reorderThumb2JumpTables - Adjust the function's block layout to ensure that
2188 /// jump tables always branch forwards, since that's what tbb and tbh need.
2189 bool ARMConstantIslands::reorderThumb2JumpTables() {
2190  bool MadeChange = false;
2191 
2192  MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
2193  if (!MJTI) return false;
2194 
2195  const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
2196  for (unsigned i = 0, e = T2JumpTables.size(); i != e; ++i) {
2197  MachineInstr *MI = T2JumpTables[i];
2198  const MCInstrDesc &MCID = MI->getDesc();
2199  unsigned NumOps = MCID.getNumOperands();
2200  unsigned JTOpIdx = NumOps - (MI->isPredicable() ? 2 : 1);
2201  MachineOperand JTOP = MI->getOperand(JTOpIdx);
2202  unsigned JTI = JTOP.getIndex();
2203  assert(JTI < JT.size());
2204 
2205  // We prefer if target blocks for the jump table come after the jump
2206  // instruction so we can use TB[BH]. Loop through the target blocks
2207  // and try to adjust them such that that's true.
2208  int JTNumber = MI->getParent()->getNumber();
2209  const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
2210  for (unsigned j = 0, ee = JTBBs.size(); j != ee; ++j) {
2211  MachineBasicBlock *MBB = JTBBs[j];
2212  int DTNumber = MBB->getNumber();
2213 
2214  if (DTNumber < JTNumber) {
2215  // The destination precedes the switch. Try to move the block forward
2216  // so we have a positive offset.
2217  MachineBasicBlock *NewBB =
2218  adjustJTTargetBlockForward(MBB, MI->getParent());
2219  if (NewBB)
2220  MJTI->ReplaceMBBInJumpTable(JTI, JTBBs[j], NewBB);
2221  MadeChange = true;
2222  }
2223  }
2224  }
2225 
2226  return MadeChange;
2227 }
2228 
2229 MachineBasicBlock *ARMConstantIslands::
2230 adjustJTTargetBlockForward(MachineBasicBlock *BB, MachineBasicBlock *JTBB) {
2231  // If the destination block is terminated by an unconditional branch,
2232  // try to move it; otherwise, create a new block following the jump
2233  // table that branches back to the actual target. This is a very simple
2234  // heuristic. FIXME: We can definitely improve it.
2235  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
2238  MachineFunction::iterator BBi = BB;
2239  MachineFunction::iterator OldPrior = std::prev(BBi);
2240 
2241  // If the block terminator isn't analyzable, don't try to move the block
2242  bool B = TII->AnalyzeBranch(*BB, TBB, FBB, Cond);
2243 
2244  // If the block ends in an unconditional branch, move it. The prior block
2245  // has to have an analyzable terminator for us to move this one. Be paranoid
2246  // and make sure we're not trying to move the entry block of the function.
2247  if (!B && Cond.empty() && BB != MF->begin() &&
2248  !TII->AnalyzeBranch(*OldPrior, TBB, FBB, CondPrior)) {
2249  BB->moveAfter(JTBB);
2250  OldPrior->updateTerminator();
2251  BB->updateTerminator();
2252  // Update numbering to account for the block being moved.
2253  MF->RenumberBlocks();
2254  ++NumJTMoved;
2255  return nullptr;
2256  }
2257 
2258  // Create a new MBB for the code after the jump BB.
2259  MachineBasicBlock *NewBB =
2260  MF->CreateMachineBasicBlock(JTBB->getBasicBlock());
2261  MachineFunction::iterator MBBI = JTBB; ++MBBI;
2262  MF->insert(MBBI, NewBB);
2263 
2264  // Add an unconditional branch from NewBB to BB.
2265  // There doesn't seem to be meaningful DebugInfo available; this doesn't
2266  // correspond directly to anything in the source.
2267  assert (isThumb2 && "Adjusting for TB[BH] but not in Thumb2?");
2268  BuildMI(NewBB, DebugLoc(), TII->get(ARM::t2B)).addMBB(BB)
2269  .addImm(ARMCC::AL).addReg(0);
2270 
2271  // Update internal data structures to account for the newly inserted MBB.
2272  MF->RenumberBlocks(NewBB);
2273 
2274  // Update the CFG.
2275  NewBB->addSuccessor(BB);
2276  JTBB->removeSuccessor(BB);
2277  JTBB->addSuccessor(NewBB);
2278 
2279  ++NumJTInserted;
2280  return NewBB;
2281 }
unsigned succ_size() const
const MachineFunction * getParent() const
getParent - Return the MachineFunction containing this basic block.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
bool AnalyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
STATISTIC(NumFunctions,"Total number of functions")
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them...
MachineBasicBlock * getMBB() const
int getNumber() const
getNumber - MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a M...
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:138
bool isPredicable(QueryType Type=AllInBundle) const
Return true if this instruction has a predicate operand that controls execution.
Definition: MachineInstr.h:457
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:264
void moveAfter(MachineBasicBlock *NewBefore)
A debug info location.
Definition: DebugLoc.h:34
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
Definition: Function.cpp:822
#define op(i)
static bool isThumb(const MCSubtargetInfo &STI)
void setAlignment(unsigned Align)
setAlignment - Set alignment of the basic block.
bool ReplaceMBBInJumpTable(unsigned Idx, MachineBasicBlock *Old, MachineBasicBlock *New)
ReplaceMBBInJumpTable - If Old is a target of the jump tables, update the jump table to branch to New...
ARMCC::CondCodes getITInstrPredicate(const MachineInstr *MI, unsigned &PredReg)
getITInstrPredicate - Valid only in Thumb2 mode.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(const char *reason, bool gen_crash_diag=true)
Reports a serious error, calling any installed error handler.
static bool CompareMBBNumbers(const MachineBasicBlock *LHS, const MachineBasicBlock *RHS)
CompareMBBNumbers - Little predicate function to sort the WaterList by MBB ID.
static bool isSimpleIndexCalc(MachineInstr &I, unsigned EntryReg, unsigned BaseReg)
const std::vector< MachineJumpTableEntry > & getJumpTables() const
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Reg
All possible values of the reg field in the ModR/M byte.
void setIndex(int Idx)
#define false
Definition: ConvertUTF.c:65
const MachineInstrBuilder & addImm(int64_t Val) const
addImm - Add a new immediate operand.
unsigned getNumOperands() const
Access to explicit operands of the instruction.
Definition: MachineInstr.h:271
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
FunctionPass * createARMConstantIslandPass()
createARMConstantIslandPass - returns an instance of the constpool island pass.
bool isKill() const
void transferSuccessors(MachineBasicBlock *fromMBB)
transferSuccessors - Transfers all the successors from MBB to this machine basic block (i...
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
iterator getLastNonDebugInstr()
getLastNonDebugInstr - returns an iterator to the last non-debug instruction in the basic block...
int64_t getImm() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
unsigned getKillRegState(bool B)
const BasicBlock * getBasicBlock() const
getBasicBlock - Return the LLVM basic block that this instance corresponded to originally.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:267
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:120
format_object< Ts...> format(const char *Fmt, const Ts &...Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:111
bundle_iterator< MachineInstr, instr_iterator > iterator
#define rc(i)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:109
ARMCC::CondCodes getInstrPredicate(const MachineInstr *MI, unsigned &PredReg)
getInstrPredicate - If instruction is predicated, returns its predicate condition, otherwise returns AL.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:273
void ensureAlignment(unsigned A)
ensureAlignment - Make sure the function is at least 1 << A bytes aligned.
void setMBB(MachineBasicBlock *MBB)
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
static bool BBIsJumpedOver(MachineBasicBlock *MBB)
BBIsJumpedOver - Return true of the specified basic block's only predecessor unconditionally branches...
void setImm(int64_t immVal)
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
static unsigned UnknownPadding(unsigned LogAlign, unsigned KnownBits)
UnknownPadding - Return the worst case padding that could result from unknown offset bits...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL, const MCInstrDesc &MCID)
BuildMI - Builder interface.
void removeSuccessor(MachineBasicBlock *succ)
removeSuccessor - Remove successor from the successors list of this MachineBasicBlock.
void setIsKill(bool Val=true)
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:388
const MachineInstrBuilder & addJumpTableIndex(unsigned Idx, unsigned char TargetFlags=0) const
MachineOperand class - Representation of each machine instruction operand.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
void updateTerminator()
updateTerminator - Update the terminator instructions in block to account for changes to the layout...
static cl::opt< bool > AdjustJumpTableBlocks("arm-adjust-jump-tables", cl::Hidden, cl::init(true), cl::desc("Adjust basic block layout to better use TB[BH]"))
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:468
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:238
static bool isARMLowRegister(unsigned Reg)
isARMLowRegister - Returns true if the register is a low register (r0-r7).
Definition: ARMBaseInfo.h:210
Representation of each machine instruction.
Definition: MachineInstr.h:51
static CondCodes getOppositeCondition(CondCodes CC)
Definition: ARMBaseInfo.h:47
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
ARMFunctionInfo - This class is derived from MachineFunctionInfo and contains private ARM-specific in...
void setReg(unsigned Reg)
Change the register this operand corresponds to.
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned char TargetFlags=0) const
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:54
void size_t size
static unsigned getUnconditionalBrDisp(int Opc)
getUnconditionalBrDisp - Returns the maximum displacement that can fit in the specific unconditional ...
unsigned getReg() const
getReg - Returns the register number.
static bool jumpTableFollowsTB(MachineInstr *JTMI, MachineInstr *CPEMI)
Returns whether CPEMI is the first instruction in the block immediately following JTMI (assumed to be...
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:185
uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
Definition: MathExtras.h:616
const MachineInstrBuilder & addOperand(const MachineOperand &MO) const
BasicBlockListType::iterator iterator
#define DEBUG(X)
Definition: Debug.h:92
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:354
ppc ctr loops verify
static bool BBHasFallthrough(MachineBasicBlock *MBB)
BBHasFallthrough - Return true if the specified basic block can fallthrough into the block immediatel...
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
addReg - Add a new virtual register operand...
void addSuccessor(MachineBasicBlock *succ, uint32_t weight=0)
addSuccessor - Add succ as a successor of this MachineBasicBlock.
unsigned pred_size() const
unsigned getAlignment() const
getAlignment - Return alignment of the basic block.