LLVM  14.0.0git
AArch64ConditionalCompares.cpp
Go to the documentation of this file.
1 //===-- AArch64ConditionalCompares.cpp --- CCMP formation for AArch64 -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64ConditionalCompares pass which reduces
10 // branching and code size by using the conditional compare instructions CCMP,
11 // CCMN, and FCMP.
12 //
13 // The CFG transformations for forming conditional compares are very similar to
14 // if-conversion, and this pass should run immediately before the early
15 // if-conversion pass.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "AArch64.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/ADT/SmallPtrSet.h"
23 #include "llvm/ADT/Statistic.h"
32 #include "llvm/CodeGen/Passes.h"
36 #include "llvm/InitializePasses.h"
38 #include "llvm/Support/Debug.h"
40 
41 using namespace llvm;
42 
43 #define DEBUG_TYPE "aarch64-ccmp"
44 
45 // Absolute maximum number of instructions allowed per speculated block.
46 // This bypasses all other heuristics, so it should be set fairly high.
48  "aarch64-ccmp-limit", cl::init(30), cl::Hidden,
49  cl::desc("Maximum number of instructions per speculated block."));
50 
51 // Stress testing mode - disable heuristics.
52 static cl::opt<bool> Stress("aarch64-stress-ccmp", cl::Hidden,
53  cl::desc("Turn all knobs to 11"));
54 
55 STATISTIC(NumConsidered, "Number of ccmps considered");
56 STATISTIC(NumPhiRejs, "Number of ccmps rejected (PHI)");
57 STATISTIC(NumPhysRejs, "Number of ccmps rejected (Physregs)");
58 STATISTIC(NumPhi2Rejs, "Number of ccmps rejected (PHI2)");
59 STATISTIC(NumHeadBranchRejs, "Number of ccmps rejected (Head branch)");
60 STATISTIC(NumCmpBranchRejs, "Number of ccmps rejected (CmpBB branch)");
61 STATISTIC(NumCmpTermRejs, "Number of ccmps rejected (CmpBB is cbz...)");
62 STATISTIC(NumImmRangeRejs, "Number of ccmps rejected (Imm out of range)");
63 STATISTIC(NumLiveDstRejs, "Number of ccmps rejected (Cmp dest live)");
64 STATISTIC(NumMultNZCVUses, "Number of ccmps rejected (NZCV used)");
65 STATISTIC(NumUnknNZCVDefs, "Number of ccmps rejected (NZCV def unknown)");
66 
67 STATISTIC(NumSpeculateRejs, "Number of ccmps rejected (Can't speculate)");
68 
69 STATISTIC(NumConverted, "Number of ccmp instructions created");
70 STATISTIC(NumCompBranches, "Number of cbz/cbnz branches converted");
71 
72 //===----------------------------------------------------------------------===//
73 // SSACCmpConv
74 //===----------------------------------------------------------------------===//
75 //
76 // The SSACCmpConv class performs ccmp-conversion on SSA form machine code
77 // after determining if it is possible. The class contains no heuristics;
78 // external code should be used to determine when ccmp-conversion is a good
79 // idea.
80 //
81 // CCmp-formation works on a CFG representing chained conditions, typically
82 // from C's short-circuit || and && operators:
83 //
84 // From: Head To: Head
85 // / | CmpBB
86 // / | / |
87 // | CmpBB / |
88 // | / | Tail |
89 // | / | | |
90 // Tail | | |
91 // | | | |
92 // ... ... ... ...
93 //
94 // The Head block is terminated by a br.cond instruction, and the CmpBB block
95 // contains compare + br.cond. Tail must be a successor of both.
96 //
97 // The cmp-conversion turns the compare instruction in CmpBB into a conditional
98 // compare, and merges CmpBB into Head, speculatively executing its
99 // instructions. The AArch64 conditional compare instructions have an immediate
100 // operand that specifies the NZCV flag values when the condition is false and
101 // the compare isn't executed. This makes it possible to chain compares with
102 // different condition codes.
103 //
104 // Example:
105 //
106 // if (a == 5 || b == 17)
107 // foo();
108 //
109 // Head:
110 // cmp w0, #5
111 // b.eq Tail
112 // CmpBB:
113 // cmp w1, #17
114 // b.eq Tail
115 // ...
116 // Tail:
117 // bl _foo
118 //
119 // Becomes:
120 //
121 // Head:
122 // cmp w0, #5
123 // ccmp w1, #17, 4, ne ; 4 = nZcv
124 // b.eq Tail
125 // ...
126 // Tail:
127 // bl _foo
128 //
129 // The ccmp condition code is the one that would cause the Head terminator to
130 // branch to CmpBB.
131 //
132 // FIXME: It should also be possible to speculate a block on the critical edge
133 // between Head and Tail, just like if-converting a diamond.
134 //
135 // FIXME: Handle PHIs in Tail by turning them into selects (if-conversion).
136 
137 namespace {
138 class SSACCmpConv {
139  MachineFunction *MF;
140  const TargetInstrInfo *TII;
141  const TargetRegisterInfo *TRI;
143  const MachineBranchProbabilityInfo *MBPI;
144 
145 public:
146  /// The first block containing a conditional branch, dominating everything
147  /// else.
148  MachineBasicBlock *Head;
149 
150  /// The block containing cmp+br.cond with a successor shared with Head.
151  MachineBasicBlock *CmpBB;
152 
153  /// The common successor for Head and CmpBB.
155 
156  /// The compare instruction in CmpBB that can be converted to a ccmp.
157  MachineInstr *CmpMI;
158 
159 private:
160  /// The branch condition in Head as determined by analyzeBranch.
162 
163  /// The condition code that makes Head branch to CmpBB.
164  AArch64CC::CondCode HeadCmpBBCC;
165 
166  /// The branch condition in CmpBB.
168 
169  /// The condition code that makes CmpBB branch to Tail.
170  AArch64CC::CondCode CmpBBTailCC;
171 
172  /// Check if the Tail PHIs are trivially convertible.
173  bool trivialTailPHIs();
174 
175  /// Remove CmpBB from the Tail PHIs.
176  void updateTailPHIs();
177 
178  /// Check if an operand defining DstReg is dead.
179  bool isDeadDef(unsigned DstReg);
180 
181  /// Find the compare instruction in MBB that controls the conditional branch.
182  /// Return NULL if a convertible instruction can't be found.
183  MachineInstr *findConvertibleCompare(MachineBasicBlock *MBB);
184 
185  /// Return true if all non-terminator instructions in MBB can be safely
186  /// speculated.
187  bool canSpeculateInstrs(MachineBasicBlock *MBB, const MachineInstr *CmpMI);
188 
189 public:
190  /// runOnMachineFunction - Initialize per-function data structures.
191  void runOnMachineFunction(MachineFunction &MF,
192  const MachineBranchProbabilityInfo *MBPI) {
193  this->MF = &MF;
194  this->MBPI = MBPI;
195  TII = MF.getSubtarget().getInstrInfo();
197  MRI = &MF.getRegInfo();
198  }
199 
200  /// If the sub-CFG headed by MBB can be cmp-converted, initialize the
201  /// internal state, and return true.
202  bool canConvert(MachineBasicBlock *MBB);
203 
204  /// Cmo-convert the last block passed to canConvertCmp(), assuming
205  /// it is possible. Add any erased blocks to RemovedBlocks.
206  void convert(SmallVectorImpl<MachineBasicBlock *> &RemovedBlocks);
207 
208  /// Return the expected code size delta if the conversion into a
209  /// conditional compare is performed.
210  int expectedCodeSizeDelta() const;
211 };
212 } // end anonymous namespace
213 
214 // Check that all PHIs in Tail are selecting the same value from Head and CmpBB.
215 // This means that no if-conversion is required when merging CmpBB into Head.
216 bool SSACCmpConv::trivialTailPHIs() {
217  for (auto &I : *Tail) {
218  if (!I.isPHI())
219  break;
220  unsigned HeadReg = 0, CmpBBReg = 0;
221  // PHI operands come in (VReg, MBB) pairs.
222  for (unsigned oi = 1, oe = I.getNumOperands(); oi != oe; oi += 2) {
223  MachineBasicBlock *MBB = I.getOperand(oi + 1).getMBB();
224  Register Reg = I.getOperand(oi).getReg();
225  if (MBB == Head) {
226  assert((!HeadReg || HeadReg == Reg) && "Inconsistent PHI operands");
227  HeadReg = Reg;
228  }
229  if (MBB == CmpBB) {
230  assert((!CmpBBReg || CmpBBReg == Reg) && "Inconsistent PHI operands");
231  CmpBBReg = Reg;
232  }
233  }
234  if (HeadReg != CmpBBReg)
235  return false;
236  }
237  return true;
238 }
239 
240 // Assuming that trivialTailPHIs() is true, update the Tail PHIs by simply
241 // removing the CmpBB operands. The Head operands will be identical.
242 void SSACCmpConv::updateTailPHIs() {
243  for (auto &I : *Tail) {
244  if (!I.isPHI())
245  break;
246  // I is a PHI. It can have multiple entries for CmpBB.
247  for (unsigned oi = I.getNumOperands(); oi > 2; oi -= 2) {
248  // PHI operands are (Reg, MBB) at (oi-2, oi-1).
249  if (I.getOperand(oi - 1).getMBB() == CmpBB) {
250  I.RemoveOperand(oi - 1);
251  I.RemoveOperand(oi - 2);
252  }
253  }
254  }
255 }
256 
257 // This pass runs before the AArch64DeadRegisterDefinitions pass, so compares
258 // are still writing virtual registers without any uses.
259 bool SSACCmpConv::isDeadDef(unsigned DstReg) {
260  // Writes to the zero register are dead.
261  if (DstReg == AArch64::WZR || DstReg == AArch64::XZR)
262  return true;
263  if (!Register::isVirtualRegister(DstReg))
264  return false;
265  // A virtual register def without any uses will be marked dead later, and
266  // eventually replaced by the zero register.
267  return MRI->use_nodbg_empty(DstReg);
268 }
269 
270 // Parse a condition code returned by analyzeBranch, and compute the CondCode
271 // corresponding to TBB.
272 // Return
274  // A normal br.cond simply has the condition code.
275  if (Cond[0].getImm() != -1) {
276  assert(Cond.size() == 1 && "Unknown Cond array format");
277  CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
278  return true;
279  }
280  // For tbz and cbz instruction, the opcode is next.
281  switch (Cond[1].getImm()) {
282  default:
283  // This includes tbz / tbnz branches which can't be converted to
284  // ccmp + br.cond.
285  return false;
286  case AArch64::CBZW:
287  case AArch64::CBZX:
288  assert(Cond.size() == 3 && "Unknown Cond array format");
289  CC = AArch64CC::EQ;
290  return true;
291  case AArch64::CBNZW:
292  case AArch64::CBNZX:
293  assert(Cond.size() == 3 && "Unknown Cond array format");
294  CC = AArch64CC::NE;
295  return true;
296  }
297 }
298 
299 MachineInstr *SSACCmpConv::findConvertibleCompare(MachineBasicBlock *MBB) {
301  if (I == MBB->end())
302  return nullptr;
303  // The terminator must be controlled by the flags.
304  if (!I->readsRegister(AArch64::NZCV)) {
305  switch (I->getOpcode()) {
306  case AArch64::CBZW:
307  case AArch64::CBZX:
308  case AArch64::CBNZW:
309  case AArch64::CBNZX:
310  // These can be converted into a ccmp against #0.
311  return &*I;
312  }
313  ++NumCmpTermRejs;
314  LLVM_DEBUG(dbgs() << "Flags not used by terminator: " << *I);
315  return nullptr;
316  }
317 
318  // Now find the instruction controlling the terminator.
319  for (MachineBasicBlock::iterator B = MBB->begin(); I != B;) {
320  I = prev_nodbg(I, MBB->begin());
321  assert(!I->isTerminator() && "Spurious terminator");
322  switch (I->getOpcode()) {
323  // cmp is an alias for subs with a dead destination register.
324  case AArch64::SUBSWri:
325  case AArch64::SUBSXri:
326  // cmn is an alias for adds with a dead destination register.
327  case AArch64::ADDSWri:
328  case AArch64::ADDSXri:
329  // Check that the immediate operand is within range, ccmp wants a uimm5.
330  // Rd = SUBSri Rn, imm, shift
331  if (I->getOperand(3).getImm() || !isUInt<5>(I->getOperand(2).getImm())) {
332  LLVM_DEBUG(dbgs() << "Immediate out of range for ccmp: " << *I);
333  ++NumImmRangeRejs;
334  return nullptr;
335  }
337  case AArch64::SUBSWrr:
338  case AArch64::SUBSXrr:
339  case AArch64::ADDSWrr:
340  case AArch64::ADDSXrr:
341  if (isDeadDef(I->getOperand(0).getReg()))
342  return &*I;
343  LLVM_DEBUG(dbgs() << "Can't convert compare with live destination: "
344  << *I);
345  ++NumLiveDstRejs;
346  return nullptr;
347  case AArch64::FCMPSrr:
348  case AArch64::FCMPDrr:
349  case AArch64::FCMPESrr:
350  case AArch64::FCMPEDrr:
351  return &*I;
352  }
353 
354  // Check for flag reads and clobbers.
355  PhysRegInfo PRI = AnalyzePhysRegInBundle(*I, AArch64::NZCV, TRI);
356 
357  if (PRI.Read) {
358  // The ccmp doesn't produce exactly the same flags as the original
359  // compare, so reject the transform if there are uses of the flags
360  // besides the terminators.
361  LLVM_DEBUG(dbgs() << "Can't create ccmp with multiple uses: " << *I);
362  ++NumMultNZCVUses;
363  return nullptr;
364  }
365 
366  if (PRI.Defined || PRI.Clobbered) {
367  LLVM_DEBUG(dbgs() << "Not convertible compare: " << *I);
368  ++NumUnknNZCVDefs;
369  return nullptr;
370  }
371  }
372  LLVM_DEBUG(dbgs() << "Flags not defined in " << printMBBReference(*MBB)
373  << '\n');
374  return nullptr;
375 }
376 
377 /// Determine if all the instructions in MBB can safely
378 /// be speculated. The terminators are not considered.
379 ///
380 /// Only CmpMI is allowed to clobber the flags.
381 ///
382 bool SSACCmpConv::canSpeculateInstrs(MachineBasicBlock *MBB,
383  const MachineInstr *CmpMI) {
384  // Reject any live-in physregs. It's probably NZCV/EFLAGS, and very hard to
385  // get right.
386  if (!MBB->livein_empty()) {
387  LLVM_DEBUG(dbgs() << printMBBReference(*MBB) << " has live-ins.\n");
388  return false;
389  }
390 
391  unsigned InstrCount = 0;
392 
393  // Check all instructions, except the terminators. It is assumed that
394  // terminators never have side effects or define any used register values.
395  for (auto &I : make_range(MBB->begin(), MBB->getFirstTerminator())) {
396  if (I.isDebugInstr())
397  continue;
398 
399  if (++InstrCount > BlockInstrLimit && !Stress) {
400  LLVM_DEBUG(dbgs() << printMBBReference(*MBB) << " has more than "
401  << BlockInstrLimit << " instructions.\n");
402  return false;
403  }
404 
405  // There shouldn't normally be any phis in a single-predecessor block.
406  if (I.isPHI()) {
407  LLVM_DEBUG(dbgs() << "Can't hoist: " << I);
408  return false;
409  }
410 
411  // Don't speculate loads. Note that it may be possible and desirable to
412  // speculate GOT or constant pool loads that are guaranteed not to trap,
413  // but we don't support that for now.
414  if (I.mayLoad()) {
415  LLVM_DEBUG(dbgs() << "Won't speculate load: " << I);
416  return false;
417  }
418 
419  // We never speculate stores, so an AA pointer isn't necessary.
420  bool DontMoveAcrossStore = true;
421  if (!I.isSafeToMove(nullptr, DontMoveAcrossStore)) {
422  LLVM_DEBUG(dbgs() << "Can't speculate: " << I);
423  return false;
424  }
425 
426  // Only CmpMI is allowed to clobber the flags.
427  if (&I != CmpMI && I.modifiesRegister(AArch64::NZCV, TRI)) {
428  LLVM_DEBUG(dbgs() << "Clobbers flags: " << I);
429  return false;
430  }
431  }
432  return true;
433 }
434 
435 /// Analyze the sub-cfg rooted in MBB, and return true if it is a potential
436 /// candidate for cmp-conversion. Fill out the internal state.
437 ///
438 bool SSACCmpConv::canConvert(MachineBasicBlock *MBB) {
439  Head = MBB;
440  Tail = CmpBB = nullptr;
441 
442  if (Head->succ_size() != 2)
443  return false;
444  MachineBasicBlock *Succ0 = Head->succ_begin()[0];
445  MachineBasicBlock *Succ1 = Head->succ_begin()[1];
446 
447  // CmpBB can only have a single predecessor. Tail is allowed many.
448  if (Succ0->pred_size() != 1)
449  std::swap(Succ0, Succ1);
450 
451  // Succ0 is our candidate for CmpBB.
452  if (Succ0->pred_size() != 1 || Succ0->succ_size() != 2)
453  return false;
454 
455  CmpBB = Succ0;
456  Tail = Succ1;
457 
458  if (!CmpBB->isSuccessor(Tail))
459  return false;
460 
461  // The CFG topology checks out.
462  LLVM_DEBUG(dbgs() << "\nTriangle: " << printMBBReference(*Head) << " -> "
463  << printMBBReference(*CmpBB) << " -> "
464  << printMBBReference(*Tail) << '\n');
465  ++NumConsidered;
466 
467  // Tail is allowed to have many predecessors, but we can't handle PHIs yet.
468  //
469  // FIXME: Real PHIs could be if-converted as long as the CmpBB values are
470  // defined before The CmpBB cmp clobbers the flags. Alternatively, it should
471  // always be safe to sink the ccmp down to immediately before the CmpBB
472  // terminators.
473  if (!trivialTailPHIs()) {
474  LLVM_DEBUG(dbgs() << "Can't handle phis in Tail.\n");
475  ++NumPhiRejs;
476  return false;
477  }
478 
479  if (!Tail->livein_empty()) {
480  LLVM_DEBUG(dbgs() << "Can't handle live-in physregs in Tail.\n");
481  ++NumPhysRejs;
482  return false;
483  }
484 
485  // CmpBB should never have PHIs since Head is its only predecessor.
486  // FIXME: Clean them up if it happens.
487  if (!CmpBB->empty() && CmpBB->front().isPHI()) {
488  LLVM_DEBUG(dbgs() << "Can't handle phis in CmpBB.\n");
489  ++NumPhi2Rejs;
490  return false;
491  }
492 
493  if (!CmpBB->livein_empty()) {
494  LLVM_DEBUG(dbgs() << "Can't handle live-in physregs in CmpBB.\n");
495  ++NumPhysRejs;
496  return false;
497  }
498 
499  // The branch we're looking to eliminate must be analyzable.
500  HeadCond.clear();
501  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
502  if (TII->analyzeBranch(*Head, TBB, FBB, HeadCond)) {
503  LLVM_DEBUG(dbgs() << "Head branch not analyzable.\n");
504  ++NumHeadBranchRejs;
505  return false;
506  }
507 
508  // This is weird, probably some sort of degenerate CFG, or an edge to a
509  // landing pad.
510  if (!TBB || HeadCond.empty()) {
511  LLVM_DEBUG(
512  dbgs() << "analyzeBranch didn't find conditional branch in Head.\n");
513  ++NumHeadBranchRejs;
514  return false;
515  }
516 
517  if (!parseCond(HeadCond, HeadCmpBBCC)) {
518  LLVM_DEBUG(dbgs() << "Unsupported branch type on Head\n");
519  ++NumHeadBranchRejs;
520  return false;
521  }
522 
523  // Make sure the branch direction is right.
524  if (TBB != CmpBB) {
525  assert(TBB == Tail && "Unexpected TBB");
526  HeadCmpBBCC = AArch64CC::getInvertedCondCode(HeadCmpBBCC);
527  }
528 
529  CmpBBCond.clear();
530  TBB = FBB = nullptr;
531  if (TII->analyzeBranch(*CmpBB, TBB, FBB, CmpBBCond)) {
532  LLVM_DEBUG(dbgs() << "CmpBB branch not analyzable.\n");
533  ++NumCmpBranchRejs;
534  return false;
535  }
536 
537  if (!TBB || CmpBBCond.empty()) {
538  LLVM_DEBUG(
539  dbgs() << "analyzeBranch didn't find conditional branch in CmpBB.\n");
540  ++NumCmpBranchRejs;
541  return false;
542  }
543 
544  if (!parseCond(CmpBBCond, CmpBBTailCC)) {
545  LLVM_DEBUG(dbgs() << "Unsupported branch type on CmpBB\n");
546  ++NumCmpBranchRejs;
547  return false;
548  }
549 
550  if (TBB != Tail)
551  CmpBBTailCC = AArch64CC::getInvertedCondCode(CmpBBTailCC);
552 
553  LLVM_DEBUG(dbgs() << "Head->CmpBB on "
554  << AArch64CC::getCondCodeName(HeadCmpBBCC)
555  << ", CmpBB->Tail on "
556  << AArch64CC::getCondCodeName(CmpBBTailCC) << '\n');
557 
558  CmpMI = findConvertibleCompare(CmpBB);
559  if (!CmpMI)
560  return false;
561 
562  if (!canSpeculateInstrs(CmpBB, CmpMI)) {
563  ++NumSpeculateRejs;
564  return false;
565  }
566  return true;
567 }
568 
569 void SSACCmpConv::convert(SmallVectorImpl<MachineBasicBlock *> &RemovedBlocks) {
570  LLVM_DEBUG(dbgs() << "Merging " << printMBBReference(*CmpBB) << " into "
571  << printMBBReference(*Head) << ":\n"
572  << *CmpBB);
573 
574  // All CmpBB instructions are moved into Head, and CmpBB is deleted.
575  // Update the CFG first.
576  updateTailPHIs();
577 
578  // Save successor probabilties before removing CmpBB and Tail from their
579  // parents.
580  BranchProbability Head2CmpBB = MBPI->getEdgeProbability(Head, CmpBB);
581  BranchProbability CmpBB2Tail = MBPI->getEdgeProbability(CmpBB, Tail);
582 
583  Head->removeSuccessor(CmpBB);
584  CmpBB->removeSuccessor(Tail);
585 
586  // If Head and CmpBB had successor probabilties, udpate the probabilities to
587  // reflect the ccmp-conversion.
588  if (Head->hasSuccessorProbabilities() && CmpBB->hasSuccessorProbabilities()) {
589 
590  // Head is allowed two successors. We've removed CmpBB, so the remaining
591  // successor is Tail. We need to increase the successor probability for
592  // Tail to account for the CmpBB path we removed.
593  //
594  // Pr(Tail|Head) += Pr(CmpBB|Head) * Pr(Tail|CmpBB).
595  assert(*Head->succ_begin() == Tail && "Head successor is not Tail");
596  BranchProbability Head2Tail = MBPI->getEdgeProbability(Head, Tail);
597  Head->setSuccProbability(Head->succ_begin(),
598  Head2Tail + Head2CmpBB * CmpBB2Tail);
599 
600  // We will transfer successors of CmpBB to Head in a moment without
601  // normalizing the successor probabilities. Set the successor probabilites
602  // before doing so.
603  //
604  // Pr(I|Head) = Pr(CmpBB|Head) * Pr(I|CmpBB).
605  for (auto I = CmpBB->succ_begin(), E = CmpBB->succ_end(); I != E; ++I) {
606  BranchProbability CmpBB2I = MBPI->getEdgeProbability(CmpBB, *I);
607  CmpBB->setSuccProbability(I, Head2CmpBB * CmpBB2I);
608  }
609  }
610 
611  Head->transferSuccessorsAndUpdatePHIs(CmpBB);
612  DebugLoc TermDL = Head->getFirstTerminator()->getDebugLoc();
613  TII->removeBranch(*Head);
614 
615  // If the Head terminator was one of the cbz / tbz branches with built-in
616  // compare, we need to insert an explicit compare instruction in its place.
617  if (HeadCond[0].getImm() == -1) {
618  ++NumCompBranches;
619  unsigned Opc = 0;
620  switch (HeadCond[1].getImm()) {
621  case AArch64::CBZW:
622  case AArch64::CBNZW:
623  Opc = AArch64::SUBSWri;
624  break;
625  case AArch64::CBZX:
626  case AArch64::CBNZX:
627  Opc = AArch64::SUBSXri;
628  break;
629  default:
630  llvm_unreachable("Cannot convert Head branch");
631  }
632  const MCInstrDesc &MCID = TII->get(Opc);
633  // Create a dummy virtual register for the SUBS def.
634  Register DestReg =
635  MRI->createVirtualRegister(TII->getRegClass(MCID, 0, TRI, *MF));
636  // Insert a SUBS Rn, #0 instruction instead of the cbz / cbnz.
637  BuildMI(*Head, Head->end(), TermDL, MCID)
639  .add(HeadCond[2])
640  .addImm(0)
641  .addImm(0);
642  // SUBS uses the GPR*sp register classes.
643  MRI->constrainRegClass(HeadCond[2].getReg(),
644  TII->getRegClass(MCID, 1, TRI, *MF));
645  }
646 
647  Head->splice(Head->end(), CmpBB, CmpBB->begin(), CmpBB->end());
648 
649  // Now replace CmpMI with a ccmp instruction that also considers the incoming
650  // flags.
651  unsigned Opc = 0;
652  unsigned FirstOp = 1; // First CmpMI operand to copy.
653  bool isZBranch = false; // CmpMI is a cbz/cbnz instruction.
654  switch (CmpMI->getOpcode()) {
655  default:
656  llvm_unreachable("Unknown compare opcode");
657  case AArch64::SUBSWri: Opc = AArch64::CCMPWi; break;
658  case AArch64::SUBSWrr: Opc = AArch64::CCMPWr; break;
659  case AArch64::SUBSXri: Opc = AArch64::CCMPXi; break;
660  case AArch64::SUBSXrr: Opc = AArch64::CCMPXr; break;
661  case AArch64::ADDSWri: Opc = AArch64::CCMNWi; break;
662  case AArch64::ADDSWrr: Opc = AArch64::CCMNWr; break;
663  case AArch64::ADDSXri: Opc = AArch64::CCMNXi; break;
664  case AArch64::ADDSXrr: Opc = AArch64::CCMNXr; break;
665  case AArch64::FCMPSrr: Opc = AArch64::FCCMPSrr; FirstOp = 0; break;
666  case AArch64::FCMPDrr: Opc = AArch64::FCCMPDrr; FirstOp = 0; break;
667  case AArch64::FCMPESrr: Opc = AArch64::FCCMPESrr; FirstOp = 0; break;
668  case AArch64::FCMPEDrr: Opc = AArch64::FCCMPEDrr; FirstOp = 0; break;
669  case AArch64::CBZW:
670  case AArch64::CBNZW:
671  Opc = AArch64::CCMPWi;
672  FirstOp = 0;
673  isZBranch = true;
674  break;
675  case AArch64::CBZX:
676  case AArch64::CBNZX:
677  Opc = AArch64::CCMPXi;
678  FirstOp = 0;
679  isZBranch = true;
680  break;
681  }
682 
683  // The ccmp instruction should set the flags according to the comparison when
684  // Head would have branched to CmpBB.
685  // The NZCV immediate operand should provide flags for the case where Head
686  // would have branched to Tail. These flags should cause the new Head
687  // terminator to branch to tail.
688  unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(CmpBBTailCC);
689  const MCInstrDesc &MCID = TII->get(Opc);
690  MRI->constrainRegClass(CmpMI->getOperand(FirstOp).getReg(),
691  TII->getRegClass(MCID, 0, TRI, *MF));
692  if (CmpMI->getOperand(FirstOp + 1).isReg())
693  MRI->constrainRegClass(CmpMI->getOperand(FirstOp + 1).getReg(),
694  TII->getRegClass(MCID, 1, TRI, *MF));
695  MachineInstrBuilder MIB = BuildMI(*Head, CmpMI, CmpMI->getDebugLoc(), MCID)
696  .add(CmpMI->getOperand(FirstOp)); // Register Rn
697  if (isZBranch)
698  MIB.addImm(0); // cbz/cbnz Rn -> ccmp Rn, #0
699  else
700  MIB.add(CmpMI->getOperand(FirstOp + 1)); // Register Rm / Immediate
701  MIB.addImm(NZCV).addImm(HeadCmpBBCC);
702 
703  // If CmpMI was a terminator, we need a new conditional branch to replace it.
704  // This now becomes a Head terminator.
705  if (isZBranch) {
706  bool isNZ = CmpMI->getOpcode() == AArch64::CBNZW ||
707  CmpMI->getOpcode() == AArch64::CBNZX;
708  BuildMI(*Head, CmpMI, CmpMI->getDebugLoc(), TII->get(AArch64::Bcc))
710  .add(CmpMI->getOperand(1)); // Branch target.
711  }
712  CmpMI->eraseFromParent();
713  Head->updateTerminator(CmpBB->getNextNode());
714 
715  RemovedBlocks.push_back(CmpBB);
716  CmpBB->eraseFromParent();
717  LLVM_DEBUG(dbgs() << "Result:\n" << *Head);
718  ++NumConverted;
719 }
720 
721 int SSACCmpConv::expectedCodeSizeDelta() const {
722  int delta = 0;
723  // If the Head terminator was one of the cbz / tbz branches with built-in
724  // compare, we need to insert an explicit compare instruction in its place
725  // plus a branch instruction.
726  if (HeadCond[0].getImm() == -1) {
727  switch (HeadCond[1].getImm()) {
728  case AArch64::CBZW:
729  case AArch64::CBNZW:
730  case AArch64::CBZX:
731  case AArch64::CBNZX:
732  // Therefore delta += 1
733  delta = 1;
734  break;
735  default:
736  llvm_unreachable("Cannot convert Head branch");
737  }
738  }
739  // If the Cmp terminator was one of the cbz / tbz branches with
740  // built-in compare, it will be turned into a compare instruction
741  // into Head, but we do not save any instruction.
742  // Otherwise, we save the branch instruction.
743  switch (CmpMI->getOpcode()) {
744  default:
745  --delta;
746  break;
747  case AArch64::CBZW:
748  case AArch64::CBNZW:
749  case AArch64::CBZX:
750  case AArch64::CBNZX:
751  break;
752  }
753  return delta;
754 }
755 
756 //===----------------------------------------------------------------------===//
757 // AArch64ConditionalCompares Pass
758 //===----------------------------------------------------------------------===//
759 
760 namespace {
761 class AArch64ConditionalCompares : public MachineFunctionPass {
762  const MachineBranchProbabilityInfo *MBPI;
763  const TargetInstrInfo *TII;
764  const TargetRegisterInfo *TRI;
765  MCSchedModel SchedModel;
766  // Does the proceeded function has Oz attribute.
767  bool MinSize;
769  MachineDominatorTree *DomTree;
771  MachineTraceMetrics *Traces;
773  SSACCmpConv CmpConv;
774 
775 public:
776  static char ID;
777  AArch64ConditionalCompares() : MachineFunctionPass(ID) {
779  }
780  void getAnalysisUsage(AnalysisUsage &AU) const override;
781  bool runOnMachineFunction(MachineFunction &MF) override;
782  StringRef getPassName() const override {
783  return "AArch64 Conditional Compares";
784  }
785 
786 private:
787  bool tryConvert(MachineBasicBlock *);
788  void updateDomTree(ArrayRef<MachineBasicBlock *> Removed);
789  void updateLoops(ArrayRef<MachineBasicBlock *> Removed);
790  void invalidateTraces();
791  bool shouldConvert();
792 };
793 } // end anonymous namespace
794 
796 
797 INITIALIZE_PASS_BEGIN(AArch64ConditionalCompares, "aarch64-ccmp",
798  "AArch64 CCMP Pass", false, false)
802 INITIALIZE_PASS_END(AArch64ConditionalCompares, "aarch64-ccmp",
803  "AArch64 CCMP Pass", false, false)
804 
806  return new AArch64ConditionalCompares();
807 }
808 
809 void AArch64ConditionalCompares::getAnalysisUsage(AnalysisUsage &AU) const {
818 }
819 
820 /// Update the dominator tree after if-conversion erased some blocks.
821 void AArch64ConditionalCompares::updateDomTree(
823  // convert() removes CmpBB which was previously dominated by Head.
824  // CmpBB children should be transferred to Head.
825  MachineDomTreeNode *HeadNode = DomTree->getNode(CmpConv.Head);
826  for (MachineBasicBlock *RemovedMBB : Removed) {
827  MachineDomTreeNode *Node = DomTree->getNode(RemovedMBB);
828  assert(Node != HeadNode && "Cannot erase the head node");
829  assert(Node->getIDom() == HeadNode && "CmpBB should be dominated by Head");
830  while (Node->getNumChildren())
831  DomTree->changeImmediateDominator(Node->back(), HeadNode);
832  DomTree->eraseNode(RemovedMBB);
833  }
834 }
835 
836 /// Update LoopInfo after if-conversion.
837 void
838 AArch64ConditionalCompares::updateLoops(ArrayRef<MachineBasicBlock *> Removed) {
839  if (!Loops)
840  return;
841  for (MachineBasicBlock *RemovedMBB : Removed)
842  Loops->removeBlock(RemovedMBB);
843 }
844 
845 /// Invalidate MachineTraceMetrics before if-conversion.
846 void AArch64ConditionalCompares::invalidateTraces() {
847  Traces->invalidate(CmpConv.Head);
848  Traces->invalidate(CmpConv.CmpBB);
849 }
850 
851 /// Apply cost model and heuristics to the if-conversion in IfConv.
852 /// Return true if the conversion is a good idea.
853 ///
855  // Stress testing mode disables all cost considerations.
856  if (Stress)
857  return true;
858  if (!MinInstr)
859  MinInstr = Traces->getEnsemble(MachineTraceMetrics::TS_MinInstrCount);
860 
861  // Head dominates CmpBB, so it is always included in its trace.
862  MachineTraceMetrics::Trace Trace = MinInstr->getTrace(CmpConv.CmpBB);
863 
864  // If code size is the main concern
865  if (MinSize) {
866  int CodeSizeDelta = CmpConv.expectedCodeSizeDelta();
867  LLVM_DEBUG(dbgs() << "Code size delta: " << CodeSizeDelta << '\n');
868  // If we are minimizing the code size, do the conversion whatever
869  // the cost is.
870  if (CodeSizeDelta < 0)
871  return true;
872  if (CodeSizeDelta > 0) {
873  LLVM_DEBUG(dbgs() << "Code size is increasing, give up on this one.\n");
874  return false;
875  }
876  // CodeSizeDelta == 0, continue with the regular heuristics
877  }
878 
879  // Heuristic: The compare conversion delays the execution of the branch
880  // instruction because we must wait for the inputs to the second compare as
881  // well. The branch has no dependent instructions, but delaying it increases
882  // the cost of a misprediction.
883  //
884  // Set a limit on the delay we will accept.
885  unsigned DelayLimit = SchedModel.MispredictPenalty * 3 / 4;
886 
887  // Instruction depths can be computed for all trace instructions above CmpBB.
888  unsigned HeadDepth =
889  Trace.getInstrCycles(*CmpConv.Head->getFirstTerminator()).Depth;
890  unsigned CmpBBDepth =
891  Trace.getInstrCycles(*CmpConv.CmpBB->getFirstTerminator()).Depth;
892  LLVM_DEBUG(dbgs() << "Head depth: " << HeadDepth
893  << "\nCmpBB depth: " << CmpBBDepth << '\n');
894  if (CmpBBDepth > HeadDepth + DelayLimit) {
895  LLVM_DEBUG(dbgs() << "Branch delay would be larger than " << DelayLimit
896  << " cycles.\n");
897  return false;
898  }
899 
900  // Check the resource depth at the bottom of CmpBB - these instructions will
901  // be speculated.
902  unsigned ResDepth = Trace.getResourceDepth(true);
903  LLVM_DEBUG(dbgs() << "Resources: " << ResDepth << '\n');
904 
905  // Heuristic: The speculatively executed instructions must all be able to
906  // merge into the Head block. The Head critical path should dominate the
907  // resource cost of the speculated instructions.
908  if (ResDepth > HeadDepth) {
909  LLVM_DEBUG(dbgs() << "Too many instructions to speculate.\n");
910  return false;
911  }
912  return true;
913 }
914 
915 bool AArch64ConditionalCompares::tryConvert(MachineBasicBlock *MBB) {
916  bool Changed = false;
917  while (CmpConv.canConvert(MBB) && shouldConvert()) {
918  invalidateTraces();
920  CmpConv.convert(RemovedBlocks);
921  Changed = true;
922  updateDomTree(RemovedBlocks);
923  updateLoops(RemovedBlocks);
924  }
925  return Changed;
926 }
927 
928 bool AArch64ConditionalCompares::runOnMachineFunction(MachineFunction &MF) {
929  LLVM_DEBUG(dbgs() << "********** AArch64 Conditional Compares **********\n"
930  << "********** Function: " << MF.getName() << '\n');
931  if (skipFunction(MF.getFunction()))
932  return false;
933 
934  TII = MF.getSubtarget().getInstrInfo();
936  SchedModel = MF.getSubtarget().getSchedModel();
937  MRI = &MF.getRegInfo();
938  DomTree = &getAnalysis<MachineDominatorTree>();
939  Loops = getAnalysisIfAvailable<MachineLoopInfo>();
940  MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
941  Traces = &getAnalysis<MachineTraceMetrics>();
942  MinInstr = nullptr;
943  MinSize = MF.getFunction().hasMinSize();
944 
945  bool Changed = false;
946  CmpConv.runOnMachineFunction(MF, MBPI);
947 
948  // Visit blocks in dominator tree pre-order. The pre-order enables multiple
949  // cmp-conversions from the same head block.
950  // Note that updateDomTree() modifies the children of the DomTree node
951  // currently being visited. The df_iterator supports that; it doesn't look at
952  // child_begin() / child_end() until after a node has been visited.
953  for (auto *I : depth_first(DomTree))
954  if (tryConvert(I->getBlock()))
955  Changed = true;
956 
957  return Changed;
958 }
llvm::MachineBasicBlock::succ_size
unsigned succ_size() const
Definition: MachineBasicBlock.h:344
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This file implements support for optimizing divisions by a constant.
Definition: AllocatorList.h:23
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
AArch64.h
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::AArch64CC::NE
@ NE
Definition: AArch64BaseInfo.h:256
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:158
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::HexagonInstrInfo::removeBranch
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
Remove the branching code at the end of the specific MBB.
Definition: HexagonInstrInfo.cpp:561
llvm::AArch64CC::getNZCVToSatisfyCondCode
static unsigned getNZCVToSatisfyCondCode(CondCode Code)
Given a condition code, return NZCV flags that would satisfy that condition.
Definition: AArch64BaseInfo.h:313
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::MCSubtargetInfo::getSchedModel
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
Definition: MCSubtargetInfo.h:162
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:224
llvm::TargetSubtargetInfo::getInstrInfo
virtual const TargetInstrInfo * getInstrInfo() const
Definition: TargetSubtargetInfo.h:92
llvm::HexagonInstrInfo::analyzeBranch
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e....
Definition: HexagonInstrInfo.cpp:391
Loops
Hexagon Hardware Loops
Definition: HexagonHardwareLoops.cpp:372
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Statistic.h
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:119
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::MachineBasicBlock::setSuccProbability
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
Definition: MachineBasicBlock.cpp:1456
llvm::TargetSubtargetInfo::getRegisterInfo
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
Definition: TargetSubtargetInfo.h:124
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::MachineTraceMetrics::TS_MinInstrCount
@ TS_MinInstrCount
Select the trace through a block that has the fewest instructions.
Definition: MachineTraceMetrics.h:379
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:233
InstrCount
static unsigned InstrCount
Definition: DFAPacketizer.cpp:53
TargetInstrInfo.h
llvm::AArch64ISD::CCMP
@ CCMP
Definition: AArch64ISelLowering.h:141
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(AArch64ConditionalCompares, "aarch64-ccmp", "AArch64 CCMP Pass", false, false) INITIALIZE_PASS_END(AArch64ConditionalCompares
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::AnalyzePhysRegInBundle
PhysRegInfo AnalyzePhysRegInBundle(const MachineInstr &MI, Register Reg, const TargetRegisterInfo *TRI)
AnalyzePhysRegInBundle - Analyze how the current instruction or bundle uses a physical register.
Definition: MachineInstrBundle.cpp:308
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:102
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
DepthFirstIterator.h
llvm::MachineLoopInfo
Definition: MachineLoopInfo.h:90
MachineRegisterInfo.h
MachineTraceMetrics.h
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::PhysRegInfo::Clobbered
bool Clobbered
There is a regmask operand indicating Reg is clobbered.
Definition: MachineInstrBundle.h:249
CommandLine.h
llvm::MachineBasicBlock::pred_size
unsigned pred_size() const
Definition: MachineBasicBlock.h:328
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:644
llvm::CallingConv::Tail
@ Tail
Tail - This calling convention attemps to make calls as fast as possible while guaranteeing that tail...
Definition: CallingConv.h:81
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
MachineLoopInfo.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::MachineBranchProbabilityInfo
Definition: MachineBranchProbabilityInfo.h:24
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineBasicBlock::succ_end
succ_iterator succ_end()
Definition: MachineBasicBlock.h:334
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:499
llvm::MachineBasicBlock::isSuccessor
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
Definition: MachineBasicBlock.cpp:913
llvm::MachineBasicBlock::eraseFromParent
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
Definition: MachineBasicBlock.cpp:1337
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::PhysRegInfo::Read
bool Read
Reg or one of its aliases is read.
Definition: MachineInstrBundle.h:260
false
Definition: StackSlotColoring.cpp:142
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:195
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::AArch64CC::getCondCodeName
static const char * getCondCodeName(CondCode Code)
Definition: AArch64BaseInfo.h:281
SmallPtrSet.h
llvm::MachineTraceMetrics::Trace
A trace represents a plausible sequence of executed basic blocks that passes through the current basi...
Definition: MachineTraceMetrics.h:255
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
Passes.h
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:634
llvm::cl::opt
Definition: CommandLine.h:1432
llvm::MachineBasicBlock::hasSuccessorProbabilities
bool hasSuccessorProbabilities() const
Return true if any of the successors have probabilities attached to them.
Definition: MachineBasicBlock.h:702
llvm::MachineInstr::getDebugLoc
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:418
llvm::initializeAArch64ConditionalComparesPass
void initializeAArch64ConditionalComparesPass(PassRegistry &)
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:321
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
llvm::prev_nodbg
IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)
Decrement It, then continue decrementing it while it points to a debug instruction.
Definition: MachineBasicBlock.h:1237
MachineFunctionPass.h
ccmp
aarch64 ccmp
Definition: AArch64ConditionalCompares.cpp:802
llvm::Register::isVirtualRegister
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
llvm::MachineFunction::getName
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Definition: MachineFunction.cpp:542
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MachineBranchProbabilityInfo.h
llvm::MachineBasicBlock::succ_begin
succ_iterator succ_begin()
Definition: MachineBasicBlock.h:332
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::PhysRegInfo
Information about how a physical register Reg is used by a set of operands.
Definition: MachineInstrBundle.h:246
llvm::MachineInstr::isPHI
bool isPHI() const
Definition: MachineInstr.h:1255
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:360
llvm::MachineBranchProbabilityInfo::getEdgeProbability
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
Definition: MachineBranchProbabilityInfo.cpp:58
llvm::MachineFunction
Definition: MachineFunction.h:234
llvm::MachineRegisterInfo::use_nodbg_empty
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
Definition: MachineRegisterInfo.h:566
llvm::MachineBasicBlock::getFirstTerminator
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
Definition: MachineBasicBlock.cpp:242
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:179
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:950
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:489
llvm::PhysRegInfo::Defined
bool Defined
Reg or one of its aliases is defined.
Definition: MachineInstrBundle.h:253
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
parseCond
static bool parseCond(ArrayRef< MachineOperand > Cond, AArch64CC::CondCode &CC)
Definition: AArch64ConditionalCompares.cpp:273
llvm::BranchProbability
Definition: BranchProbability.h:30
TargetSubtargetInfo.h
llvm::DomTreeNodeBase
Base class for the actual dominator tree node.
Definition: LiveIntervalCalc.h:24
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:286
llvm::AArch64CC::EQ
@ EQ
Definition: AArch64BaseInfo.h:255
llvm::Trace
Definition: Trace.h:30
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:229
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::MachineTraceMetrics::Ensemble
A trace ensemble is a collection of traces selected using the same strategy, for example 'minimum res...
Definition: MachineTraceMetrics.h:321
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:600
llvm::MachineBasicBlock::removeSuccessor
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
Definition: MachineBasicBlock.cpp:789
llvm::MachineBasicBlock::front
MachineInstr & front()
Definition: MachineBasicBlock.h:247
llvm::MCSchedModel
Machine model for scheduling, bundling, and heuristics.
Definition: MCSchedule.h:244
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:585
llvm::AArch64CC::getInvertedCondCode
static CondCode getInvertedCondCode(CondCode Code)
Definition: AArch64BaseInfo.h:303
llvm::MachineBasicBlock::livein_empty
bool livein_empty() const
Definition: MachineBasicBlock.h:411
Stress
static cl::opt< bool > Stress("aarch64-stress-ccmp", cl::Hidden, cl::desc("Turn all knobs to 11"))
llvm::Pass
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
MachineInstrBuilder.h
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::MachineBasicBlock::empty
bool empty() const
Definition: MachineBasicBlock.h:240
llvm::MachineRegisterInfo::constrainRegClass
const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
Definition: MachineRegisterInfo.cpp:85
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::AArch64CC::CondCode
CondCode
Definition: AArch64BaseInfo.h:254
llvm::Function::hasMinSize
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:669
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:890
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:412
llvm::MachineBasicBlock::updateTerminator
void updateTerminator(MachineBasicBlock *PreviousLayoutSuccessor)
Update the terminator instructions in block to account for changes to block layout which may have bee...
Definition: MachineBasicBlock.cpp:646
shouldConvert
static bool shouldConvert(Constant &C, AArch64PromoteConstant::PromotionCacheTy &PromotionCache)
Definition: AArch64PromoteConstant.cpp:362
raw_ostream.h
llvm::createAArch64ConditionalCompares
FunctionPass * createAArch64ConditionalCompares()
Definition: AArch64ConditionalCompares.cpp:805
llvm::MachineDominatorTree
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
Definition: MachineDominators.h:45
MachineFunction.h
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition: MachineInstr.cpp:680
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::MachineTraceMetrics
Definition: MachineTraceMetrics.h:87
InitializePasses.h
TargetRegisterInfo.h
Debug.h
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:270
SetVector.h
getReg
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
Definition: MipsDisassembler.cpp:572
MachineDominators.h
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37
BlockInstrLimit
static cl::opt< unsigned > BlockInstrLimit("aarch64-ccmp-limit", cl::init(30), cl::Hidden, cl::desc("Maximum number of instructions per speculated block."))