LLVM  4.0.0
InductiveRangeCheckElimination.cpp
Go to the documentation of this file.
1 //===-- InductiveRangeCheckElimination.cpp - ------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // The InductiveRangeCheckElimination pass splits a loop's iteration space into
10 // three disjoint ranges. It does that in a way such that the loop running in
11 // the middle loop provably does not need range checks. As an example, it will
12 // convert
13 //
14 // len = < known positive >
15 // for (i = 0; i < n; i++) {
16 // if (0 <= i && i < len) {
17 // do_something();
18 // } else {
19 // throw_out_of_bounds();
20 // }
21 // }
22 //
23 // to
24 //
25 // len = < known positive >
26 // limit = smin(n, len)
27 // // no first segment
28 // for (i = 0; i < limit; i++) {
29 // if (0 <= i && i < len) { // this check is fully redundant
30 // do_something();
31 // } else {
32 // throw_out_of_bounds();
33 // }
34 // }
35 // for (i = limit; i < n; i++) {
36 // if (0 <= i && i < len) {
37 // do_something();
38 // } else {
39 // throw_out_of_bounds();
40 // }
41 // }
42 //===----------------------------------------------------------------------===//
43 
44 #include "llvm/ADT/Optional.h"
46 #include "llvm/Analysis/LoopInfo.h"
47 #include "llvm/Analysis/LoopPass.h"
51 #include "llvm/IR/Dominators.h"
52 #include "llvm/IR/Function.h"
53 #include "llvm/IR/IRBuilder.h"
54 #include "llvm/IR/Instructions.h"
55 #include "llvm/IR/PatternMatch.h"
56 #include "llvm/Pass.h"
57 #include "llvm/Support/Debug.h"
59 #include "llvm/Transforms/Scalar.h"
64 
65 using namespace llvm;
66 
67 static cl::opt<unsigned> LoopSizeCutoff("irce-loop-size-cutoff", cl::Hidden,
68  cl::init(64));
69 
70 static cl::opt<bool> PrintChangedLoops("irce-print-changed-loops", cl::Hidden,
71  cl::init(false));
72 
73 static cl::opt<bool> PrintRangeChecks("irce-print-range-checks", cl::Hidden,
74  cl::init(false));
75 
76 static cl::opt<int> MaxExitProbReciprocal("irce-max-exit-prob-reciprocal",
77  cl::Hidden, cl::init(10));
78 
79 static cl::opt<bool> SkipProfitabilityChecks("irce-skip-profitability-checks",
80  cl::Hidden, cl::init(false));
81 
82 static const char *ClonedLoopTag = "irce.loop.clone";
83 
84 #define DEBUG_TYPE "irce"
85 
86 namespace {
87 
88 /// An inductive range check is conditional branch in a loop with
89 ///
90 /// 1. a very cold successor (i.e. the branch jumps to that successor very
91 /// rarely)
92 ///
93 /// and
94 ///
95 /// 2. a condition that is provably true for some contiguous range of values
96 /// taken by the containing loop's induction variable.
97 ///
98 class InductiveRangeCheck {
99  // Classifies a range check
100  enum RangeCheckKind : unsigned {
101  // Range check of the form "0 <= I".
102  RANGE_CHECK_LOWER = 1,
103 
104  // Range check of the form "I < L" where L is known positive.
105  RANGE_CHECK_UPPER = 2,
106 
107  // The logical and of the RANGE_CHECK_LOWER and RANGE_CHECK_UPPER
108  // conditions.
109  RANGE_CHECK_BOTH = RANGE_CHECK_LOWER | RANGE_CHECK_UPPER,
110 
111  // Unrecognized range check condition.
112  RANGE_CHECK_UNKNOWN = (unsigned)-1
113  };
114 
115  static StringRef rangeCheckKindToStr(RangeCheckKind);
116 
117  const SCEV *Offset = nullptr;
118  const SCEV *Scale = nullptr;
119  Value *Length = nullptr;
120  Use *CheckUse = nullptr;
121  RangeCheckKind Kind = RANGE_CHECK_UNKNOWN;
122 
123  static RangeCheckKind parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
124  ScalarEvolution &SE, Value *&Index,
125  Value *&Length);
126 
127  static void
128  extractRangeChecksFromCond(Loop *L, ScalarEvolution &SE, Use &ConditionUse,
130  SmallPtrSetImpl<Value *> &Visited);
131 
132 public:
133  const SCEV *getOffset() const { return Offset; }
134  const SCEV *getScale() const { return Scale; }
135  Value *getLength() const { return Length; }
136 
137  void print(raw_ostream &OS) const {
138  OS << "InductiveRangeCheck:\n";
139  OS << " Kind: " << rangeCheckKindToStr(Kind) << "\n";
140  OS << " Offset: ";
141  Offset->print(OS);
142  OS << " Scale: ";
143  Scale->print(OS);
144  OS << " Length: ";
145  if (Length)
146  Length->print(OS);
147  else
148  OS << "(null)";
149  OS << "\n CheckUse: ";
150  getCheckUse()->getUser()->print(OS);
151  OS << " Operand: " << getCheckUse()->getOperandNo() << "\n";
152  }
153 
155  void dump() {
156  print(dbgs());
157  }
158 
159  Use *getCheckUse() const { return CheckUse; }
160 
161  /// Represents an signed integer range [Range.getBegin(), Range.getEnd()). If
162  /// R.getEnd() sle R.getBegin(), then R denotes the empty range.
163 
164  class Range {
165  const SCEV *Begin;
166  const SCEV *End;
167 
168  public:
169  Range(const SCEV *Begin, const SCEV *End) : Begin(Begin), End(End) {
170  assert(Begin->getType() == End->getType() && "ill-typed range!");
171  }
172 
173  Type *getType() const { return Begin->getType(); }
174  const SCEV *getBegin() const { return Begin; }
175  const SCEV *getEnd() const { return End; }
176  };
177 
178  /// This is the value the condition of the branch needs to evaluate to for the
179  /// branch to take the hot successor (see (1) above).
180  bool getPassingDirection() { return true; }
181 
182  /// Computes a range for the induction variable (IndVar) in which the range
183  /// check is redundant and can be constant-folded away. The induction
184  /// variable is not required to be the canonical {0,+,1} induction variable.
185  Optional<Range> computeSafeIterationSpace(ScalarEvolution &SE,
186  const SCEVAddRecExpr *IndVar) const;
187 
188  /// Parse out a set of inductive range checks from \p BI and append them to \p
189  /// Checks.
190  ///
191  /// NB! There may be conditions feeding into \p BI that aren't inductive range
192  /// checks, and hence don't end up in \p Checks.
193  static void
194  extractRangeChecksFromBranch(BranchInst *BI, Loop *L, ScalarEvolution &SE,
197 };
198 
199 class InductiveRangeCheckElimination : public LoopPass {
200 public:
201  static char ID;
202  InductiveRangeCheckElimination() : LoopPass(ID) {
205  }
206 
207  void getAnalysisUsage(AnalysisUsage &AU) const override {
210  }
211 
212  bool runOnLoop(Loop *L, LPPassManager &LPM) override;
213 };
214 
216 }
217 
218 INITIALIZE_PASS_BEGIN(InductiveRangeCheckElimination, "irce",
219  "Inductive range check elimination", false, false)
222 INITIALIZE_PASS_END(InductiveRangeCheckElimination, "irce",
223  "Inductive range check elimination", false, false)
224 
225 StringRef InductiveRangeCheck::rangeCheckKindToStr(
226  InductiveRangeCheck::RangeCheckKind RCK) {
227  switch (RCK) {
228  case InductiveRangeCheck::RANGE_CHECK_UNKNOWN:
229  return "RANGE_CHECK_UNKNOWN";
230 
231  case InductiveRangeCheck::RANGE_CHECK_UPPER:
232  return "RANGE_CHECK_UPPER";
233 
234  case InductiveRangeCheck::RANGE_CHECK_LOWER:
235  return "RANGE_CHECK_LOWER";
236 
237  case InductiveRangeCheck::RANGE_CHECK_BOTH:
238  return "RANGE_CHECK_BOTH";
239  }
240 
241  llvm_unreachable("unknown range check type!");
242 }
243 
244 /// Parse a single ICmp instruction, `ICI`, into a range check. If `ICI` cannot
245 /// be interpreted as a range check, return `RANGE_CHECK_UNKNOWN` and set
246 /// `Index` and `Length` to `nullptr`. Otherwise set `Index` to the value being
247 /// range checked, and set `Length` to the upper limit `Index` is being range
248 /// checked with if (and only if) the range check type is stronger or equal to
249 /// RANGE_CHECK_UPPER.
250 ///
251 InductiveRangeCheck::RangeCheckKind
252 InductiveRangeCheck::parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
253  ScalarEvolution &SE, Value *&Index,
254  Value *&Length) {
255 
256  auto IsNonNegativeAndNotLoopVarying = [&SE, L](Value *V) {
257  const SCEV *S = SE.getSCEV(V);
258  if (isa<SCEVCouldNotCompute>(S))
259  return false;
260 
262  SE.isKnownNonNegative(S);
263  };
264 
265  using namespace llvm::PatternMatch;
266 
267  ICmpInst::Predicate Pred = ICI->getPredicate();
268  Value *LHS = ICI->getOperand(0);
269  Value *RHS = ICI->getOperand(1);
270 
271  switch (Pred) {
272  default:
273  return RANGE_CHECK_UNKNOWN;
274 
275  case ICmpInst::ICMP_SLE:
276  std::swap(LHS, RHS);
278  case ICmpInst::ICMP_SGE:
279  if (match(RHS, m_ConstantInt<0>())) {
280  Index = LHS;
281  return RANGE_CHECK_LOWER;
282  }
283  return RANGE_CHECK_UNKNOWN;
284 
285  case ICmpInst::ICMP_SLT:
286  std::swap(LHS, RHS);
288  case ICmpInst::ICMP_SGT:
289  if (match(RHS, m_ConstantInt<-1>())) {
290  Index = LHS;
291  return RANGE_CHECK_LOWER;
292  }
293 
294  if (IsNonNegativeAndNotLoopVarying(LHS)) {
295  Index = RHS;
296  Length = LHS;
297  return RANGE_CHECK_UPPER;
298  }
299  return RANGE_CHECK_UNKNOWN;
300 
301  case ICmpInst::ICMP_ULT:
302  std::swap(LHS, RHS);
304  case ICmpInst::ICMP_UGT:
305  if (IsNonNegativeAndNotLoopVarying(LHS)) {
306  Index = RHS;
307  Length = LHS;
308  return RANGE_CHECK_BOTH;
309  }
310  return RANGE_CHECK_UNKNOWN;
311  }
312 
313  llvm_unreachable("default clause returns!");
314 }
315 
316 void InductiveRangeCheck::extractRangeChecksFromCond(
317  Loop *L, ScalarEvolution &SE, Use &ConditionUse,
319  SmallPtrSetImpl<Value *> &Visited) {
320  using namespace llvm::PatternMatch;
321 
322  Value *Condition = ConditionUse.get();
323  if (!Visited.insert(Condition).second)
324  return;
325 
326  if (match(Condition, m_And(m_Value(), m_Value()))) {
328  extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(0),
329  SubChecks, Visited);
330  extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(1),
331  SubChecks, Visited);
332 
333  if (SubChecks.size() == 2) {
334  // Handle a special case where we know how to merge two checks separately
335  // checking the upper and lower bounds into a full range check.
336  const auto &RChkA = SubChecks[0];
337  const auto &RChkB = SubChecks[1];
338  if ((RChkA.Length == RChkB.Length || !RChkA.Length || !RChkB.Length) &&
339  RChkA.Offset == RChkB.Offset && RChkA.Scale == RChkB.Scale) {
340 
341  // If RChkA.Kind == RChkB.Kind then we just found two identical checks.
342  // But if one of them is a RANGE_CHECK_LOWER and the other is a
343  // RANGE_CHECK_UPPER (only possibility if they're different) then
344  // together they form a RANGE_CHECK_BOTH.
345  SubChecks[0].Kind =
346  (InductiveRangeCheck::RangeCheckKind)(RChkA.Kind | RChkB.Kind);
347  SubChecks[0].Length = RChkA.Length ? RChkA.Length : RChkB.Length;
348  SubChecks[0].CheckUse = &ConditionUse;
349 
350  // We updated one of the checks in place, now erase the other.
351  SubChecks.pop_back();
352  }
353  }
354 
355  Checks.insert(Checks.end(), SubChecks.begin(), SubChecks.end());
356  return;
357  }
358 
359  ICmpInst *ICI = dyn_cast<ICmpInst>(Condition);
360  if (!ICI)
361  return;
362 
363  Value *Length = nullptr, *Index;
364  auto RCKind = parseRangeCheckICmp(L, ICI, SE, Index, Length);
365  if (RCKind == InductiveRangeCheck::RANGE_CHECK_UNKNOWN)
366  return;
367 
368  const auto *IndexAddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Index));
369  bool IsAffineIndex =
370  IndexAddRec && (IndexAddRec->getLoop() == L) && IndexAddRec->isAffine();
371 
372  if (!IsAffineIndex)
373  return;
374 
375  InductiveRangeCheck IRC;
376  IRC.Length = Length;
377  IRC.Offset = IndexAddRec->getStart();
378  IRC.Scale = IndexAddRec->getStepRecurrence(SE);
379  IRC.CheckUse = &ConditionUse;
380  IRC.Kind = RCKind;
381  Checks.push_back(IRC);
382 }
383 
384 void InductiveRangeCheck::extractRangeChecksFromBranch(
387 
388  if (BI->isUnconditional() || BI->getParent() == L->getLoopLatch())
389  return;
390 
391  BranchProbability LikelyTaken(15, 16);
392 
394  BPI.getEdgeProbability(BI->getParent(), (unsigned)0) < LikelyTaken)
395  return;
396 
397  SmallPtrSet<Value *, 8> Visited;
398  InductiveRangeCheck::extractRangeChecksFromCond(L, SE, BI->getOperandUse(0),
399  Checks, Visited);
400 }
401 
402 // Add metadata to the loop L to disable loop optimizations. Callers need to
403 // confirm that optimizing loop L is not beneficial.
405  // We do not care about any existing loopID related metadata for L, since we
406  // are setting all loop metadata to false.
407  LLVMContext &Context = L.getHeader()->getContext();
408  // Reserve first location for self reference to the LoopID metadata node.
409  MDNode *Dummy = MDNode::get(Context, {});
410  MDNode *DisableUnroll = MDNode::get(
411  Context, {MDString::get(Context, "llvm.loop.unroll.disable")});
412  Metadata *FalseVal =
414  MDNode *DisableVectorize = MDNode::get(
415  Context,
416  {MDString::get(Context, "llvm.loop.vectorize.enable"), FalseVal});
417  MDNode *DisableLICMVersioning = MDNode::get(
418  Context, {MDString::get(Context, "llvm.loop.licm_versioning.disable")});
419  MDNode *DisableDistribution= MDNode::get(
420  Context,
421  {MDString::get(Context, "llvm.loop.distribute.enable"), FalseVal});
422  MDNode *NewLoopID =
423  MDNode::get(Context, {Dummy, DisableUnroll, DisableVectorize,
424  DisableLICMVersioning, DisableDistribution});
425  // Set operand 0 to refer to the loop id itself.
426  NewLoopID->replaceOperandWith(0, NewLoopID);
427  L.setLoopID(NewLoopID);
428 }
429 
430 namespace {
431 
432 // Keeps track of the structure of a loop. This is similar to llvm::Loop,
433 // except that it is more lightweight and can track the state of a loop through
434 // changing and potentially invalid IR. This structure also formalizes the
435 // kinds of loops we can deal with -- ones that have a single latch that is also
436 // an exiting block *and* have a canonical induction variable.
437 struct LoopStructure {
438  const char *Tag;
439 
440  BasicBlock *Header;
441  BasicBlock *Latch;
442 
443  // `Latch's terminator instruction is `LatchBr', and it's `LatchBrExitIdx'th
444  // successor is `LatchExit', the exit block of the loop.
445  BranchInst *LatchBr;
446  BasicBlock *LatchExit;
447  unsigned LatchBrExitIdx;
448 
449  Value *IndVarNext;
450  Value *IndVarStart;
451  Value *LoopExitAt;
452  bool IndVarIncreasing;
453 
454  LoopStructure()
455  : Tag(""), Header(nullptr), Latch(nullptr), LatchBr(nullptr),
456  LatchExit(nullptr), LatchBrExitIdx(-1), IndVarNext(nullptr),
457  IndVarStart(nullptr), LoopExitAt(nullptr), IndVarIncreasing(false) {}
458 
459  template <typename M> LoopStructure map(M Map) const {
460  LoopStructure Result;
461  Result.Tag = Tag;
462  Result.Header = cast<BasicBlock>(Map(Header));
463  Result.Latch = cast<BasicBlock>(Map(Latch));
464  Result.LatchBr = cast<BranchInst>(Map(LatchBr));
465  Result.LatchExit = cast<BasicBlock>(Map(LatchExit));
466  Result.LatchBrExitIdx = LatchBrExitIdx;
467  Result.IndVarNext = Map(IndVarNext);
468  Result.IndVarStart = Map(IndVarStart);
469  Result.LoopExitAt = Map(LoopExitAt);
470  Result.IndVarIncreasing = IndVarIncreasing;
471  return Result;
472  }
473 
474  static Optional<LoopStructure> parseLoopStructure(ScalarEvolution &,
476  Loop &,
477  const char *&);
478 };
479 
480 /// This class is used to constrain loops to run within a given iteration space.
481 /// The algorithm this class implements is given a Loop and a range [Begin,
482 /// End). The algorithm then tries to break out a "main loop" out of the loop
483 /// it is given in a way that the "main loop" runs with the induction variable
484 /// in a subset of [Begin, End). The algorithm emits appropriate pre and post
485 /// loops to run any remaining iterations. The pre loop runs any iterations in
486 /// which the induction variable is < Begin, and the post loop runs any
487 /// iterations in which the induction variable is >= End.
488 ///
489 class LoopConstrainer {
490  // The representation of a clone of the original loop we started out with.
491  struct ClonedLoop {
492  // The cloned blocks
493  std::vector<BasicBlock *> Blocks;
494 
495  // `Map` maps values in the clonee into values in the cloned version
496  ValueToValueMapTy Map;
497 
498  // An instance of `LoopStructure` for the cloned loop
499  LoopStructure Structure;
500  };
501 
502  // Result of rewriting the range of a loop. See changeIterationSpaceEnd for
503  // more details on what these fields mean.
504  struct RewrittenRangeInfo {
505  BasicBlock *PseudoExit;
506  BasicBlock *ExitSelector;
507  std::vector<PHINode *> PHIValuesAtPseudoExit;
508  PHINode *IndVarEnd;
509 
510  RewrittenRangeInfo()
511  : PseudoExit(nullptr), ExitSelector(nullptr), IndVarEnd(nullptr) {}
512  };
513 
514  // Calculated subranges we restrict the iteration space of the main loop to.
515  // See the implementation of `calculateSubRanges' for more details on how
516  // these fields are computed. `LowLimit` is None if there is no restriction
517  // on low end of the restricted iteration space of the main loop. `HighLimit`
518  // is None if there is no restriction on high end of the restricted iteration
519  // space of the main loop.
520 
521  struct SubRanges {
522  Optional<const SCEV *> LowLimit;
523  Optional<const SCEV *> HighLimit;
524  };
525 
526  // A utility function that does a `replaceUsesOfWith' on the incoming block
527  // set of a `PHINode' -- replaces instances of `Block' in the `PHINode's
528  // incoming block list with `ReplaceBy'.
529  static void replacePHIBlock(PHINode *PN, BasicBlock *Block,
530  BasicBlock *ReplaceBy);
531 
532  // Compute a safe set of limits for the main loop to run in -- effectively the
533  // intersection of `Range' and the iteration space of the original loop.
534  // Return None if unable to compute the set of subranges.
535  //
536  Optional<SubRanges> calculateSubRanges() const;
537 
538  // Clone `OriginalLoop' and return the result in CLResult. The IR after
539  // running `cloneLoop' is well formed except for the PHI nodes in CLResult --
540  // the PHI nodes say that there is an incoming edge from `OriginalPreheader`
541  // but there is no such edge.
542  //
543  void cloneLoop(ClonedLoop &CLResult, const char *Tag) const;
544 
545  // Create the appropriate loop structure needed to describe a cloned copy of
546  // `Original`. The clone is described by `VM`.
547  Loop *createClonedLoopStructure(Loop *Original, Loop *Parent,
548  ValueToValueMapTy &VM);
549 
550  // Rewrite the iteration space of the loop denoted by (LS, Preheader). The
551  // iteration space of the rewritten loop ends at ExitLoopAt. The start of the
552  // iteration space is not changed. `ExitLoopAt' is assumed to be slt
553  // `OriginalHeaderCount'.
554  //
555  // If there are iterations left to execute, control is made to jump to
556  // `ContinuationBlock', otherwise they take the normal loop exit. The
557  // returned `RewrittenRangeInfo' object is populated as follows:
558  //
559  // .PseudoExit is a basic block that unconditionally branches to
560  // `ContinuationBlock'.
561  //
562  // .ExitSelector is a basic block that decides, on exit from the loop,
563  // whether to branch to the "true" exit or to `PseudoExit'.
564  //
565  // .PHIValuesAtPseudoExit are PHINodes in `PseudoExit' that compute the value
566  // for each PHINode in the loop header on taking the pseudo exit.
567  //
568  // After changeIterationSpaceEnd, `Preheader' is no longer a legitimate
569  // preheader because it is made to branch to the loop header only
570  // conditionally.
571  //
572  RewrittenRangeInfo
573  changeIterationSpaceEnd(const LoopStructure &LS, BasicBlock *Preheader,
574  Value *ExitLoopAt,
575  BasicBlock *ContinuationBlock) const;
576 
577  // The loop denoted by `LS' has `OldPreheader' as its preheader. This
578  // function creates a new preheader for `LS' and returns it.
579  //
580  BasicBlock *createPreheader(const LoopStructure &LS, BasicBlock *OldPreheader,
581  const char *Tag) const;
582 
583  // `ContinuationBlockAndPreheader' was the continuation block for some call to
584  // `changeIterationSpaceEnd' and is the preheader to the loop denoted by `LS'.
585  // This function rewrites the PHI nodes in `LS.Header' to start with the
586  // correct value.
587  void rewriteIncomingValuesForPHIs(
588  LoopStructure &LS, BasicBlock *ContinuationBlockAndPreheader,
589  const LoopConstrainer::RewrittenRangeInfo &RRI) const;
590 
591  // Even though we do not preserve any passes at this time, we at least need to
592  // keep the parent loop structure consistent. The `LPPassManager' seems to
593  // verify this after running a loop pass. This function adds the list of
594  // blocks denoted by BBs to this loops parent loop if required.
595  void addToParentLoopIfNeeded(ArrayRef<BasicBlock *> BBs);
596 
597  // Some global state.
598  Function &F;
599  LLVMContext &Ctx;
600  ScalarEvolution &SE;
601  DominatorTree &DT;
602  LPPassManager &LPM;
603  LoopInfo &LI;
604 
605  // Information about the original loop we started out with.
606  Loop &OriginalLoop;
607  const SCEV *LatchTakenCount;
608  BasicBlock *OriginalPreheader;
609 
610  // The preheader of the main loop. This may or may not be different from
611  // `OriginalPreheader'.
612  BasicBlock *MainLoopPreheader;
613 
614  // The range we need to run the main loop in.
615  InductiveRangeCheck::Range Range;
616 
617  // The structure of the main loop (see comment at the beginning of this class
618  // for a definition)
619  LoopStructure MainLoopStructure;
620 
621 public:
622  LoopConstrainer(Loop &L, LoopInfo &LI, LPPassManager &LPM,
623  const LoopStructure &LS, ScalarEvolution &SE,
624  DominatorTree &DT, InductiveRangeCheck::Range R)
625  : F(*L.getHeader()->getParent()), Ctx(L.getHeader()->getContext()),
626  SE(SE), DT(DT), LPM(LPM), LI(LI), OriginalLoop(L),
627  LatchTakenCount(nullptr), OriginalPreheader(nullptr),
628  MainLoopPreheader(nullptr), Range(R), MainLoopStructure(LS) {}
629 
630  // Entry point for the algorithm. Returns true on success.
631  bool run();
632 };
633 
634 }
635 
636 void LoopConstrainer::replacePHIBlock(PHINode *PN, BasicBlock *Block,
637  BasicBlock *ReplaceBy) {
638  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
639  if (PN->getIncomingBlock(i) == Block)
640  PN->setIncomingBlock(i, ReplaceBy);
641 }
642 
643 static bool CanBeSMax(ScalarEvolution &SE, const SCEV *S) {
644  APInt SMax =
645  APInt::getSignedMaxValue(cast<IntegerType>(S->getType())->getBitWidth());
646  return SE.getSignedRange(S).contains(SMax) &&
647  SE.getUnsignedRange(S).contains(SMax);
648 }
649 
650 static bool CanBeSMin(ScalarEvolution &SE, const SCEV *S) {
651  APInt SMin =
652  APInt::getSignedMinValue(cast<IntegerType>(S->getType())->getBitWidth());
653  return SE.getSignedRange(S).contains(SMin) &&
654  SE.getUnsignedRange(S).contains(SMin);
655 }
656 
658 LoopStructure::parseLoopStructure(ScalarEvolution &SE, BranchProbabilityInfo &BPI,
659  Loop &L, const char *&FailureReason) {
660  if (!L.isLoopSimplifyForm()) {
661  FailureReason = "loop not in LoopSimplify form";
662  return None;
663  }
664 
665  BasicBlock *Latch = L.getLoopLatch();
666  assert(Latch && "Simplified loops only have one latch!");
667 
668  if (Latch->getTerminator()->getMetadata(ClonedLoopTag)) {
669  FailureReason = "loop has already been cloned";
670  return None;
671  }
672 
673  if (!L.isLoopExiting(Latch)) {
674  FailureReason = "no loop latch";
675  return None;
676  }
677 
678  BasicBlock *Header = L.getHeader();
679  BasicBlock *Preheader = L.getLoopPreheader();
680  if (!Preheader) {
681  FailureReason = "no preheader";
682  return None;
683  }
684 
685  BranchInst *LatchBr = dyn_cast<BranchInst>(Latch->getTerminator());
686  if (!LatchBr || LatchBr->isUnconditional()) {
687  FailureReason = "latch terminator not conditional branch";
688  return None;
689  }
690 
691  unsigned LatchBrExitIdx = LatchBr->getSuccessor(0) == Header ? 1 : 0;
692 
693  BranchProbability ExitProbability =
694  BPI.getEdgeProbability(LatchBr->getParent(), LatchBrExitIdx);
695 
697  ExitProbability > BranchProbability(1, MaxExitProbReciprocal)) {
698  FailureReason = "short running loop, not profitable";
699  return None;
700  }
701 
702  ICmpInst *ICI = dyn_cast<ICmpInst>(LatchBr->getCondition());
703  if (!ICI || !isa<IntegerType>(ICI->getOperand(0)->getType())) {
704  FailureReason = "latch terminator branch not conditional on integral icmp";
705  return None;
706  }
707 
708  const SCEV *LatchCount = SE.getExitCount(&L, Latch);
709  if (isa<SCEVCouldNotCompute>(LatchCount)) {
710  FailureReason = "could not compute latch count";
711  return None;
712  }
713 
714  ICmpInst::Predicate Pred = ICI->getPredicate();
715  Value *LeftValue = ICI->getOperand(0);
716  const SCEV *LeftSCEV = SE.getSCEV(LeftValue);
717  IntegerType *IndVarTy = cast<IntegerType>(LeftValue->getType());
718 
719  Value *RightValue = ICI->getOperand(1);
720  const SCEV *RightSCEV = SE.getSCEV(RightValue);
721 
722  // We canonicalize `ICI` such that `LeftSCEV` is an add recurrence.
723  if (!isa<SCEVAddRecExpr>(LeftSCEV)) {
724  if (isa<SCEVAddRecExpr>(RightSCEV)) {
725  std::swap(LeftSCEV, RightSCEV);
726  std::swap(LeftValue, RightValue);
727  Pred = ICmpInst::getSwappedPredicate(Pred);
728  } else {
729  FailureReason = "no add recurrences in the icmp";
730  return None;
731  }
732  }
733 
734  auto HasNoSignedWrap = [&](const SCEVAddRecExpr *AR) {
735  if (AR->getNoWrapFlags(SCEV::FlagNSW))
736  return true;
737 
738  IntegerType *Ty = cast<IntegerType>(AR->getType());
739  IntegerType *WideTy =
740  IntegerType::get(Ty->getContext(), Ty->getBitWidth() * 2);
741 
742  const SCEVAddRecExpr *ExtendAfterOp =
743  dyn_cast<SCEVAddRecExpr>(SE.getSignExtendExpr(AR, WideTy));
744  if (ExtendAfterOp) {
745  const SCEV *ExtendedStart = SE.getSignExtendExpr(AR->getStart(), WideTy);
746  const SCEV *ExtendedStep =
747  SE.getSignExtendExpr(AR->getStepRecurrence(SE), WideTy);
748 
749  bool NoSignedWrap = ExtendAfterOp->getStart() == ExtendedStart &&
750  ExtendAfterOp->getStepRecurrence(SE) == ExtendedStep;
751 
752  if (NoSignedWrap)
753  return true;
754  }
755 
756  // We may have proved this when computing the sign extension above.
757  return AR->getNoWrapFlags(SCEV::FlagNSW) != SCEV::FlagAnyWrap;
758  };
759 
760  auto IsInductionVar = [&](const SCEVAddRecExpr *AR, bool &IsIncreasing) {
761  if (!AR->isAffine())
762  return false;
763 
764  // Currently we only work with induction variables that have been proved to
765  // not wrap. This restriction can potentially be lifted in the future.
766 
767  if (!HasNoSignedWrap(AR))
768  return false;
769 
770  if (const SCEVConstant *StepExpr =
771  dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE))) {
772  ConstantInt *StepCI = StepExpr->getValue();
773  if (StepCI->isOne() || StepCI->isMinusOne()) {
774  IsIncreasing = StepCI->isOne();
775  return true;
776  }
777  }
778 
779  return false;
780  };
781 
782  // `ICI` is interpreted as taking the backedge if the *next* value of the
783  // induction variable satisfies some constraint.
784 
785  const SCEVAddRecExpr *IndVarNext = cast<SCEVAddRecExpr>(LeftSCEV);
786  bool IsIncreasing = false;
787  if (!IsInductionVar(IndVarNext, IsIncreasing)) {
788  FailureReason = "LHS in icmp not induction variable";
789  return None;
790  }
791 
792  ConstantInt *One = ConstantInt::get(IndVarTy, 1);
793  // TODO: generalize the predicates here to also match their unsigned variants.
794  if (IsIncreasing) {
795  bool FoundExpectedPred =
796  (Pred == ICmpInst::ICMP_SLT && LatchBrExitIdx == 1) ||
797  (Pred == ICmpInst::ICMP_SGT && LatchBrExitIdx == 0);
798 
799  if (!FoundExpectedPred) {
800  FailureReason = "expected icmp slt semantically, found something else";
801  return None;
802  }
803 
804  if (LatchBrExitIdx == 0) {
805  if (CanBeSMax(SE, RightSCEV)) {
806  // TODO: this restriction is easily removable -- we just have to
807  // remember that the icmp was an slt and not an sle.
808  FailureReason = "limit may overflow when coercing sle to slt";
809  return None;
810  }
811 
812  IRBuilder<> B(Preheader->getTerminator());
813  RightValue = B.CreateAdd(RightValue, One);
814  }
815 
816  } else {
817  bool FoundExpectedPred =
818  (Pred == ICmpInst::ICMP_SGT && LatchBrExitIdx == 1) ||
819  (Pred == ICmpInst::ICMP_SLT && LatchBrExitIdx == 0);
820 
821  if (!FoundExpectedPred) {
822  FailureReason = "expected icmp sgt semantically, found something else";
823  return None;
824  }
825 
826  if (LatchBrExitIdx == 0) {
827  if (CanBeSMin(SE, RightSCEV)) {
828  // TODO: this restriction is easily removable -- we just have to
829  // remember that the icmp was an sgt and not an sge.
830  FailureReason = "limit may overflow when coercing sge to sgt";
831  return None;
832  }
833 
834  IRBuilder<> B(Preheader->getTerminator());
835  RightValue = B.CreateSub(RightValue, One);
836  }
837  }
838 
839  const SCEV *StartNext = IndVarNext->getStart();
840  const SCEV *Addend = SE.getNegativeSCEV(IndVarNext->getStepRecurrence(SE));
841  const SCEV *IndVarStart = SE.getAddExpr(StartNext, Addend);
842 
843  BasicBlock *LatchExit = LatchBr->getSuccessor(LatchBrExitIdx);
844 
845  assert(SE.getLoopDisposition(LatchCount, &L) ==
847  "loop variant exit count doesn't make sense!");
848 
849  assert(!L.contains(LatchExit) && "expected an exit block!");
850  const DataLayout &DL = Preheader->getModule()->getDataLayout();
851  Value *IndVarStartV =
852  SCEVExpander(SE, DL, "irce")
853  .expandCodeFor(IndVarStart, IndVarTy, Preheader->getTerminator());
854  IndVarStartV->setName("indvar.start");
855 
856  LoopStructure Result;
857 
858  Result.Tag = "main";
859  Result.Header = Header;
860  Result.Latch = Latch;
861  Result.LatchBr = LatchBr;
862  Result.LatchExit = LatchExit;
863  Result.LatchBrExitIdx = LatchBrExitIdx;
864  Result.IndVarStart = IndVarStartV;
865  Result.IndVarNext = LeftValue;
866  Result.IndVarIncreasing = IsIncreasing;
867  Result.LoopExitAt = RightValue;
868 
869  FailureReason = nullptr;
870 
871  return Result;
872 }
873 
875 LoopConstrainer::calculateSubRanges() const {
876  IntegerType *Ty = cast<IntegerType>(LatchTakenCount->getType());
877 
878  if (Range.getType() != Ty)
879  return None;
880 
881  LoopConstrainer::SubRanges Result;
882 
883  // I think we can be more aggressive here and make this nuw / nsw if the
884  // addition that feeds into the icmp for the latch's terminating branch is nuw
885  // / nsw. In any case, a wrapping 2's complement addition is safe.
886  ConstantInt *One = ConstantInt::get(Ty, 1);
887  const SCEV *Start = SE.getSCEV(MainLoopStructure.IndVarStart);
888  const SCEV *End = SE.getSCEV(MainLoopStructure.LoopExitAt);
889 
890  bool Increasing = MainLoopStructure.IndVarIncreasing;
891 
892  // We compute `Smallest` and `Greatest` such that [Smallest, Greatest) is the
893  // range of values the induction variable takes.
894 
895  const SCEV *Smallest = nullptr, *Greatest = nullptr;
896 
897  if (Increasing) {
898  Smallest = Start;
899  Greatest = End;
900  } else {
901  // These two computations may sign-overflow. Here is why that is okay:
902  //
903  // We know that the induction variable does not sign-overflow on any
904  // iteration except the last one, and it starts at `Start` and ends at
905  // `End`, decrementing by one every time.
906  //
907  // * if `Smallest` sign-overflows we know `End` is `INT_SMAX`. Since the
908  // induction variable is decreasing we know that that the smallest value
909  // the loop body is actually executed with is `INT_SMIN` == `Smallest`.
910  //
911  // * if `Greatest` sign-overflows, we know it can only be `INT_SMIN`. In
912  // that case, `Clamp` will always return `Smallest` and
913  // [`Result.LowLimit`, `Result.HighLimit`) = [`Smallest`, `Smallest`)
914  // will be an empty range. Returning an empty range is always safe.
915  //
916 
917  Smallest = SE.getAddExpr(End, SE.getSCEV(One));
918  Greatest = SE.getAddExpr(Start, SE.getSCEV(One));
919  }
920 
921  auto Clamp = [this, Smallest, Greatest](const SCEV *S) {
922  return SE.getSMaxExpr(Smallest, SE.getSMinExpr(Greatest, S));
923  };
924 
925  // In some cases we can prove that we don't need a pre or post loop
926 
927  bool ProvablyNoPreloop =
928  SE.isKnownPredicate(ICmpInst::ICMP_SLE, Range.getBegin(), Smallest);
929  if (!ProvablyNoPreloop)
930  Result.LowLimit = Clamp(Range.getBegin());
931 
932  bool ProvablyNoPostLoop =
933  SE.isKnownPredicate(ICmpInst::ICMP_SLE, Greatest, Range.getEnd());
934  if (!ProvablyNoPostLoop)
935  Result.HighLimit = Clamp(Range.getEnd());
936 
937  return Result;
938 }
939 
940 void LoopConstrainer::cloneLoop(LoopConstrainer::ClonedLoop &Result,
941  const char *Tag) const {
942  for (BasicBlock *BB : OriginalLoop.getBlocks()) {
943  BasicBlock *Clone = CloneBasicBlock(BB, Result.Map, Twine(".") + Tag, &F);
944  Result.Blocks.push_back(Clone);
945  Result.Map[BB] = Clone;
946  }
947 
948  auto GetClonedValue = [&Result](Value *V) {
949  assert(V && "null values not in domain!");
950  auto It = Result.Map.find(V);
951  if (It == Result.Map.end())
952  return V;
953  return static_cast<Value *>(It->second);
954  };
955 
956  auto *ClonedLatch =
957  cast<BasicBlock>(GetClonedValue(OriginalLoop.getLoopLatch()));
958  ClonedLatch->getTerminator()->setMetadata(ClonedLoopTag,
959  MDNode::get(Ctx, {}));
960 
961  Result.Structure = MainLoopStructure.map(GetClonedValue);
962  Result.Structure.Tag = Tag;
963 
964  for (unsigned i = 0, e = Result.Blocks.size(); i != e; ++i) {
965  BasicBlock *ClonedBB = Result.Blocks[i];
966  BasicBlock *OriginalBB = OriginalLoop.getBlocks()[i];
967 
968  assert(Result.Map[OriginalBB] == ClonedBB && "invariant!");
969 
970  for (Instruction &I : *ClonedBB)
971  RemapInstruction(&I, Result.Map,
973 
974  // Exit blocks will now have one more predecessor and their PHI nodes need
975  // to be edited to reflect that. No phi nodes need to be introduced because
976  // the loop is in LCSSA.
977 
978  for (auto *SBB : successors(OriginalBB)) {
979  if (OriginalLoop.contains(SBB))
980  continue; // not an exit block
981 
982  for (Instruction &I : *SBB) {
983  auto *PN = dyn_cast<PHINode>(&I);
984  if (!PN)
985  break;
986 
987  Value *OldIncoming = PN->getIncomingValueForBlock(OriginalBB);
988  PN->addIncoming(GetClonedValue(OldIncoming), ClonedBB);
989  }
990  }
991  }
992 }
993 
994 LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
995  const LoopStructure &LS, BasicBlock *Preheader, Value *ExitSubloopAt,
996  BasicBlock *ContinuationBlock) const {
997 
998  // We start with a loop with a single latch:
999  //
1000  // +--------------------+
1001  // | |
1002  // | preheader |
1003  // | |
1004  // +--------+-----------+
1005  // | ----------------\
1006  // | / |
1007  // +--------v----v------+ |
1008  // | | |
1009  // | header | |
1010  // | | |
1011  // +--------------------+ |
1012  // |
1013  // ..... |
1014  // |
1015  // +--------------------+ |
1016  // | | |
1017  // | latch >----------/
1018  // | |
1019  // +-------v------------+
1020  // |
1021  // |
1022  // | +--------------------+
1023  // | | |
1024  // +---> original exit |
1025  // | |
1026  // +--------------------+
1027  //
1028  // We change the control flow to look like
1029  //
1030  //
1031  // +--------------------+
1032  // | |
1033  // | preheader >-------------------------+
1034  // | | |
1035  // +--------v-----------+ |
1036  // | /-------------+ |
1037  // | / | |
1038  // +--------v--v--------+ | |
1039  // | | | |
1040  // | header | | +--------+ |
1041  // | | | | | |
1042  // +--------------------+ | | +-----v-----v-----------+
1043  // | | | |
1044  // | | | .pseudo.exit |
1045  // | | | |
1046  // | | +-----------v-----------+
1047  // | | |
1048  // ..... | | |
1049  // | | +--------v-------------+
1050  // +--------------------+ | | | |
1051  // | | | | | ContinuationBlock |
1052  // | latch >------+ | | |
1053  // | | | +----------------------+
1054  // +---------v----------+ |
1055  // | |
1056  // | |
1057  // | +---------------^-----+
1058  // | | |
1059  // +-----> .exit.selector |
1060  // | |
1061  // +----------v----------+
1062  // |
1063  // +--------------------+ |
1064  // | | |
1065  // | original exit <----+
1066  // | |
1067  // +--------------------+
1068  //
1069 
1070  RewrittenRangeInfo RRI;
1071 
1072  BasicBlock *BBInsertLocation = LS.Latch->getNextNode();
1073  RRI.ExitSelector = BasicBlock::Create(Ctx, Twine(LS.Tag) + ".exit.selector",
1074  &F, BBInsertLocation);
1075  RRI.PseudoExit = BasicBlock::Create(Ctx, Twine(LS.Tag) + ".pseudo.exit", &F,
1076  BBInsertLocation);
1077 
1078  BranchInst *PreheaderJump = cast<BranchInst>(Preheader->getTerminator());
1079  bool Increasing = LS.IndVarIncreasing;
1080 
1081  IRBuilder<> B(PreheaderJump);
1082 
1083  // EnterLoopCond - is it okay to start executing this `LS'?
1084  Value *EnterLoopCond = Increasing
1085  ? B.CreateICmpSLT(LS.IndVarStart, ExitSubloopAt)
1086  : B.CreateICmpSGT(LS.IndVarStart, ExitSubloopAt);
1087 
1088  B.CreateCondBr(EnterLoopCond, LS.Header, RRI.PseudoExit);
1089  PreheaderJump->eraseFromParent();
1090 
1091  LS.LatchBr->setSuccessor(LS.LatchBrExitIdx, RRI.ExitSelector);
1092  B.SetInsertPoint(LS.LatchBr);
1093  Value *TakeBackedgeLoopCond =
1094  Increasing ? B.CreateICmpSLT(LS.IndVarNext, ExitSubloopAt)
1095  : B.CreateICmpSGT(LS.IndVarNext, ExitSubloopAt);
1096  Value *CondForBranch = LS.LatchBrExitIdx == 1
1097  ? TakeBackedgeLoopCond
1098  : B.CreateNot(TakeBackedgeLoopCond);
1099 
1100  LS.LatchBr->setCondition(CondForBranch);
1101 
1102  B.SetInsertPoint(RRI.ExitSelector);
1103 
1104  // IterationsLeft - are there any more iterations left, given the original
1105  // upper bound on the induction variable? If not, we branch to the "real"
1106  // exit.
1107  Value *IterationsLeft = Increasing
1108  ? B.CreateICmpSLT(LS.IndVarNext, LS.LoopExitAt)
1109  : B.CreateICmpSGT(LS.IndVarNext, LS.LoopExitAt);
1110  B.CreateCondBr(IterationsLeft, RRI.PseudoExit, LS.LatchExit);
1111 
1112  BranchInst *BranchToContinuation =
1113  BranchInst::Create(ContinuationBlock, RRI.PseudoExit);
1114 
1115  // We emit PHI nodes into `RRI.PseudoExit' that compute the "latest" value of
1116  // each of the PHI nodes in the loop header. This feeds into the initial
1117  // value of the same PHI nodes if/when we continue execution.
1118  for (Instruction &I : *LS.Header) {
1119  auto *PN = dyn_cast<PHINode>(&I);
1120  if (!PN)
1121  break;
1122 
1123  PHINode *NewPHI = PHINode::Create(PN->getType(), 2, PN->getName() + ".copy",
1124  BranchToContinuation);
1125 
1126  NewPHI->addIncoming(PN->getIncomingValueForBlock(Preheader), Preheader);
1127  NewPHI->addIncoming(PN->getIncomingValueForBlock(LS.Latch),
1128  RRI.ExitSelector);
1129  RRI.PHIValuesAtPseudoExit.push_back(NewPHI);
1130  }
1131 
1132  RRI.IndVarEnd = PHINode::Create(LS.IndVarNext->getType(), 2, "indvar.end",
1133  BranchToContinuation);
1134  RRI.IndVarEnd->addIncoming(LS.IndVarStart, Preheader);
1135  RRI.IndVarEnd->addIncoming(LS.IndVarNext, RRI.ExitSelector);
1136 
1137  // The latch exit now has a branch from `RRI.ExitSelector' instead of
1138  // `LS.Latch'. The PHI nodes need to be updated to reflect that.
1139  for (Instruction &I : *LS.LatchExit) {
1140  if (PHINode *PN = dyn_cast<PHINode>(&I))
1141  replacePHIBlock(PN, LS.Latch, RRI.ExitSelector);
1142  else
1143  break;
1144  }
1145 
1146  return RRI;
1147 }
1148 
1149 void LoopConstrainer::rewriteIncomingValuesForPHIs(
1150  LoopStructure &LS, BasicBlock *ContinuationBlock,
1151  const LoopConstrainer::RewrittenRangeInfo &RRI) const {
1152 
1153  unsigned PHIIndex = 0;
1154  for (Instruction &I : *LS.Header) {
1155  auto *PN = dyn_cast<PHINode>(&I);
1156  if (!PN)
1157  break;
1158 
1159  for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i)
1160  if (PN->getIncomingBlock(i) == ContinuationBlock)
1161  PN->setIncomingValue(i, RRI.PHIValuesAtPseudoExit[PHIIndex++]);
1162  }
1163 
1164  LS.IndVarStart = RRI.IndVarEnd;
1165 }
1166 
1167 BasicBlock *LoopConstrainer::createPreheader(const LoopStructure &LS,
1168  BasicBlock *OldPreheader,
1169  const char *Tag) const {
1170 
1171  BasicBlock *Preheader = BasicBlock::Create(Ctx, Tag, &F, LS.Header);
1172  BranchInst::Create(LS.Header, Preheader);
1173 
1174  for (Instruction &I : *LS.Header) {
1175  auto *PN = dyn_cast<PHINode>(&I);
1176  if (!PN)
1177  break;
1178 
1179  for (unsigned i = 0, e = PN->getNumIncomingValues(); i < e; ++i)
1180  replacePHIBlock(PN, OldPreheader, Preheader);
1181  }
1182 
1183  return Preheader;
1184 }
1185 
1186 void LoopConstrainer::addToParentLoopIfNeeded(ArrayRef<BasicBlock *> BBs) {
1187  Loop *ParentLoop = OriginalLoop.getParentLoop();
1188  if (!ParentLoop)
1189  return;
1190 
1191  for (BasicBlock *BB : BBs)
1192  ParentLoop->addBasicBlockToLoop(BB, LI);
1193 }
1194 
1195 Loop *LoopConstrainer::createClonedLoopStructure(Loop *Original, Loop *Parent,
1196  ValueToValueMapTy &VM) {
1197  Loop &New = LPM.addLoop(Parent);
1198 
1199  // Add all of the blocks in Original to the new loop.
1200  for (auto *BB : Original->blocks())
1201  if (LI.getLoopFor(BB) == Original)
1202  New.addBasicBlockToLoop(cast<BasicBlock>(VM[BB]), LI);
1203 
1204  // Add all of the subloops to the new loop.
1205  for (Loop *SubLoop : *Original)
1206  createClonedLoopStructure(SubLoop, &New, VM);
1207 
1208  return &New;
1209 }
1210 
1211 bool LoopConstrainer::run() {
1212  BasicBlock *Preheader = nullptr;
1213  LatchTakenCount = SE.getExitCount(&OriginalLoop, MainLoopStructure.Latch);
1214  Preheader = OriginalLoop.getLoopPreheader();
1215  assert(!isa<SCEVCouldNotCompute>(LatchTakenCount) && Preheader != nullptr &&
1216  "preconditions!");
1217 
1218  OriginalPreheader = Preheader;
1219  MainLoopPreheader = Preheader;
1220 
1221  Optional<SubRanges> MaybeSR = calculateSubRanges();
1222  if (!MaybeSR.hasValue()) {
1223  DEBUG(dbgs() << "irce: could not compute subranges\n");
1224  return false;
1225  }
1226 
1227  SubRanges SR = MaybeSR.getValue();
1228  bool Increasing = MainLoopStructure.IndVarIncreasing;
1229  IntegerType *IVTy =
1230  cast<IntegerType>(MainLoopStructure.IndVarNext->getType());
1231 
1232  SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "irce");
1233  Instruction *InsertPt = OriginalPreheader->getTerminator();
1234 
1235  // It would have been better to make `PreLoop' and `PostLoop'
1236  // `Optional<ClonedLoop>'s, but `ValueToValueMapTy' does not have a copy
1237  // constructor.
1238  ClonedLoop PreLoop, PostLoop;
1239  bool NeedsPreLoop =
1240  Increasing ? SR.LowLimit.hasValue() : SR.HighLimit.hasValue();
1241  bool NeedsPostLoop =
1242  Increasing ? SR.HighLimit.hasValue() : SR.LowLimit.hasValue();
1243 
1244  Value *ExitPreLoopAt = nullptr;
1245  Value *ExitMainLoopAt = nullptr;
1246  const SCEVConstant *MinusOneS =
1247  cast<SCEVConstant>(SE.getConstant(IVTy, -1, true /* isSigned */));
1248 
1249  if (NeedsPreLoop) {
1250  const SCEV *ExitPreLoopAtSCEV = nullptr;
1251 
1252  if (Increasing)
1253  ExitPreLoopAtSCEV = *SR.LowLimit;
1254  else {
1255  if (CanBeSMin(SE, *SR.HighLimit)) {
1256  DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
1257  << "preloop exit limit. HighLimit = " << *(*SR.HighLimit)
1258  << "\n");
1259  return false;
1260  }
1261  ExitPreLoopAtSCEV = SE.getAddExpr(*SR.HighLimit, MinusOneS);
1262  }
1263 
1264  ExitPreLoopAt = Expander.expandCodeFor(ExitPreLoopAtSCEV, IVTy, InsertPt);
1265  ExitPreLoopAt->setName("exit.preloop.at");
1266  }
1267 
1268  if (NeedsPostLoop) {
1269  const SCEV *ExitMainLoopAtSCEV = nullptr;
1270 
1271  if (Increasing)
1272  ExitMainLoopAtSCEV = *SR.HighLimit;
1273  else {
1274  if (CanBeSMin(SE, *SR.LowLimit)) {
1275  DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
1276  << "mainloop exit limit. LowLimit = " << *(*SR.LowLimit)
1277  << "\n");
1278  return false;
1279  }
1280  ExitMainLoopAtSCEV = SE.getAddExpr(*SR.LowLimit, MinusOneS);
1281  }
1282 
1283  ExitMainLoopAt = Expander.expandCodeFor(ExitMainLoopAtSCEV, IVTy, InsertPt);
1284  ExitMainLoopAt->setName("exit.mainloop.at");
1285  }
1286 
1287  // We clone these ahead of time so that we don't have to deal with changing
1288  // and temporarily invalid IR as we transform the loops.
1289  if (NeedsPreLoop)
1290  cloneLoop(PreLoop, "preloop");
1291  if (NeedsPostLoop)
1292  cloneLoop(PostLoop, "postloop");
1293 
1294  RewrittenRangeInfo PreLoopRRI;
1295 
1296  if (NeedsPreLoop) {
1297  Preheader->getTerminator()->replaceUsesOfWith(MainLoopStructure.Header,
1298  PreLoop.Structure.Header);
1299 
1300  MainLoopPreheader =
1301  createPreheader(MainLoopStructure, Preheader, "mainloop");
1302  PreLoopRRI = changeIterationSpaceEnd(PreLoop.Structure, Preheader,
1303  ExitPreLoopAt, MainLoopPreheader);
1304  rewriteIncomingValuesForPHIs(MainLoopStructure, MainLoopPreheader,
1305  PreLoopRRI);
1306  }
1307 
1308  BasicBlock *PostLoopPreheader = nullptr;
1309  RewrittenRangeInfo PostLoopRRI;
1310 
1311  if (NeedsPostLoop) {
1312  PostLoopPreheader =
1313  createPreheader(PostLoop.Structure, Preheader, "postloop");
1314  PostLoopRRI = changeIterationSpaceEnd(MainLoopStructure, MainLoopPreheader,
1315  ExitMainLoopAt, PostLoopPreheader);
1316  rewriteIncomingValuesForPHIs(PostLoop.Structure, PostLoopPreheader,
1317  PostLoopRRI);
1318  }
1319 
1320  BasicBlock *NewMainLoopPreheader =
1321  MainLoopPreheader != Preheader ? MainLoopPreheader : nullptr;
1322  BasicBlock *NewBlocks[] = {PostLoopPreheader, PreLoopRRI.PseudoExit,
1323  PreLoopRRI.ExitSelector, PostLoopRRI.PseudoExit,
1324  PostLoopRRI.ExitSelector, NewMainLoopPreheader};
1325 
1326  // Some of the above may be nullptr, filter them out before passing to
1327  // addToParentLoopIfNeeded.
1328  auto NewBlocksEnd =
1329  std::remove(std::begin(NewBlocks), std::end(NewBlocks), nullptr);
1330 
1331  addToParentLoopIfNeeded(makeArrayRef(std::begin(NewBlocks), NewBlocksEnd));
1332 
1333  DT.recalculate(F);
1334 
1335  if (!PreLoop.Blocks.empty()) {
1336  auto *L = createClonedLoopStructure(
1337  &OriginalLoop, OriginalLoop.getParentLoop(), PreLoop.Map);
1338  formLCSSARecursively(*L, DT, &LI, &SE);
1339  simplifyLoop(L, &DT, &LI, &SE, nullptr, true);
1340  // Pre loops are slow paths, we do not need to perform any loop
1341  // optimizations on them.
1343  }
1344 
1345  if (!PostLoop.Blocks.empty()) {
1346  auto *L = createClonedLoopStructure(
1347  &OriginalLoop, OriginalLoop.getParentLoop(), PostLoop.Map);
1348  formLCSSARecursively(*L, DT, &LI, &SE);
1349  simplifyLoop(L, &DT, &LI, &SE, nullptr, true);
1350  // Post loops are slow paths, we do not need to perform any loop
1351  // optimizations on them.
1353  }
1354 
1355  formLCSSARecursively(OriginalLoop, DT, &LI, &SE);
1356  simplifyLoop(&OriginalLoop, &DT, &LI, &SE, nullptr, true);
1357 
1358  return true;
1359 }
1360 
1361 /// Computes and returns a range of values for the induction variable (IndVar)
1362 /// in which the range check can be safely elided. If it cannot compute such a
1363 /// range, returns None.
1365 InductiveRangeCheck::computeSafeIterationSpace(
1366  ScalarEvolution &SE, const SCEVAddRecExpr *IndVar) const {
1367  // IndVar is of the form "A + B * I" (where "I" is the canonical induction
1368  // variable, that may or may not exist as a real llvm::Value in the loop) and
1369  // this inductive range check is a range check on the "C + D * I" ("C" is
1370  // getOffset() and "D" is getScale()). We rewrite the value being range
1371  // checked to "M + N * IndVar" where "N" = "D * B^(-1)" and "M" = "C - NA".
1372  // Currently we support this only for "B" = "D" = { 1 or -1 }, but the code
1373  // can be generalized as needed.
1374  //
1375  // The actual inequalities we solve are of the form
1376  //
1377  // 0 <= M + 1 * IndVar < L given L >= 0 (i.e. N == 1)
1378  //
1379  // The inequality is satisfied by -M <= IndVar < (L - M) [^1]. All additions
1380  // and subtractions are twos-complement wrapping and comparisons are signed.
1381  //
1382  // Proof:
1383  //
1384  // If there exists IndVar such that -M <= IndVar < (L - M) then it follows
1385  // that -M <= (-M + L) [== Eq. 1]. Since L >= 0, if (-M + L) sign-overflows
1386  // then (-M + L) < (-M). Hence by [Eq. 1], (-M + L) could not have
1387  // overflown.
1388  //
1389  // This means IndVar = t + (-M) for t in [0, L). Hence (IndVar + M) = t.
1390  // Hence 0 <= (IndVar + M) < L
1391 
1392  // [^1]: Note that the solution does _not_ apply if L < 0; consider values M =
1393  // 127, IndVar = 126 and L = -2 in an i8 world.
1394 
1395  if (!IndVar->isAffine())
1396  return None;
1397 
1398  const SCEV *A = IndVar->getStart();
1399  const SCEVConstant *B = dyn_cast<SCEVConstant>(IndVar->getStepRecurrence(SE));
1400  if (!B)
1401  return None;
1402 
1403  const SCEV *C = getOffset();
1404  const SCEVConstant *D = dyn_cast<SCEVConstant>(getScale());
1405  if (D != B)
1406  return None;
1407 
1408  ConstantInt *ConstD = D->getValue();
1409  if (!(ConstD->isMinusOne() || ConstD->isOne()))
1410  return None;
1411 
1412  const SCEV *M = SE.getMinusSCEV(C, A);
1413 
1414  const SCEV *Begin = SE.getNegativeSCEV(M);
1415  const SCEV *UpperLimit = nullptr;
1416 
1417  // We strengthen "0 <= I" to "0 <= I < INT_SMAX" and "I < L" to "0 <= I < L".
1418  // We can potentially do much better here.
1419  if (Value *V = getLength()) {
1420  UpperLimit = SE.getSCEV(V);
1421  } else {
1422  assert(Kind == InductiveRangeCheck::RANGE_CHECK_LOWER && "invariant!");
1423  unsigned BitWidth = cast<IntegerType>(IndVar->getType())->getBitWidth();
1424  UpperLimit = SE.getConstant(APInt::getSignedMaxValue(BitWidth));
1425  }
1426 
1427  const SCEV *End = SE.getMinusSCEV(UpperLimit, M);
1428  return InductiveRangeCheck::Range(Begin, End);
1429 }
1430 
1434  const InductiveRangeCheck::Range &R2) {
1435  if (!R1.hasValue())
1436  return R2;
1437  auto &R1Value = R1.getValue();
1438 
1439  // TODO: we could widen the smaller range and have this work; but for now we
1440  // bail out to keep things simple.
1441  if (R1Value.getType() != R2.getType())
1442  return None;
1443 
1444  const SCEV *NewBegin = SE.getSMaxExpr(R1Value.getBegin(), R2.getBegin());
1445  const SCEV *NewEnd = SE.getSMinExpr(R1Value.getEnd(), R2.getEnd());
1446 
1447  return InductiveRangeCheck::Range(NewBegin, NewEnd);
1448 }
1449 
1450 bool InductiveRangeCheckElimination::runOnLoop(Loop *L, LPPassManager &LPM) {
1451  if (skipLoop(L))
1452  return false;
1453 
1454  if (L->getBlocks().size() >= LoopSizeCutoff) {
1455  DEBUG(dbgs() << "irce: giving up constraining loop, too large\n";);
1456  return false;
1457  }
1458 
1459  BasicBlock *Preheader = L->getLoopPreheader();
1460  if (!Preheader) {
1461  DEBUG(dbgs() << "irce: loop has no preheader, leaving\n");
1462  return false;
1463  }
1464 
1465  LLVMContext &Context = Preheader->getContext();
1467  ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1468  BranchProbabilityInfo &BPI =
1469  getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();
1470 
1471  for (auto BBI : L->getBlocks())
1472  if (BranchInst *TBI = dyn_cast<BranchInst>(BBI->getTerminator()))
1473  InductiveRangeCheck::extractRangeChecksFromBranch(TBI, L, SE, BPI,
1474  RangeChecks);
1475 
1476  if (RangeChecks.empty())
1477  return false;
1478 
1479  auto PrintRecognizedRangeChecks = [&](raw_ostream &OS) {
1480  OS << "irce: looking at loop "; L->print(OS);
1481  OS << "irce: loop has " << RangeChecks.size()
1482  << " inductive range checks: \n";
1483  for (InductiveRangeCheck &IRC : RangeChecks)
1484  IRC.print(OS);
1485  };
1486 
1487  DEBUG(PrintRecognizedRangeChecks(dbgs()));
1488 
1489  if (PrintRangeChecks)
1490  PrintRecognizedRangeChecks(errs());
1491 
1492  const char *FailureReason = nullptr;
1493  Optional<LoopStructure> MaybeLoopStructure =
1494  LoopStructure::parseLoopStructure(SE, BPI, *L, FailureReason);
1495  if (!MaybeLoopStructure.hasValue()) {
1496  DEBUG(dbgs() << "irce: could not parse loop structure: " << FailureReason
1497  << "\n";);
1498  return false;
1499  }
1500  LoopStructure LS = MaybeLoopStructure.getValue();
1501  bool Increasing = LS.IndVarIncreasing;
1502  const SCEV *MinusOne =
1503  SE.getConstant(LS.IndVarNext->getType(), Increasing ? -1 : 1, true);
1504  const SCEVAddRecExpr *IndVar =
1505  cast<SCEVAddRecExpr>(SE.getAddExpr(SE.getSCEV(LS.IndVarNext), MinusOne));
1506 
1508  Instruction *ExprInsertPt = Preheader->getTerminator();
1509 
1510  SmallVector<InductiveRangeCheck, 4> RangeChecksToEliminate;
1511 
1512  IRBuilder<> B(ExprInsertPt);
1513  for (InductiveRangeCheck &IRC : RangeChecks) {
1514  auto Result = IRC.computeSafeIterationSpace(SE, IndVar);
1515  if (Result.hasValue()) {
1516  auto MaybeSafeIterRange =
1517  IntersectRange(SE, SafeIterRange, Result.getValue());
1518  if (MaybeSafeIterRange.hasValue()) {
1519  RangeChecksToEliminate.push_back(IRC);
1520  SafeIterRange = MaybeSafeIterRange.getValue();
1521  }
1522  }
1523  }
1524 
1525  if (!SafeIterRange.hasValue())
1526  return false;
1527 
1528  auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1529  LoopConstrainer LC(*L, getAnalysis<LoopInfoWrapperPass>().getLoopInfo(), LPM,
1530  LS, SE, DT, SafeIterRange.getValue());
1531  bool Changed = LC.run();
1532 
1533  if (Changed) {
1534  auto PrintConstrainedLoopInfo = [L]() {
1535  dbgs() << "irce: in function ";
1536  dbgs() << L->getHeader()->getParent()->getName() << ": ";
1537  dbgs() << "constrained ";
1538  L->print(dbgs());
1539  };
1540 
1541  DEBUG(PrintConstrainedLoopInfo());
1542 
1543  if (PrintChangedLoops)
1544  PrintConstrainedLoopInfo();
1545 
1546  // Optimize away the now-redundant range checks.
1547 
1548  for (InductiveRangeCheck &IRC : RangeChecksToEliminate) {
1549  ConstantInt *FoldedRangeCheck = IRC.getPassingDirection()
1550  ? ConstantInt::getTrue(Context)
1551  : ConstantInt::getFalse(Context);
1552  IRC.getCheckUse()->set(FoldedRangeCheck);
1553  }
1554  }
1555 
1556  return Changed;
1557 }
1558 
1560  return new InductiveRangeCheckElimination;
1561 }
MachineLoop * L
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:81
const NoneType None
Definition: None.h:23
const Use & getOperandUse(unsigned i) const
Definition: User.h:158
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
Definition: PatternMatch.h:506
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:76
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:241
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:64
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:166
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & Context
const SCEV * getExitCount(Loop *L, BasicBlock *ExitingBlock)
Get the expression for the number of loop iterations for which this loop is guaranteed not to exit vi...
const SCEV * getConstant(ConstantInt *V)
bool hasValue() const
Definition: Optional.h:125
size_t i
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
Definition: Compiler.h:450
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
Definition: Metadata.cpp:819
Inductive range check false
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:414
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
The main scalar evolution driver.
bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
unsigned less than
Definition: InstrTypes.h:905
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop...
Definition: LoopInfo.h:160
const_iterator begin(StringRef path)
Get begin iterator over path.
Definition: Path.cpp:233
LoopT * getParentLoop() const
Definition: LoopInfo.h:103
Metadata node.
Definition: Metadata.h:830
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
Definition: DerivedTypes.h:65
FunctionType * getType(LLVMContext &Context, ID id, ArrayRef< Type * > Tys=None)
Return the function type for an intrinsic.
Definition: Function.cpp:905
#define R2(n)
static cl::opt< bool > PrintRangeChecks("irce-print-range-checks", cl::Hidden, cl::init(false))
const std::vector< BlockT * > & getBlocks() const
Get a list of the basic blocks which make up this loop.
Definition: LoopInfo.h:139
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
Definition: Constants.h:214
BlockT * getHeader() const
Definition: LoopInfo.h:102
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:431
const SCEV * getStart() const
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
Definition: LoopInfoImpl.h:157
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
Definition: LCSSA.cpp:299
The SCEV is loop-invariant.
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:41
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
bool isUnconditional() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:143
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:440
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
Definition: BasicBlock.cpp:116
void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
Definition: AsmWriter.cpp:3394
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:588
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:257
static cl::opt< int > MaxExitProbReciprocal("irce-max-exit-prob-reciprocal", cl::Hidden, cl::init(10))
bool isLoopSimplifyForm() const
Return true if the Loop is in the form that the LoopSimplify form transforms loops to...
Definition: LoopInfo.cpp:190
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS...
static void DisableAllLoopOptsOnLoop(Loop &L)
#define F(x, y, z)
Definition: MD5.cpp:51
ConstantRange getSignedRange(const SCEV *S)
Determine the signed range for a particular SCEV.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
Definition: LoopInfoImpl.h:188
This node represents a polynomial recurrence on the trip count of the specified loop.
static const char * ClonedLoopTag
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
const T & getValue() const LLVM_LVALUE_FUNCTION
Definition: Optional.h:121
Inductive range check elimination
BasicBlock * getSuccessor(unsigned i) const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:392
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
static Optional< InductiveRangeCheck::Range > IntersectRange(ScalarEvolution &SE, const Optional< InductiveRangeCheck::Range > &R1, const InductiveRangeCheck::Range &R2)
Legacy analysis pass which computes BranchProbabilityInfo.
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
Definition: ValueMapper.h:62
unsigned getNumIncomingValues() const
Return the number of incoming edges.
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:24
iterator_range< block_iterator > blocks() const
Definition: LoopInfo.h:143
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values...
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:109
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
Type * getType() const
Return the LLVM type of this SCEV expression.
LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)
Return the "disposition" of the given SCEV with respect to the given loop.
Conditional or Unconditional Branch instruction.
void initializeInductiveRangeCheckEliminationPass(PassRegistry &)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
static bool CanBeSMin(ScalarEvolution &SE, const SCEV *S)
const SCEV * getSMaxExpr(const SCEV *LHS, const SCEV *RHS)
Represent the analysis usage information of a pass.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
Definition: LoopInfo.h:109
uint32_t Offset
Value * get() const
Definition: Use.h:82
This instruction compares its operands according to the predicate given to the constructor.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return LHS-RHS. Minus is represented in SCEV as A+B*-1.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:880
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
Value * expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I)
Insert code to directly compute the specified SCEV expression into the program.
static const unsigned End
Value * getOperand(unsigned i) const
Definition: User.h:145
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:93
Class to represent integer types.
Definition: DerivedTypes.h:39
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:960
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
signed greater than
Definition: InstrTypes.h:907
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
Definition: LoopInfo.cpp:246
const SCEV * getSMinExpr(const SCEV *LHS, const SCEV *RHS)
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:234
static cl::opt< bool > SkipProfitabilityChecks("irce-skip-profitability-checks", cl::Hidden, cl::init(false))
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
void setIncomingBlock(unsigned i, BasicBlock *BB)
INITIALIZE_PASS_BEGIN(InductiveRangeCheckElimination,"irce","Inductive range check elimination", false, false) INITIALIZE_PASS_END(InductiveRangeCheckElimination
static bool CanBeSMax(ScalarEvolution &SE, const SCEV *S)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:175
signed less than
Definition: InstrTypes.h:909
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:1000
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:558
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
ConstantInt * getValue() const
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:506
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
signed less or equal
Definition: InstrTypes.h:910
Class for arbitrary precision integers.
Definition: APInt.h:77
static Constant * getFalse(Type *Ty)
For a boolean type, or a vector of boolean type, return false, or a vector with every element false...
const SCEV * getSignExtendExpr(const SCEV *Op, Type *Ty)
Value * getIncomingValueForBlock(const BasicBlock *BB) const
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM...
Definition: ValueMapper.h:243
This class uses information about analyze scalars to rewrite expressions in canonical form...
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical add expression, or something simpler if possible.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:464
If this flag is set, the remapper ignores missing function-local entries (Argument, Instruction, BasicBlock) that are not in the value map.
Definition: ValueMapper.h:80
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1132
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:384
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
Value * getCondition() const
Analysis providing branch probability information.
This class represents an analyzed expression in the program.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:368
#define I(x, y, z)
Definition: MD5.cpp:54
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
void getLoopAnalysisUsage(AnalysisUsage &AU)
Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.
Definition: LoopUtils.cpp:938
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
static cl::opt< unsigned > LoopSizeCutoff("irce-loop-size-cutoff", cl::Hidden, cl::init(64))
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
const unsigned Kind
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:441
static cl::opt< bool > PrintChangedLoops("irce-print-changed-loops", cl::Hidden, cl::init(false))
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:33
const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
void print(raw_ostream &OS) const
Print out the internal representation of this scalar to the specified stream.
LLVM Value Representation.
Definition: Value.h:71
succ_range successors(BasicBlock *BB)
Definition: IR/CFG.h:143
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:239
static const Function * getParent(const Value *V)
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
#define DEBUG(X)
Definition: Debug.h:100
unsigned greater than
Definition: InstrTypes.h:903
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
void print(raw_ostream &OS, unsigned Depth=0, bool Verbose=false) const
Print loop with all the BBs inside it.
Definition: LoopInfoImpl.h:316
void setIncomingValue(unsigned i, Value *V)
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
Root of the metadata hierarchy.
Definition: Metadata.h:55
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr)
CloneBasicBlock - Return a copy of the specified basic block, but without embedding the block into a ...
const BasicBlock * getParent() const
Definition: Instruction.h:62
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:206
Pass * createInductiveRangeCheckEliminationPass()
signed greater or equal
Definition: InstrTypes.h:908
This class represents a constant integer value.