LLVM  4.0.0
LoopUtils.cpp
Go to the documentation of this file.
1 //===-- LoopUtils.cpp - Loop Utility functions -------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines common loop utility functions.
11 //
12 //===----------------------------------------------------------------------===//
13 
19 #include "llvm/Analysis/LoopInfo.h"
20 #include "llvm/Analysis/LoopPass.h"
25 #include "llvm/IR/Dominators.h"
26 #include "llvm/IR/Instructions.h"
27 #include "llvm/IR/Module.h"
28 #include "llvm/IR/PatternMatch.h"
29 #include "llvm/IR/ValueHandle.h"
30 #include "llvm/Pass.h"
31 #include "llvm/Support/Debug.h"
32 
33 using namespace llvm;
34 using namespace llvm::PatternMatch;
35 
36 #define DEBUG_TYPE "loop-utils"
37 
40  for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use)
41  if (!Set.count(dyn_cast<Instruction>(*Use)))
42  return false;
43  return true;
44 }
45 
47  switch (Kind) {
48  default:
49  break;
50  case RK_IntegerAdd:
51  case RK_IntegerMult:
52  case RK_IntegerOr:
53  case RK_IntegerAnd:
54  case RK_IntegerXor:
55  case RK_IntegerMinMax:
56  return true;
57  }
58  return false;
59 }
60 
62  return (Kind != RK_NoRecurrence) && !isIntegerRecurrenceKind(Kind);
63 }
64 
66  switch (Kind) {
67  default:
68  break;
69  case RK_IntegerAdd:
70  case RK_IntegerMult:
71  case RK_FloatAdd:
72  case RK_FloatMult:
73  return true;
74  }
75  return false;
76 }
77 
82  if (!Phi->hasOneUse())
83  return Phi;
84 
85  const APInt *M = nullptr;
86  Instruction *I, *J = cast<Instruction>(Phi->use_begin()->getUser());
87 
88  // Matches either I & 2^x-1 or 2^x-1 & I. If we find a match, we update RT
89  // with a new integer type of the corresponding bit width.
91  m_And(m_APInt(M), m_Instruction(I))))) {
92  int32_t Bits = (*M + 1).exactLogBase2();
93  if (Bits > 0) {
94  RT = IntegerType::get(Phi->getContext(), Bits);
95  Visited.insert(Phi);
96  CI.insert(J);
97  return J;
98  }
99  }
100  return Phi;
101 }
102 
104  Instruction *Start, Instruction *Exit, Type *RT, bool &IsSigned,
107 
109  bool FoundOneOperand = false;
110  unsigned DstSize = RT->getPrimitiveSizeInBits();
111  Worklist.push_back(Exit);
112 
113  // Traverse the instructions in the reduction expression, beginning with the
114  // exit value.
115  while (!Worklist.empty()) {
116  Instruction *I = Worklist.pop_back_val();
117  for (Use &U : I->operands()) {
118 
119  // Terminate the traversal if the operand is not an instruction, or we
120  // reach the starting value.
121  Instruction *J = dyn_cast<Instruction>(U.get());
122  if (!J || J == Start)
123  continue;
124 
125  // Otherwise, investigate the operation if it is also in the expression.
126  if (Visited.count(J)) {
127  Worklist.push_back(J);
128  continue;
129  }
130 
131  // If the operand is not in Visited, it is not a reduction operation, but
132  // it does feed into one. Make sure it is either a single-use sign- or
133  // zero-extend instruction.
134  CastInst *Cast = dyn_cast<CastInst>(J);
135  bool IsSExtInst = isa<SExtInst>(J);
136  if (!Cast || !Cast->hasOneUse() || !(isa<ZExtInst>(J) || IsSExtInst))
137  return false;
138 
139  // Ensure the source type of the extend is no larger than the reduction
140  // type. It is not necessary for the types to be identical.
141  unsigned SrcSize = Cast->getSrcTy()->getPrimitiveSizeInBits();
142  if (SrcSize > DstSize)
143  return false;
144 
145  // Furthermore, ensure that all such extends are of the same kind.
146  if (FoundOneOperand) {
147  if (IsSigned != IsSExtInst)
148  return false;
149  } else {
150  FoundOneOperand = true;
151  IsSigned = IsSExtInst;
152  }
153 
154  // Lastly, if the source type of the extend matches the reduction type,
155  // add the extend to CI so that we can avoid accounting for it in the
156  // cost model.
157  if (SrcSize == DstSize)
158  CI.insert(Cast);
159  }
160  }
161  return true;
162 }
163 
165  Loop *TheLoop, bool HasFunNoNaNAttr,
166  RecurrenceDescriptor &RedDes) {
167  if (Phi->getNumIncomingValues() != 2)
168  return false;
169 
170  // Reduction variables are only found in the loop header block.
171  if (Phi->getParent() != TheLoop->getHeader())
172  return false;
173 
174  // Obtain the reduction start value from the value that comes from the loop
175  // preheader.
176  Value *RdxStart = Phi->getIncomingValueForBlock(TheLoop->getLoopPreheader());
177 
178  // ExitInstruction is the single value which is used outside the loop.
179  // We only allow for a single reduction value to be used outside the loop.
180  // This includes users of the reduction, variables (which form a cycle
181  // which ends in the phi node).
182  Instruction *ExitInstruction = nullptr;
183  // Indicates that we found a reduction operation in our scan.
184  bool FoundReduxOp = false;
185 
186  // We start with the PHI node and scan for all of the users of this
187  // instruction. All users must be instructions that can be used as reduction
188  // variables (such as ADD). We must have a single out-of-block user. The cycle
189  // must include the original PHI.
190  bool FoundStartPHI = false;
191 
192  // To recognize min/max patterns formed by a icmp select sequence, we store
193  // the number of instruction we saw from the recognized min/max pattern,
194  // to make sure we only see exactly the two instructions.
195  unsigned NumCmpSelectPatternInst = 0;
196  InstDesc ReduxDesc(false, nullptr);
197 
198  // Data used for determining if the recurrence has been type-promoted.
199  Type *RecurrenceType = Phi->getType();
201  Instruction *Start = Phi;
202  bool IsSigned = false;
203 
204  SmallPtrSet<Instruction *, 8> VisitedInsts;
206 
207  // Return early if the recurrence kind does not match the type of Phi. If the
208  // recurrence kind is arithmetic, we attempt to look through AND operations
209  // resulting from the type promotion performed by InstCombine. Vector
210  // operations are not limited to the legal integer widths, so we may be able
211  // to evaluate the reduction in the narrower width.
212  if (RecurrenceType->isFloatingPointTy()) {
213  if (!isFloatingPointRecurrenceKind(Kind))
214  return false;
215  } else {
216  if (!isIntegerRecurrenceKind(Kind))
217  return false;
218  if (isArithmeticRecurrenceKind(Kind))
219  Start = lookThroughAnd(Phi, RecurrenceType, VisitedInsts, CastInsts);
220  }
221 
222  Worklist.push_back(Start);
223  VisitedInsts.insert(Start);
224 
225  // A value in the reduction can be used:
226  // - By the reduction:
227  // - Reduction operation:
228  // - One use of reduction value (safe).
229  // - Multiple use of reduction value (not safe).
230  // - PHI:
231  // - All uses of the PHI must be the reduction (safe).
232  // - Otherwise, not safe.
233  // - By one instruction outside of the loop (safe).
234  // - By further instructions outside of the loop (not safe).
235  // - By an instruction that is not part of the reduction (not safe).
236  // This is either:
237  // * An instruction type other than PHI or the reduction operation.
238  // * A PHI in the header other than the initial PHI.
239  while (!Worklist.empty()) {
240  Instruction *Cur = Worklist.back();
241  Worklist.pop_back();
242 
243  // No Users.
244  // If the instruction has no users then this is a broken chain and can't be
245  // a reduction variable.
246  if (Cur->use_empty())
247  return false;
248 
249  bool IsAPhi = isa<PHINode>(Cur);
250 
251  // A header PHI use other than the original PHI.
252  if (Cur != Phi && IsAPhi && Cur->getParent() == Phi->getParent())
253  return false;
254 
255  // Reductions of instructions such as Div, and Sub is only possible if the
256  // LHS is the reduction variable.
257  if (!Cur->isCommutative() && !IsAPhi && !isa<SelectInst>(Cur) &&
258  !isa<ICmpInst>(Cur) && !isa<FCmpInst>(Cur) &&
259  !VisitedInsts.count(dyn_cast<Instruction>(Cur->getOperand(0))))
260  return false;
261 
262  // Any reduction instruction must be of one of the allowed kinds. We ignore
263  // the starting value (the Phi or an AND instruction if the Phi has been
264  // type-promoted).
265  if (Cur != Start) {
266  ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);
267  if (!ReduxDesc.isRecurrence())
268  return false;
269  }
270 
271  // A reduction operation must only have one use of the reduction value.
272  if (!IsAPhi && Kind != RK_IntegerMinMax && Kind != RK_FloatMinMax &&
273  hasMultipleUsesOf(Cur, VisitedInsts))
274  return false;
275 
276  // All inputs to a PHI node must be a reduction value.
277  if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
278  return false;
279 
280  if (Kind == RK_IntegerMinMax &&
281  (isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
282  ++NumCmpSelectPatternInst;
283  if (Kind == RK_FloatMinMax && (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
284  ++NumCmpSelectPatternInst;
285 
286  // Check whether we found a reduction operator.
287  FoundReduxOp |= !IsAPhi && Cur != Start;
288 
289  // Process users of current instruction. Push non-PHI nodes after PHI nodes
290  // onto the stack. This way we are going to have seen all inputs to PHI
291  // nodes once we get to them.
294  for (User *U : Cur->users()) {
295  Instruction *UI = cast<Instruction>(U);
296 
297  // Check if we found the exit user.
298  BasicBlock *Parent = UI->getParent();
299  if (!TheLoop->contains(Parent)) {
300  // Exit if you find multiple outside users or if the header phi node is
301  // being used. In this case the user uses the value of the previous
302  // iteration, in which case we would loose "VF-1" iterations of the
303  // reduction operation if we vectorize.
304  if (ExitInstruction != nullptr || Cur == Phi)
305  return false;
306 
307  // The instruction used by an outside user must be the last instruction
308  // before we feed back to the reduction phi. Otherwise, we loose VF-1
309  // operations on the value.
310  if (!is_contained(Phi->operands(), Cur))
311  return false;
312 
313  ExitInstruction = Cur;
314  continue;
315  }
316 
317  // Process instructions only once (termination). Each reduction cycle
318  // value must only be used once, except by phi nodes and min/max
319  // reductions which are represented as a cmp followed by a select.
320  InstDesc IgnoredVal(false, nullptr);
321  if (VisitedInsts.insert(UI).second) {
322  if (isa<PHINode>(UI))
323  PHIs.push_back(UI);
324  else
325  NonPHIs.push_back(UI);
326  } else if (!isa<PHINode>(UI) &&
327  ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) &&
328  !isa<SelectInst>(UI)) ||
329  !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence()))
330  return false;
331 
332  // Remember that we completed the cycle.
333  if (UI == Phi)
334  FoundStartPHI = true;
335  }
336  Worklist.append(PHIs.begin(), PHIs.end());
337  Worklist.append(NonPHIs.begin(), NonPHIs.end());
338  }
339 
340  // This means we have seen one but not the other instruction of the
341  // pattern or more than just a select and cmp.
342  if ((Kind == RK_IntegerMinMax || Kind == RK_FloatMinMax) &&
343  NumCmpSelectPatternInst != 2)
344  return false;
345 
346  if (!FoundStartPHI || !FoundReduxOp || !ExitInstruction)
347  return false;
348 
349  // If we think Phi may have been type-promoted, we also need to ensure that
350  // all source operands of the reduction are either SExtInsts or ZEstInsts. If
351  // so, we will be able to evaluate the reduction in the narrower bit width.
352  if (Start != Phi)
353  if (!getSourceExtensionKind(Start, ExitInstruction, RecurrenceType,
354  IsSigned, VisitedInsts, CastInsts))
355  return false;
356 
357  // We found a reduction var if we have reached the original phi node and we
358  // only have a single instruction with out-of-loop users.
359 
360  // The ExitInstruction(Instruction which is allowed to have out-of-loop users)
361  // is saved as part of the RecurrenceDescriptor.
362 
363  // Save the description of this reduction variable.
365  RdxStart, ExitInstruction, Kind, ReduxDesc.getMinMaxKind(),
366  ReduxDesc.getUnsafeAlgebraInst(), RecurrenceType, IsSigned, CastInsts);
367  RedDes = RD;
368 
369  return true;
370 }
371 
372 /// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction
373 /// pattern corresponding to a min(X, Y) or max(X, Y).
376 
377  assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) &&
378  "Expect a select instruction");
379  Instruction *Cmp = nullptr;
380  SelectInst *Select = nullptr;
381 
382  // We must handle the select(cmp()) as a single instruction. Advance to the
383  // select.
384  if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) {
385  if (!Cmp->hasOneUse() || !(Select = dyn_cast<SelectInst>(*I->user_begin())))
386  return InstDesc(false, I);
387  return InstDesc(Select, Prev.getMinMaxKind());
388  }
389 
390  // Only handle single use cases for now.
391  if (!(Select = dyn_cast<SelectInst>(I)))
392  return InstDesc(false, I);
393  if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))) &&
394  !(Cmp = dyn_cast<FCmpInst>(I->getOperand(0))))
395  return InstDesc(false, I);
396  if (!Cmp->hasOneUse())
397  return InstDesc(false, I);
398 
399  Value *CmpLeft;
400  Value *CmpRight;
401 
402  // Look for a min/max pattern.
403  if (m_UMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
404  return InstDesc(Select, MRK_UIntMin);
405  else if (m_UMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
406  return InstDesc(Select, MRK_UIntMax);
407  else if (m_SMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
408  return InstDesc(Select, MRK_SIntMax);
409  else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
410  return InstDesc(Select, MRK_SIntMin);
411  else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
412  return InstDesc(Select, MRK_FloatMin);
413  else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
414  return InstDesc(Select, MRK_FloatMax);
415  else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
416  return InstDesc(Select, MRK_FloatMin);
417  else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select))
418  return InstDesc(Select, MRK_FloatMax);
419 
420  return InstDesc(false, I);
421 }
422 
425  InstDesc &Prev, bool HasFunNoNaNAttr) {
426  bool FP = I->getType()->isFloatingPointTy();
427  Instruction *UAI = Prev.getUnsafeAlgebraInst();
428  if (!UAI && FP && !I->hasUnsafeAlgebra())
429  UAI = I; // Found an unsafe (unvectorizable) algebra instruction.
430 
431  switch (I->getOpcode()) {
432  default:
433  return InstDesc(false, I);
434  case Instruction::PHI:
435  return InstDesc(I, Prev.getMinMaxKind(), Prev.getUnsafeAlgebraInst());
436  case Instruction::Sub:
437  case Instruction::Add:
438  return InstDesc(Kind == RK_IntegerAdd, I);
439  case Instruction::Mul:
440  return InstDesc(Kind == RK_IntegerMult, I);
441  case Instruction::And:
442  return InstDesc(Kind == RK_IntegerAnd, I);
443  case Instruction::Or:
444  return InstDesc(Kind == RK_IntegerOr, I);
445  case Instruction::Xor:
446  return InstDesc(Kind == RK_IntegerXor, I);
447  case Instruction::FMul:
448  return InstDesc(Kind == RK_FloatMult, I, UAI);
449  case Instruction::FSub:
450  case Instruction::FAdd:
451  return InstDesc(Kind == RK_FloatAdd, I, UAI);
452  case Instruction::FCmp:
453  case Instruction::ICmp:
454  case Instruction::Select:
455  if (Kind != RK_IntegerMinMax &&
456  (!HasFunNoNaNAttr || Kind != RK_FloatMinMax))
457  return InstDesc(false, I);
458  return isMinMaxSelectCmpPattern(I, Prev);
459  }
460 }
461 
464  unsigned NumUses = 0;
465  for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E;
466  ++Use) {
467  if (Insts.count(dyn_cast<Instruction>(*Use)))
468  ++NumUses;
469  if (NumUses > 1)
470  return true;
471  }
472 
473  return false;
474 }
476  RecurrenceDescriptor &RedDes) {
477 
478  BasicBlock *Header = TheLoop->getHeader();
479  Function &F = *Header->getParent();
480  bool HasFunNoNaNAttr =
481  F.getFnAttribute("no-nans-fp-math").getValueAsString() == "true";
482 
483  if (AddReductionVar(Phi, RK_IntegerAdd, TheLoop, HasFunNoNaNAttr, RedDes)) {
484  DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");
485  return true;
486  }
487  if (AddReductionVar(Phi, RK_IntegerMult, TheLoop, HasFunNoNaNAttr, RedDes)) {
488  DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n");
489  return true;
490  }
491  if (AddReductionVar(Phi, RK_IntegerOr, TheLoop, HasFunNoNaNAttr, RedDes)) {
492  DEBUG(dbgs() << "Found an OR reduction PHI." << *Phi << "\n");
493  return true;
494  }
495  if (AddReductionVar(Phi, RK_IntegerAnd, TheLoop, HasFunNoNaNAttr, RedDes)) {
496  DEBUG(dbgs() << "Found an AND reduction PHI." << *Phi << "\n");
497  return true;
498  }
499  if (AddReductionVar(Phi, RK_IntegerXor, TheLoop, HasFunNoNaNAttr, RedDes)) {
500  DEBUG(dbgs() << "Found a XOR reduction PHI." << *Phi << "\n");
501  return true;
502  }
503  if (AddReductionVar(Phi, RK_IntegerMinMax, TheLoop, HasFunNoNaNAttr,
504  RedDes)) {
505  DEBUG(dbgs() << "Found a MINMAX reduction PHI." << *Phi << "\n");
506  return true;
507  }
508  if (AddReductionVar(Phi, RK_FloatMult, TheLoop, HasFunNoNaNAttr, RedDes)) {
509  DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
510  return true;
511  }
512  if (AddReductionVar(Phi, RK_FloatAdd, TheLoop, HasFunNoNaNAttr, RedDes)) {
513  DEBUG(dbgs() << "Found an FAdd reduction PHI." << *Phi << "\n");
514  return true;
515  }
516  if (AddReductionVar(Phi, RK_FloatMinMax, TheLoop, HasFunNoNaNAttr, RedDes)) {
517  DEBUG(dbgs() << "Found an float MINMAX reduction PHI." << *Phi << "\n");
518  return true;
519  }
520  // Not a reduction of known type.
521  return false;
522 }
523 
525  DominatorTree *DT) {
526 
527  // Ensure the phi node is in the loop header and has two incoming values.
528  if (Phi->getParent() != TheLoop->getHeader() ||
529  Phi->getNumIncomingValues() != 2)
530  return false;
531 
532  // Ensure the loop has a preheader and a single latch block. The loop
533  // vectorizer will need the latch to set up the next iteration of the loop.
534  auto *Preheader = TheLoop->getLoopPreheader();
535  auto *Latch = TheLoop->getLoopLatch();
536  if (!Preheader || !Latch)
537  return false;
538 
539  // Ensure the phi node's incoming blocks are the loop preheader and latch.
540  if (Phi->getBasicBlockIndex(Preheader) < 0 ||
541  Phi->getBasicBlockIndex(Latch) < 0)
542  return false;
543 
544  // Get the previous value. The previous value comes from the latch edge while
545  // the initial value comes form the preheader edge.
546  auto *Previous = dyn_cast<Instruction>(Phi->getIncomingValueForBlock(Latch));
547  if (!Previous || !TheLoop->contains(Previous) || isa<PHINode>(Previous))
548  return false;
549 
550  // Ensure every user of the phi node is dominated by the previous value. The
551  // dominance requirement ensures the loop vectorizer will not need to
552  // vectorize the initial value prior to the first iteration of the loop.
553  for (User *U : Phi->users())
554  if (auto *I = dyn_cast<Instruction>(U))
555  if (!DT->dominates(Previous, I))
556  return false;
557 
558  return true;
559 }
560 
561 /// This function returns the identity element (or neutral element) for
562 /// the operation K.
564  Type *Tp) {
565  switch (K) {
566  case RK_IntegerXor:
567  case RK_IntegerAdd:
568  case RK_IntegerOr:
569  // Adding, Xoring, Oring zero to a number does not change it.
570  return ConstantInt::get(Tp, 0);
571  case RK_IntegerMult:
572  // Multiplying a number by 1 does not change it.
573  return ConstantInt::get(Tp, 1);
574  case RK_IntegerAnd:
575  // AND-ing a number with an all-1 value does not change it.
576  return ConstantInt::get(Tp, -1, true);
577  case RK_FloatMult:
578  // Multiplying a number by 1 does not change it.
579  return ConstantFP::get(Tp, 1.0L);
580  case RK_FloatAdd:
581  // Adding zero to a number does not change it.
582  return ConstantFP::get(Tp, 0.0L);
583  default:
584  llvm_unreachable("Unknown recurrence kind");
585  }
586 }
587 
588 /// This function translates the recurrence kind to an LLVM binary operator.
590  switch (Kind) {
591  case RK_IntegerAdd:
592  return Instruction::Add;
593  case RK_IntegerMult:
594  return Instruction::Mul;
595  case RK_IntegerOr:
596  return Instruction::Or;
597  case RK_IntegerAnd:
598  return Instruction::And;
599  case RK_IntegerXor:
600  return Instruction::Xor;
601  case RK_FloatMult:
602  return Instruction::FMul;
603  case RK_FloatAdd:
604  return Instruction::FAdd;
605  case RK_IntegerMinMax:
606  return Instruction::ICmp;
607  case RK_FloatMinMax:
608  return Instruction::FCmp;
609  default:
610  llvm_unreachable("Unknown recurrence operation");
611  }
612 }
613 
616  Value *Left, Value *Right) {
618  switch (RK) {
619  default:
620  llvm_unreachable("Unknown min/max recurrence kind");
621  case MRK_UIntMin:
622  P = CmpInst::ICMP_ULT;
623  break;
624  case MRK_UIntMax:
625  P = CmpInst::ICMP_UGT;
626  break;
627  case MRK_SIntMin:
628  P = CmpInst::ICMP_SLT;
629  break;
630  case MRK_SIntMax:
631  P = CmpInst::ICMP_SGT;
632  break;
633  case MRK_FloatMin:
634  P = CmpInst::FCMP_OLT;
635  break;
636  case MRK_FloatMax:
637  P = CmpInst::FCMP_OGT;
638  break;
639  }
640 
641  // We only match FP sequences with unsafe algebra, so we can unconditionally
642  // set it on any generated instructions.
643  IRBuilder<>::FastMathFlagGuard FMFG(Builder);
644  FastMathFlags FMF;
645  FMF.setUnsafeAlgebra();
646  Builder.setFastMathFlags(FMF);
647 
648  Value *Cmp;
649  if (RK == MRK_FloatMin || RK == MRK_FloatMax)
650  Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp");
651  else
652  Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp");
653 
654  Value *Select = Builder.CreateSelect(Cmp, Left, Right, "rdx.minmax.select");
655  return Select;
656 }
657 
658 InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K,
659  const SCEV *Step, BinaryOperator *BOp)
660  : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp) {
661  assert(IK != IK_NoInduction && "Not an induction");
662 
663  // Start value type should match the induction kind and the value
664  // itself should not be null.
665  assert(StartValue && "StartValue is null");
666  assert((IK != IK_PtrInduction || StartValue->getType()->isPointerTy()) &&
667  "StartValue is not a pointer for pointer induction");
668  assert((IK != IK_IntInduction || StartValue->getType()->isIntegerTy()) &&
669  "StartValue is not an integer for integer induction");
670 
671  // Check the Step Value. It should be non-zero integer value.
672  assert((!getConstIntStepValue() || !getConstIntStepValue()->isZero()) &&
673  "Step value is zero");
674 
675  assert((IK != IK_PtrInduction || getConstIntStepValue()) &&
676  "Step value should be constant for pointer induction");
677  assert((IK == IK_FpInduction || Step->getType()->isIntegerTy()) &&
678  "StepValue is not an integer");
679 
680  assert((IK != IK_FpInduction || Step->getType()->isFloatingPointTy()) &&
681  "StepValue is not FP for FpInduction");
682  assert((IK != IK_FpInduction || (InductionBinOp &&
683  (InductionBinOp->getOpcode() == Instruction::FAdd ||
684  InductionBinOp->getOpcode() == Instruction::FSub))) &&
685  "Binary opcode should be specified for FP induction");
686 }
687 
689  ConstantInt *ConstStep = getConstIntStepValue();
690  if (ConstStep && (ConstStep->isOne() || ConstStep->isMinusOne()))
691  return ConstStep->getSExtValue();
692  return 0;
693 }
694 
696  if (isa<SCEVConstant>(Step))
697  return dyn_cast<ConstantInt>(cast<SCEVConstant>(Step)->getValue());
698  return nullptr;
699 }
700 
702  ScalarEvolution *SE,
703  const DataLayout& DL) const {
704 
705  SCEVExpander Exp(*SE, DL, "induction");
706  assert(Index->getType() == Step->getType() &&
707  "Index type does not match StepValue type");
708  switch (IK) {
709  case IK_IntInduction: {
710  assert(Index->getType() == StartValue->getType() &&
711  "Index type does not match StartValue type");
712 
713  // FIXME: Theoretically, we can call getAddExpr() of ScalarEvolution
714  // and calculate (Start + Index * Step) for all cases, without
715  // special handling for "isOne" and "isMinusOne".
716  // But in the real life the result code getting worse. We mix SCEV
717  // expressions and ADD/SUB operations and receive redundant
718  // intermediate values being calculated in different ways and
719  // Instcombine is unable to reduce them all.
720 
721  if (getConstIntStepValue() &&
723  return B.CreateSub(StartValue, Index);
724  if (getConstIntStepValue() &&
725  getConstIntStepValue()->isOne())
726  return B.CreateAdd(StartValue, Index);
727  const SCEV *S = SE->getAddExpr(SE->getSCEV(StartValue),
728  SE->getMulExpr(Step, SE->getSCEV(Index)));
729  return Exp.expandCodeFor(S, StartValue->getType(), &*B.GetInsertPoint());
730  }
731  case IK_PtrInduction: {
732  assert(isa<SCEVConstant>(Step) &&
733  "Expected constant step for pointer induction");
734  const SCEV *S = SE->getMulExpr(SE->getSCEV(Index), Step);
735  Index = Exp.expandCodeFor(S, Index->getType(), &*B.GetInsertPoint());
736  return B.CreateGEP(nullptr, StartValue, Index);
737  }
738  case IK_FpInduction: {
739  assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value");
740  assert(InductionBinOp &&
741  (InductionBinOp->getOpcode() == Instruction::FAdd ||
742  InductionBinOp->getOpcode() == Instruction::FSub) &&
743  "Original bin op should be defined for FP induction");
744 
745  Value *StepValue = cast<SCEVUnknown>(Step)->getValue();
746 
747  // Floating point operations had to be 'fast' to enable the induction.
749  Flags.setUnsafeAlgebra();
750 
751  Value *MulExp = B.CreateFMul(StepValue, Index);
752  if (isa<Instruction>(MulExp))
753  // We have to check, the MulExp may be a constant.
754  cast<Instruction>(MulExp)->setFastMathFlags(Flags);
755 
756  Value *BOp = B.CreateBinOp(InductionBinOp->getOpcode() , StartValue,
757  MulExp, "induction");
758  if (isa<Instruction>(BOp))
759  cast<Instruction>(BOp)->setFastMathFlags(Flags);
760 
761  return BOp;
762  }
763  case IK_NoInduction:
764  return nullptr;
765  }
766  llvm_unreachable("invalid enum");
767 }
768 
770  ScalarEvolution *SE,
772 
773  // Here we only handle FP induction variables.
774  assert(Phi->getType()->isFloatingPointTy() && "Unexpected Phi type");
775 
776  if (TheLoop->getHeader() != Phi->getParent())
777  return false;
778 
779  // The loop may have multiple entrances or multiple exits; we can analyze
780  // this phi if it has a unique entry value and a unique backedge value.
781  if (Phi->getNumIncomingValues() != 2)
782  return false;
783  Value *BEValue = nullptr, *StartValue = nullptr;
784  if (TheLoop->contains(Phi->getIncomingBlock(0))) {
785  BEValue = Phi->getIncomingValue(0);
786  StartValue = Phi->getIncomingValue(1);
787  } else {
788  assert(TheLoop->contains(Phi->getIncomingBlock(1)) &&
789  "Unexpected Phi node in the loop");
790  BEValue = Phi->getIncomingValue(1);
791  StartValue = Phi->getIncomingValue(0);
792  }
793 
794  BinaryOperator *BOp = dyn_cast<BinaryOperator>(BEValue);
795  if (!BOp)
796  return false;
797 
798  Value *Addend = nullptr;
799  if (BOp->getOpcode() == Instruction::FAdd) {
800  if (BOp->getOperand(0) == Phi)
801  Addend = BOp->getOperand(1);
802  else if (BOp->getOperand(1) == Phi)
803  Addend = BOp->getOperand(0);
804  } else if (BOp->getOpcode() == Instruction::FSub)
805  if (BOp->getOperand(0) == Phi)
806  Addend = BOp->getOperand(1);
807 
808  if (!Addend)
809  return false;
810 
811  // The addend should be loop invariant
812  if (auto *I = dyn_cast<Instruction>(Addend))
813  if (TheLoop->contains(I))
814  return false;
815 
816  // FP Step has unknown SCEV
817  const SCEV *Step = SE->getUnknown(Addend);
818  D = InductionDescriptor(StartValue, IK_FpInduction, Step, BOp);
819  return true;
820 }
821 
825  bool Assume) {
826  Type *PhiTy = Phi->getType();
827 
828  // Handle integer and pointer inductions variables.
829  // Now we handle also FP induction but not trying to make a
830  // recurrent expression from the PHI node in-place.
831 
832  if (!PhiTy->isIntegerTy() && !PhiTy->isPointerTy() &&
833  !PhiTy->isFloatTy() && !PhiTy->isDoubleTy() && !PhiTy->isHalfTy())
834  return false;
835 
836  if (PhiTy->isFloatingPointTy())
837  return isFPInductionPHI(Phi, TheLoop, PSE.getSE(), D);
838 
839  const SCEV *PhiScev = PSE.getSCEV(Phi);
840  const auto *AR = dyn_cast<SCEVAddRecExpr>(PhiScev);
841 
842  // We need this expression to be an AddRecExpr.
843  if (Assume && !AR)
844  AR = PSE.getAsAddRec(Phi);
845 
846  if (!AR) {
847  DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n");
848  return false;
849  }
850 
851  return isInductionPHI(Phi, TheLoop, PSE.getSE(), D, AR);
852 }
853 
855  ScalarEvolution *SE,
857  const SCEV *Expr) {
858  Type *PhiTy = Phi->getType();
859  // We only handle integer and pointer inductions variables.
860  if (!PhiTy->isIntegerTy() && !PhiTy->isPointerTy())
861  return false;
862 
863  // Check that the PHI is consecutive.
864  const SCEV *PhiScev = Expr ? Expr : SE->getSCEV(Phi);
865  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PhiScev);
866 
867  if (!AR) {
868  DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n");
869  return false;
870  }
871 
872  if (AR->getLoop() != TheLoop) {
873  // FIXME: We should treat this as a uniform. Unfortunately, we
874  // don't currently know how to handled uniform PHIs.
875  DEBUG(dbgs() << "LV: PHI is a recurrence with respect to an outer loop.\n");
876  return false;
877  }
878 
879  Value *StartValue =
881  const SCEV *Step = AR->getStepRecurrence(*SE);
882  // Calculate the pointer stride and check if it is consecutive.
883  // The stride may be a constant or a loop invariant integer value.
884  const SCEVConstant *ConstStep = dyn_cast<SCEVConstant>(Step);
885  if (!ConstStep && !SE->isLoopInvariant(Step, TheLoop))
886  return false;
887 
888  if (PhiTy->isIntegerTy()) {
889  D = InductionDescriptor(StartValue, IK_IntInduction, Step);
890  return true;
891  }
892 
893  assert(PhiTy->isPointerTy() && "The PHI must be a pointer");
894  // Pointer induction should be a constant.
895  if (!ConstStep)
896  return false;
897 
898  ConstantInt *CV = ConstStep->getValue();
899  Type *PointerElementType = PhiTy->getPointerElementType();
900  // The pointer stride cannot be determined if the pointer element type is not
901  // sized.
902  if (!PointerElementType->isSized())
903  return false;
904 
905  const DataLayout &DL = Phi->getModule()->getDataLayout();
906  int64_t Size = static_cast<int64_t>(DL.getTypeAllocSize(PointerElementType));
907  if (!Size)
908  return false;
909 
910  int64_t CVSize = CV->getSExtValue();
911  if (CVSize % Size)
912  return false;
913  auto *StepValue = SE->getConstant(CV->getType(), CVSize / Size,
914  true /* signed */);
915  D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue);
916  return true;
917 }
918 
919 /// \brief Returns the instructions that use values defined in the loop.
921  SmallVector<Instruction *, 8> UsedOutside;
922 
923  for (auto *Block : L->getBlocks())
924  // FIXME: I believe that this could use copy_if if the Inst reference could
925  // be adapted into a pointer.
926  for (auto &Inst : *Block) {
927  auto Users = Inst.users();
928  if (any_of(Users, [&](User *U) {
929  auto *Use = cast<Instruction>(U);
930  return !L->contains(Use->getParent());
931  }))
932  UsedOutside.push_back(&Inst);
933  }
934 
935  return UsedOutside;
936 }
937 
939  // By definition, all loop passes need the LoopInfo analysis and the
940  // Dominator tree it depends on. Because they all participate in the loop
941  // pass manager, they must also preserve these.
946 
947  // We must also preserve LoopSimplify and LCSSA. We locally access their IDs
948  // here because users shouldn't directly get them from this header.
949  extern char &LoopSimplifyID;
950  extern char &LCSSAID;
951  AU.addRequiredID(LoopSimplifyID);
952  AU.addPreservedID(LoopSimplifyID);
953  AU.addRequiredID(LCSSAID);
954  AU.addPreservedID(LCSSAID);
955  // This is used in the LPPassManager to perform LCSSA verification on passes
956  // which preserve lcssa form
959 
960  // Loop passes are designed to run inside of a loop pass manager which means
961  // that any function analyses they require must be required by the first loop
962  // pass in the manager (so that it is computed before the loop pass manager
963  // runs) and preserved by all loop pasess in the manager. To make this
964  // reasonably robust, the set needed for most loop passes is maintained here.
965  // If your loop pass requires an analysis not listed here, you will need to
966  // carefully audit the loop pass manager nesting structure that results.
974 }
975 
976 /// Manually defined generic "LoopPass" dependency initialization. This is used
977 /// to initialize the exact set of passes from above in \c
978 /// getLoopAnalysisUsage. It can be used within a loop pass's initialization
979 /// with:
980 ///
981 /// INITIALIZE_PASS_DEPENDENCY(LoopPass)
982 ///
983 /// As-if "LoopPass" were a pass.
987  INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
988  INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass)
994 }
995 
996 /// \brief Find string metadata for loop
997 ///
998 /// If it has a value (e.g. {"llvm.distribute", 1} return the value as an
999 /// operand or null otherwise. If the string metadata is not found return
1000 /// Optional's not-a-value.
1002  StringRef Name) {
1003  MDNode *LoopID = TheLoop->getLoopID();
1004  // Return none if LoopID is false.
1005  if (!LoopID)
1006  return None;
1007 
1008  // First operand should refer to the loop id itself.
1009  assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
1010  assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
1011 
1012  // Iterate over LoopID operands and look for MDString Metadata
1013  for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
1014  MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
1015  if (!MD)
1016  continue;
1017  MDString *S = dyn_cast<MDString>(MD->getOperand(0));
1018  if (!S)
1019  continue;
1020  // Return true if MDString holds expected MetaData.
1021  if (Name.equals(S->getString()))
1022  switch (MD->getNumOperands()) {
1023  case 1:
1024  return nullptr;
1025  case 2:
1026  return &MD->getOperand(1);
1027  default:
1028  llvm_unreachable("loop metadata has 0 or 1 operand");
1029  }
1030  }
1031  return None;
1032 }
1033 
1034 /// Returns true if the instruction in a loop is guaranteed to execute at least
1035 /// once.
1037  const DominatorTree *DT, const Loop *CurLoop,
1038  const LoopSafetyInfo *SafetyInfo) {
1039  // We have to check to make sure that the instruction dominates all
1040  // of the exit blocks. If it doesn't, then there is a path out of the loop
1041  // which does not execute this instruction, so we can't hoist it.
1042 
1043  // If the instruction is in the header block for the loop (which is very
1044  // common), it is always guaranteed to dominate the exit blocks. Since this
1045  // is a common case, and can save some work, check it now.
1046  if (Inst.getParent() == CurLoop->getHeader())
1047  // If there's a throw in the header block, we can't guarantee we'll reach
1048  // Inst.
1049  return !SafetyInfo->HeaderMayThrow;
1050 
1051  // Somewhere in this loop there is an instruction which may throw and make us
1052  // exit the loop.
1053  if (SafetyInfo->MayThrow)
1054  return false;
1055 
1056  // Get the exit blocks for the current loop.
1057  SmallVector<BasicBlock *, 8> ExitBlocks;
1058  CurLoop->getExitBlocks(ExitBlocks);
1059 
1060  // Verify that the block dominates each of the exit blocks of the loop.
1061  for (BasicBlock *ExitBlock : ExitBlocks)
1062  if (!DT->dominates(Inst.getParent(), ExitBlock))
1063  return false;
1064 
1065  // As a degenerate case, if the loop is statically infinite then we haven't
1066  // proven anything since there are no exit blocks.
1067  if (ExitBlocks.empty())
1068  return false;
1069 
1070  // FIXME: In general, we have to prove that the loop isn't an infinite loop.
1071  // See http::llvm.org/PR24078 . (The "ExitBlocks.empty()" check above is
1072  // just a special case of this.)
1073  return true;
1074 }
1075 
1077  // Only support loops with a unique exiting block, and a latch.
1078  if (!L->getExitingBlock())
1079  return None;
1080 
1081  // Get the branch weights for the the loop's backedge.
1082  BranchInst *LatchBR =
1083  dyn_cast<BranchInst>(L->getLoopLatch()->getTerminator());
1084  if (!LatchBR || LatchBR->getNumSuccessors() != 2)
1085  return None;
1086 
1087  assert((LatchBR->getSuccessor(0) == L->getHeader() ||
1088  LatchBR->getSuccessor(1) == L->getHeader()) &&
1089  "At least one edge out of the latch must go to the header");
1090 
1091  // To estimate the number of times the loop body was executed, we want to
1092  // know the number of times the backedge was taken, vs. the number of times
1093  // we exited the loop.
1094  uint64_t TrueVal, FalseVal;
1095  if (!LatchBR->extractProfMetadata(TrueVal, FalseVal))
1096  return None;
1097 
1098  if (!TrueVal || !FalseVal)
1099  return 0;
1100 
1101  // Divide the count of the backedge by the count of the edge exiting the loop,
1102  // rounding to nearest.
1103  if (LatchBR->getSuccessor(0) == L->getHeader())
1104  return (TrueVal + (FalseVal / 2)) / FalseVal;
1105  else
1106  return (FalseVal + (TrueVal / 2)) / TrueVal;
1107 }
Legacy wrapper pass to provide the GlobalsAAResult object.
MachineLoop * L
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
const NoneType None
Definition: None.h:23
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
Definition: PatternMatch.h:506
static bool isArithmeticRecurrenceKind(RecurrenceKind Kind)
Returns true if the recurrence kind is an arithmetic kind.
Definition: LoopUtils.cpp:65
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
Definition: Constants.h:177
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:64
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1554
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:122
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1032
static bool isFPInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, InductionDescriptor &D)
Returns true if Phi is a floating point induction in the loop L.
Definition: LoopUtils.cpp:769
const SCEV * getConstant(ConstantInt *V)
size_t i
This is the interface for a simple mod/ref and alias analysis over globals.
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1040
The main scalar evolution driver.
int getConsecutiveDirection() const
Get the consecutive direction.
Definition: LoopUtils.cpp:688
A global registry used in conjunction with static constructors to make pluggable components (like tar...
Definition: Registry.h:45
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:380
unsigned less than
Definition: InstrTypes.h:905
static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes)
Returns true if Phi is a reduction in TheLoop.
Definition: LoopUtils.cpp:475
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:886
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition: Type.h:148
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:100
Metadata node.
Definition: Metadata.h:830
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:234
MaxMin_match< FCmpInst, LHS, RHS, ufmax_pred_ty > m_UnordFMax(const LHS &L, const RHS &R)
Match an 'unordered' floating point maximum function.
iv Induction Variable Users
Definition: IVUsers.cpp:51
static InstDesc isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, InstDesc &Prev, bool HasFunNoNaNAttr)
Returns a struct describing if the instruction 'I' can be a recurrence variable of type 'Kind'...
Definition: LoopUtils.cpp:424
const std::vector< BlockT * > & getBlocks() const
Get a list of the basic blocks which make up this loop.
Definition: LoopInfo.h:139
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
Definition: Constants.h:214
op_iterator op_begin()
Definition: User.h:205
BlockT * getHeader() const
Definition: LoopInfo.h:102
static Instruction * lookThroughAnd(PHINode *Phi, Type *&RT, SmallPtrSetImpl< Instruction * > &Visited, SmallPtrSetImpl< Instruction * > &CI)
Determines if Phi may have been type-promoted.
Definition: LoopUtils.cpp:79
Type * getPointerElementType() const
Definition: Type.h:358
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
Definition: LoopInfoImpl.h:157
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:41
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
This is the interface for a SCEV-based alias analysis.
void initializeLoopPassPass(PassRegistry &)
Manually defined generic "LoopPass" dependency initialization.
Definition: LoopUtils.cpp:984
This class represents the LLVM 'select' instruction.
struct fuzzer::@269 Flags
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:578
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:588
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:166
static InstDesc isMinMaxSelectCmpPattern(Instruction *I, InstDesc &Prev)
Returns a struct describing if the instruction if the instruction is a Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y) or max(X, Y).
Definition: LoopUtils.cpp:375
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:813
MaxMin_match< FCmpInst, LHS, RHS, ufmin_pred_ty > m_UnordFMin(const LHS &L, const RHS &R)
Match an 'unordered' floating point minimum function.
ConstantInt * getConstIntStepValue() const
Definition: LoopUtils.cpp:695
SmallVector< Instruction *, 8 > findDefsUsedOutsideOfLoop(Loop *L)
Returns the instructions that use values defined in the loop.
Definition: LoopUtils.cpp:920
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:65
#define F(x, y, z)
Definition: MD5.cpp:51
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
Definition: PatternMatch.h:117
Value * transform(IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL) const
Compute the transformed value of Index at offset StartValue using step StepValue. ...
Definition: LoopUtils.cpp:701
This node represents a polynomial recurrence on the trip count of the specified loop.
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition: Type.h:142
This instruction compares its operands according to the predicate given to the constructor.
BasicBlock * getSuccessor(unsigned i) const
MaxMin_match< FCmpInst, LHS, RHS, ofmin_pred_ty > m_OrdFMin(const LHS &L, const RHS &R)
Match an 'ordered' floating point minimum function.
bool hasUnsafeAlgebra() const
Determine whether the unsafe-algebra flag is set.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
static bool areAllUsesIn(Instruction *I, SmallPtrSetImpl< Instruction * > &Set)
Returns true if all uses of the instruction I is within the Set.
Definition: LoopUtils.cpp:38
const SCEVAddRecExpr * getAsAddRec(Value *V)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
Definition: Type.h:160
AnalysisUsage & addPreservedID(const void *ID)
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:835
Not an induction variable.
Definition: LoopUtils.h:265
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
This POD struct holds information about a potential recurrence operation.
Definition: LoopUtils.h:105
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
static bool hasMultipleUsesOf(Instruction *I, SmallPtrSetImpl< Instruction * > &Insts)
Returns true if instruction I has multiple uses in Insts.
Definition: LoopUtils.cpp:462
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Pointer induction var. Step = C / sizeof(elem).
Definition: LoopUtils.h:267
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop, bool HasFunNoNaNAttr, RecurrenceDescriptor &RedDes)
Returns true if Phi is a reduction of type Kind and adds it to the RecurrenceDescriptor.
Definition: LoopUtils.cpp:164
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1561
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:254
#define P(N)
Integer induction variable. Step = C.
Definition: LoopUtils.h:266
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt...
Definition: PatternMatch.h:180
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:109
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Type * getType() const
Return the LLVM type of this SCEV expression.
Conditional or Unconditional Branch instruction.
This is an important base class in LLVM.
Definition: Constant.h:42
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.h:1609
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition: Type.h:145
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1947
char & LCSSAID
Definition: LCSSA.cpp:379
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
APInt Xor(const APInt &LHS, const APInt &RHS)
Bitwise XOR function for APInt.
Definition: APInt.h:1952
bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) const
Retrieve the raw weight values of a conditional branch or select.
Definition: Metadata.cpp:1272
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
Definition: LoopInfo.cpp:212
void setUnsafeAlgebra()
Definition: Operator.h:205
Represent the analysis usage information of a pass.
op_iterator op_end()
Definition: User.h:207
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
Definition: LoopInfo.h:109
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:743
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:880
Value * expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I)
Insert code to directly compute the specified SCEV expression into the program.
static bool isFirstOrderRecurrence(PHINode *Phi, Loop *TheLoop, DominatorTree *DT)
Returns true if Phi is a first-order recurrence.
Definition: LoopUtils.cpp:524
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
Definition: LoopInfoImpl.h:52
Value * getOperand(unsigned i) const
Definition: User.h:145
op_range operands()
Definition: User.h:213
bool isCommutative() const
Return true if the instruction is commutative:
Definition: Instruction.h:385
Optional< unsigned > getLoopEstimatedTripCount(Loop *L)
Get a loop's estimated trip count based on branch weight metadata.
Definition: LoopUtils.cpp:1076
Type * getSrcTy() const
Return the source type, as a convenience.
Definition: InstrTypes.h:845
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:392
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:213
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:654
Optional< const MDOperand * > findStringMetadataForLoop(Loop *TheLoop, StringRef Name)
Find string metadata for loop.
Definition: LoopUtils.cpp:1001
static unsigned getRecurrenceBinOp(RecurrenceKind Kind)
Returns the opcode of binary operation corresponding to the RecurrenceKind.
Definition: LoopUtils.cpp:589
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
signed greater than
Definition: InstrTypes.h:907
char & LoopSimplifyID
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
Definition: Dominators.cpp:218
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Definition: LoopUtils.h:63
0 0 1 0 True if ordered and greater than
Definition: InstrTypes.h:884
BinaryOps getOpcode() const
Definition: InstrTypes.h:541
StringRef getString() const
Definition: Metadata.cpp:424
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:234
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1034
MinMaxRecurrenceKind getMinMaxKind()
Definition: LoopUtils.h:122
Legacy wrapper pass to provide the SCEVAAResult object.
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
A struct for saving information about induction variables.
Definition: LoopUtils.h:261
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:408
AnalysisUsage & addRequiredID(const void *ID)
Definition: Pass.cpp:289
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:58
static bool getSourceExtensionKind(Instruction *Start, Instruction *Exit, Type *RT, bool &IsSigned, SmallPtrSetImpl< Instruction * > &Visited, SmallPtrSetImpl< Instruction * > &CI)
Returns true if all the source operands of a recurrence are either SExtInsts or ZExtInsts.
Definition: LoopUtils.cpp:103
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
static bool isFloatingPointRecurrenceKind(RecurrenceKind Kind)
Returns true if the recurrence kind is a floating point kind.
Definition: LoopUtils.cpp:61
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
signed less than
Definition: InstrTypes.h:909
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:558
ConstantInt * getValue() const
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:623
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
static Constant * getRecurrenceIdentity(RecurrenceKind K, Type *Tp)
Returns identity corresponding to the RecurrenceKind.
Definition: LoopUtils.cpp:563
Value * CreateGEP(Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Definition: IRBuilder.h:1141
Class for arbitrary precision integers.
Definition: APInt.h:77
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:195
iterator_range< user_iterator > users()
Definition: Value.h:370
static Value * createMinMaxOp(IRBuilder<> &Builder, MinMaxRecurrenceKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
Definition: LoopUtils.cpp:614
This class uses information about analyze scalars to rewrite expressions in canonical form...
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical add expression, or something simpler if possible.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1942
use_iterator use_begin()
Definition: Value.h:310
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:528
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:384
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
Captures loop safety information.
Definition: LoopUtils.h:42
This class represents an analyzed expression in the program.
unsigned getNumSuccessors() const
static bool isIntegerRecurrenceKind(RecurrenceKind Kind)
Returns true if the recurrence kind is an integer kind.
Definition: LoopUtils.cpp:46
Floating point induction variable.
Definition: LoopUtils.h:268
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:368
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
#define I(x, y, z)
Definition: MD5.cpp:54
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:383
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
void getLoopAnalysisUsage(AnalysisUsage &AU)
Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.
Definition: LoopUtils.cpp:938
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
const Loop * getLoop() const
InductionDescriptor()
Default constructor - creates an invalid induction.
Definition: LoopUtils.h:273
const unsigned Kind
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:178
bool use_empty() const
Definition: Value.h:299
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:346
MaxMin_match< FCmpInst, LHS, RHS, ofmax_pred_ty > m_OrdFMax(const LHS &L, const RHS &R)
Match an 'ordered' floating point maximum function.
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:108
LLVM Value Representation.
Definition: Value.h:71
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:111
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:220
#define DEBUG(X)
Definition: Debug.h:100
const SCEV * getUnknown(Value *V)
The legacy pass manager's analysis pass to compute loop information.
Definition: LoopInfo.h:831
Convenience struct for specifying and reasoning about fast-math flags.
Definition: Operator.h:168
unsigned greater than
Definition: InstrTypes.h:903
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
This is the interface for LLVM's primary stateless and local alias analysis.
PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...
Definition: PassRegistry.h:40
A single uniqued string.
Definition: Metadata.h:586
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:217
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
Value * CreateFMul(Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:871
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:162
bool isGuaranteedToExecute(const Instruction &Inst, const DominatorTree *DT, const Loop *CurLoop, const LoopSafetyInfo *SafetyInfo)
Returns true if the instruction in a loop is guaranteed to execute at least once. ...
Definition: LoopUtils.cpp:1036
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical multiply expression, or something simpler if possible.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
const BasicBlock * getParent() const
Definition: Instruction.h:62
static bool isInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, InductionDescriptor &D, const SCEV *Expr=nullptr)
Returns true if Phi is an induction in the loop L.
Definition: LoopUtils.cpp:854
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:206
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
Definition: PatternMatch.h:300
RecurrenceKind
This enum represents the kinds of recurrences that we support.
Definition: LoopUtils.h:67
This class represents a constant integer value.
Legacy wrapper pass to provide the BasicAAResult object.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:783