LLVM  16.0.0git
VPlanRecipes.cpp
Go to the documentation of this file.
1 //===- VPlanRecipes.cpp - Implementations for VPlan recipes ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file contains implementations for different VPlan recipes.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlan.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/Twine.h"
19 #include "llvm/IR/BasicBlock.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Instruction.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/Type.h"
24 #include "llvm/IR/Value.h"
25 #include "llvm/Support/Casting.h"
27 #include "llvm/Support/Debug.h"
31 #include <cassert>
32 
33 using namespace llvm;
34 
36 
38 
39 #define LV_NAME "loop-vectorize"
40 #define DEBUG_TYPE LV_NAME
41 
43  switch (getVPDefID()) {
44  case VPWidenMemoryInstructionSC: {
45  return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
46  }
47  case VPReplicateSC:
48  case VPWidenCallSC:
49  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
50  ->mayWriteToMemory();
51  case VPBranchOnMaskSC:
52  return false;
53  case VPWidenIntOrFpInductionSC:
54  case VPWidenCanonicalIVSC:
55  case VPWidenPHISC:
56  case VPBlendSC:
57  case VPWidenSC:
58  case VPWidenGEPSC:
59  case VPReductionSC:
60  case VPWidenSelectSC: {
61  const Instruction *I =
62  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
63  (void)I;
64  assert((!I || !I->mayWriteToMemory()) &&
65  "underlying instruction may write to memory");
66  return false;
67  }
68  default:
69  return true;
70  }
71 }
72 
74  switch (getVPDefID()) {
75  case VPWidenMemoryInstructionSC: {
76  return !cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
77  }
78  case VPReplicateSC:
79  case VPWidenCallSC:
80  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
81  ->mayReadFromMemory();
82  case VPBranchOnMaskSC:
83  return false;
84  case VPWidenIntOrFpInductionSC:
85  case VPWidenCanonicalIVSC:
86  case VPWidenPHISC:
87  case VPBlendSC:
88  case VPWidenSC:
89  case VPWidenGEPSC:
90  case VPReductionSC:
91  case VPWidenSelectSC: {
92  const Instruction *I =
93  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
94  (void)I;
95  assert((!I || !I->mayReadFromMemory()) &&
96  "underlying instruction may read from memory");
97  return false;
98  }
99  default:
100  return true;
101  }
102 }
103 
105  switch (getVPDefID()) {
106  case VPPredInstPHISC:
107  return false;
108  case VPWidenIntOrFpInductionSC:
109  case VPWidenPointerInductionSC:
110  case VPWidenCanonicalIVSC:
111  case VPWidenPHISC:
112  case VPBlendSC:
113  case VPWidenSC:
114  case VPWidenGEPSC:
115  case VPReductionSC:
116  case VPWidenSelectSC:
117  case VPScalarIVStepsSC: {
118  const Instruction *I =
119  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
120  (void)I;
121  assert((!I || !I->mayHaveSideEffects()) &&
122  "underlying instruction has side-effects");
123  return false;
124  }
125  case VPReplicateSC: {
126  auto *R = cast<VPReplicateRecipe>(this);
127  return R->getUnderlyingInstr()->mayHaveSideEffects();
128  }
129  default:
130  return true;
131  }
132 }
133 
135  auto Lane = VPLane::getLastLaneForVF(State.VF);
136  VPValue *ExitValue = getOperand(0);
138  Lane = VPLane::getFirstLane();
139  Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)),
140  State.Builder.GetInsertBlock());
141 }
142 
144  assert(!Parent && "Recipe already in some VPBasicBlock");
145  assert(InsertPos->getParent() &&
146  "Insertion position not in any VPBasicBlock");
147  Parent = InsertPos->getParent();
148  Parent->getRecipeList().insert(InsertPos->getIterator(), this);
149 }
150 
153  assert(!Parent && "Recipe already in some VPBasicBlock");
154  assert(I == BB.end() || I->getParent() == &BB);
155  Parent = &BB;
156  BB.getRecipeList().insert(I, this);
157 }
158 
160  assert(!Parent && "Recipe already in some VPBasicBlock");
161  assert(InsertPos->getParent() &&
162  "Insertion position not in any VPBasicBlock");
163  Parent = InsertPos->getParent();
164  Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
165 }
166 
168  assert(getParent() && "Recipe not in any VPBasicBlock");
170  Parent = nullptr;
171 }
172 
174  assert(getParent() && "Recipe not in any VPBasicBlock");
175  return getParent()->getRecipeList().erase(getIterator());
176 }
177 
180  insertAfter(InsertPos);
181 }
182 
186  insertBefore(BB, I);
187 }
188 
189 void VPInstruction::generateInstruction(VPTransformState &State,
190  unsigned Part) {
191  IRBuilderBase &Builder = State.Builder;
192  Builder.SetCurrentDebugLocation(DL);
193 
195  Value *A = State.get(getOperand(0), Part);
196  Value *B = State.get(getOperand(1), Part);
197  Value *V =
198  Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
199  State.set(this, V, Part);
200  return;
201  }
202 
203  switch (getOpcode()) {
204  case VPInstruction::Not: {
205  Value *A = State.get(getOperand(0), Part);
206  Value *V = Builder.CreateNot(A, Name);
207  State.set(this, V, Part);
208  break;
209  }
210  case VPInstruction::ICmpULE: {
211  Value *IV = State.get(getOperand(0), Part);
212  Value *TC = State.get(getOperand(1), Part);
213  Value *V = Builder.CreateICmpULE(IV, TC, Name);
214  State.set(this, V, Part);
215  break;
216  }
217  case Instruction::Select: {
218  Value *Cond = State.get(getOperand(0), Part);
219  Value *Op1 = State.get(getOperand(1), Part);
220  Value *Op2 = State.get(getOperand(2), Part);
221  Value *V = Builder.CreateSelect(Cond, Op1, Op2, Name);
222  State.set(this, V, Part);
223  break;
224  }
226  // Get first lane of vector induction variable.
227  Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
228  // Get the original loop tripcount.
229  Value *ScalarTC = State.get(getOperand(1), Part);
230 
231  auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
232  auto *PredTy = VectorType::get(Int1Ty, State.VF);
233  Instruction *Call = Builder.CreateIntrinsic(
234  Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()},
235  {VIVElem0, ScalarTC}, nullptr, Name);
236  State.set(this, Call, Part);
237  break;
238  }
240  // Generate code to combine the previous and current values in vector v3.
241  //
242  // vector.ph:
243  // v_init = vector(..., ..., ..., a[-1])
244  // br vector.body
245  //
246  // vector.body
247  // i = phi [0, vector.ph], [i+4, vector.body]
248  // v1 = phi [v_init, vector.ph], [v2, vector.body]
249  // v2 = a[i, i+1, i+2, i+3];
250  // v3 = vector(v1(3), v2(0, 1, 2))
251 
252  // For the first part, use the recurrence phi (v1), otherwise v2.
253  auto *V1 = State.get(getOperand(0), 0);
254  Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1);
255  if (!PartMinus1->getType()->isVectorTy()) {
256  State.set(this, PartMinus1, Part);
257  } else {
258  Value *V2 = State.get(getOperand(1), Part);
259  State.set(this, Builder.CreateVectorSplice(PartMinus1, V2, -1, Name),
260  Part);
261  }
262  break;
263  }
266  Value *Next = nullptr;
267  if (Part == 0) {
269  auto *Phi = State.get(getOperand(0), 0);
270  // The loop step is equal to the vectorization factor (num of SIMD
271  // elements) times the unroll factor (num of SIMD instructions).
272  Value *Step =
273  createStepForVF(Builder, Phi->getType(), State.VF, State.UF);
274  Next = Builder.CreateAdd(Phi, Step, Name, IsNUW, false);
275  } else {
276  Next = State.get(this, 0);
277  }
278 
279  State.set(this, Next, Part);
280  break;
281  }
282 
286  auto *IV = State.get(getOperand(0), VPIteration(0, 0));
287  if (Part == 0) {
288  State.set(this, IV, Part);
289  break;
290  }
291 
292  // The canonical IV is incremented by the vectorization factor (num of SIMD
293  // elements) times the unroll part.
294  Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
295  Value *Next = Builder.CreateAdd(IV, Step, Name, IsNUW, false);
296  State.set(this, Next, Part);
297  break;
298  }
300  if (Part != 0)
301  break;
302 
303  Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
304  VPRegionBlock *ParentRegion = getParent()->getParent();
305  VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
306 
307  // Replace the temporary unreachable terminator with a new conditional
308  // branch, hooking it up to backward destination for exiting blocks now and
309  // to forward destination(s) later when they are created.
310  BranchInst *CondBr =
311  Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr);
312 
313  if (getParent()->isExiting())
314  CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
315 
316  CondBr->setSuccessor(0, nullptr);
317  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
318  break;
319  }
321  if (Part != 0)
322  break;
323  // First create the compare.
324  Value *IV = State.get(getOperand(0), Part);
325  Value *TC = State.get(getOperand(1), Part);
326  Value *Cond = Builder.CreateICmpEQ(IV, TC);
327 
328  // Now create the branch.
329  auto *Plan = getParent()->getPlan();
330  VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
331  VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
332 
333  // Replace the temporary unreachable terminator with a new conditional
334  // branch, hooking it up to backward destination (the header) now and to the
335  // forward destination (the exit/middle block) later when it is created.
336  // Note that CreateCondBr expects a valid BB as first argument, so we need
337  // to set it to nullptr later.
338  BranchInst *CondBr = Builder.CreateCondBr(Cond, Builder.GetInsertBlock(),
339  State.CFG.VPBB2IRBB[Header]);
340  CondBr->setSuccessor(0, nullptr);
341  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
342  break;
343  }
344  default:
345  llvm_unreachable("Unsupported opcode for instruction");
346  }
347 }
348 
350  assert(!State.Instance && "VPInstruction executing an Instance");
352  State.Builder.setFastMathFlags(FMF);
353  for (unsigned Part = 0; Part < State.UF; ++Part)
354  generateInstruction(State, Part);
355 }
356 
357 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
358 void VPInstruction::dump() const {
359  VPSlotTracker SlotTracker(getParent()->getPlan());
360  print(dbgs(), "", SlotTracker);
361 }
362 
364  VPSlotTracker &SlotTracker) const {
365  O << Indent << "EMIT ";
366 
367  if (hasResult()) {
369  O << " = ";
370  }
371 
372  switch (getOpcode()) {
373  case VPInstruction::Not:
374  O << "not";
375  break;
377  O << "icmp ule";
378  break;
380  O << "combined load";
381  break;
383  O << "combined store";
384  break;
386  O << "active lane mask";
387  break;
389  O << "first-order splice";
390  break;
392  O << "VF * UF + ";
393  break;
395  O << "VF * UF +(nuw) ";
396  break;
398  O << "branch-on-cond";
399  break;
401  O << "VF * Part + ";
402  break;
404  O << "VF * Part +(nuw) ";
405  break;
407  O << "branch-on-count ";
408  break;
409  default:
411  }
412 
413  O << FMF;
414 
415  for (const VPValue *Operand : operands()) {
416  O << " ";
417  Operand->printAsOperand(O, SlotTracker);
418  }
419 
420  if (DL) {
421  O << ", !dbg ";
422  DL.print(O);
423  }
424 }
425 #endif
426 
428  // Make sure the VPInstruction is a floating-point operation.
429  assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
430  Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
431  Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
432  Opcode == Instruction::FCmp) &&
433  "this op can't take fast-math flags");
434  FMF = FMFNew;
435 }
436 
438  auto &CI = *cast<CallInst>(getUnderlyingInstr());
439  assert(!isa<DbgInfoIntrinsic>(CI) &&
440  "DbgInfoIntrinsic should have been dropped during VPlan construction");
441  State.setDebugLocFromInst(&CI);
442 
444  for (Value *ArgOperand : CI.args())
445  Tys.push_back(
446  ToVectorTy(ArgOperand->getType(), State.VF.getKnownMinValue()));
447 
448  for (unsigned Part = 0; Part < State.UF; ++Part) {
449  SmallVector<Type *, 2> TysForDecl = {CI.getType()};
451  for (const auto &I : enumerate(operands())) {
452  // Some intrinsics have a scalar argument - don't replace it with a
453  // vector.
454  Value *Arg;
455  if (VectorIntrinsicID == Intrinsic::not_intrinsic ||
456  !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
457  Arg = State.get(I.value(), Part);
458  else
459  Arg = State.get(I.value(), VPIteration(0, 0));
460  if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
461  TysForDecl.push_back(Arg->getType());
462  Args.push_back(Arg);
463  }
464 
465  Function *VectorF;
466  if (VectorIntrinsicID != Intrinsic::not_intrinsic) {
467  // Use vector version of the intrinsic.
468  if (State.VF.isVector())
469  TysForDecl[0] =
470  VectorType::get(CI.getType()->getScalarType(), State.VF);
471  Module *M = State.Builder.GetInsertBlock()->getModule();
472  VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
473  assert(VectorF && "Can't retrieve vector intrinsic.");
474  } else {
475  // Use vector version of the function call.
476  const VFShape Shape = VFShape::get(CI, State.VF, false /*HasGlobalPred*/);
477 #ifndef NDEBUG
478  assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
479  "Can't create vector function.");
480 #endif
481  VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
482  }
484  CI.getOperandBundlesAsDefs(OpBundles);
485  CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
486 
487  if (isa<FPMathOperator>(V))
488  V->copyFastMathFlags(&CI);
489 
490  State.set(this, V, Part);
491  State.addMetadata(V, &CI);
492  }
493 }
494 
495 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
497  VPSlotTracker &SlotTracker) const {
498  O << Indent << "WIDEN-CALL ";
499 
500  auto *CI = cast<CallInst>(getUnderlyingInstr());
501  if (CI->getType()->isVoidTy())
502  O << "void ";
503  else {
505  O << " = ";
506  }
507 
508  O << "call @" << CI->getCalledFunction()->getName() << "(";
510  O << ")";
511 
512  if (VectorIntrinsicID)
513  O << " (using vector intrinsic)";
514  else
515  O << " (using library function)";
516 }
517 
519  VPSlotTracker &SlotTracker) const {
520  O << Indent << "WIDEN-SELECT ";
522  O << " = select ";
524  O << ", ";
526  O << ", ";
528  O << (InvariantCond ? " (condition is loop invariant)" : "");
529 }
530 #endif
531 
533  auto &I = *cast<SelectInst>(getUnderlyingInstr());
534  State.setDebugLocFromInst(&I);
535 
536  // The condition can be loop invariant but still defined inside the
537  // loop. This means that we can't just use the original 'cond' value.
538  // We have to take the 'vectorized' value and pick the first lane.
539  // Instcombine will make this a no-op.
540  auto *InvarCond =
541  InvariantCond ? State.get(getOperand(0), VPIteration(0, 0)) : nullptr;
542 
543  for (unsigned Part = 0; Part < State.UF; ++Part) {
544  Value *Cond = InvarCond ? InvarCond : State.get(getOperand(0), Part);
545  Value *Op0 = State.get(getOperand(1), Part);
546  Value *Op1 = State.get(getOperand(2), Part);
547  Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
548  State.set(this, Sel, Part);
549  State.addMetadata(Sel, &I);
550  }
551 }
552 
554  auto &I = *cast<Instruction>(getUnderlyingValue());
555  auto &Builder = State.Builder;
556  switch (I.getOpcode()) {
557  case Instruction::Call:
558  case Instruction::Br:
559  case Instruction::PHI:
560  case Instruction::GetElementPtr:
561  case Instruction::Select:
562  llvm_unreachable("This instruction is handled by a different recipe.");
563  case Instruction::UDiv:
564  case Instruction::SDiv:
565  case Instruction::SRem:
566  case Instruction::URem:
567  case Instruction::Add:
568  case Instruction::FAdd:
569  case Instruction::Sub:
570  case Instruction::FSub:
571  case Instruction::FNeg:
572  case Instruction::Mul:
573  case Instruction::FMul:
574  case Instruction::FDiv:
575  case Instruction::FRem:
576  case Instruction::Shl:
577  case Instruction::LShr:
578  case Instruction::AShr:
579  case Instruction::And:
580  case Instruction::Or:
581  case Instruction::Xor: {
582  // Just widen unops and binops.
583  State.setDebugLocFromInst(&I);
584 
585  for (unsigned Part = 0; Part < State.UF; ++Part) {
587  for (VPValue *VPOp : operands())
588  Ops.push_back(State.get(VPOp, Part));
589 
590  Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
591 
592  if (auto *VecOp = dyn_cast<Instruction>(V)) {
593  VecOp->copyIRFlags(&I);
594 
595  // If the instruction is vectorized and was in a basic block that needed
596  // predication, we can't propagate poison-generating flags (nuw/nsw,
597  // exact, etc.). The control flow has been linearized and the
598  // instruction is no longer guarded by the predicate, which could make
599  // the flag properties to no longer hold.
600  if (State.MayGeneratePoisonRecipes.contains(this))
601  VecOp->dropPoisonGeneratingFlags();
602  }
603 
604  // Use this vector value for all users of the original instruction.
605  State.set(this, V, Part);
606  State.addMetadata(V, &I);
607  }
608 
609  break;
610  }
611  case Instruction::Freeze: {
612  State.setDebugLocFromInst(&I);
613 
614  for (unsigned Part = 0; Part < State.UF; ++Part) {
615  Value *Op = State.get(getOperand(0), Part);
616 
617  Value *Freeze = Builder.CreateFreeze(Op);
618  State.set(this, Freeze, Part);
619  }
620  break;
621  }
622  case Instruction::ICmp:
623  case Instruction::FCmp: {
624  // Widen compares. Generate vector compares.
625  bool FCmp = (I.getOpcode() == Instruction::FCmp);
626  auto *Cmp = cast<CmpInst>(&I);
627  State.setDebugLocFromInst(Cmp);
628  for (unsigned Part = 0; Part < State.UF; ++Part) {
629  Value *A = State.get(getOperand(0), Part);
630  Value *B = State.get(getOperand(1), Part);
631  Value *C = nullptr;
632  if (FCmp) {
633  // Propagate fast math flags.
635  Builder.setFastMathFlags(Cmp->getFastMathFlags());
636  C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
637  } else {
638  C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
639  }
640  State.set(this, C, Part);
641  State.addMetadata(C, &I);
642  }
643 
644  break;
645  }
646 
647  case Instruction::ZExt:
648  case Instruction::SExt:
649  case Instruction::FPToUI:
650  case Instruction::FPToSI:
651  case Instruction::FPExt:
652  case Instruction::PtrToInt:
653  case Instruction::IntToPtr:
654  case Instruction::SIToFP:
655  case Instruction::UIToFP:
656  case Instruction::Trunc:
657  case Instruction::FPTrunc:
658  case Instruction::BitCast: {
659  auto *CI = cast<CastInst>(&I);
660  State.setDebugLocFromInst(CI);
661 
662  /// Vectorize casts.
663  Type *DestTy = (State.VF.isScalar())
664  ? CI->getType()
665  : VectorType::get(CI->getType(), State.VF);
666 
667  for (unsigned Part = 0; Part < State.UF; ++Part) {
668  Value *A = State.get(getOperand(0), Part);
669  Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
670  State.set(this, Cast, Part);
671  State.addMetadata(Cast, &I);
672  }
673  break;
674  }
675  default:
676  // This instruction is not vectorized by simple widening.
677  LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
678  llvm_unreachable("Unhandled instruction!");
679  } // end of switch.
680 }
681 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
683  VPSlotTracker &SlotTracker) const {
684  O << Indent << "WIDEN ";
686  const Instruction *UI = getUnderlyingInstr();
687  O << " = " << UI->getOpcodeName() << " ";
688  if (auto *Cmp = dyn_cast<CmpInst>(UI))
689  O << CmpInst::getPredicateName(Cmp->getPredicate()) << " ";
691 }
692 
694  VPSlotTracker &SlotTracker) const {
695  O << Indent << "WIDEN-INDUCTION";
696  if (getTruncInst()) {
697  O << "\\l\"";
698  O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
699  O << " +\n" << Indent << "\" ";
701  } else
702  O << " " << VPlanIngredient(IV);
703 
704  O << ", ";
706 }
707 #endif
708 
710  auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
711  auto *StepC = dyn_cast<SCEVConstant>(getInductionDescriptor().getStep());
712  return StartC && StartC->isZero() && StepC && StepC->isOne();
713 }
714 
716  return cast<VPCanonicalIVPHIRecipe>(getOperand(0));
717 }
718 
720  auto *CanIV = getCanonicalIV();
721  // The start value of the steps-recipe must match the start value of the
722  // canonical induction and it must step by 1.
723  if (CanIV->getStartValue() != getStartValue())
724  return false;
725  auto *StepVPV = getStepValue();
726  if (StepVPV->hasDefiningRecipe())
727  return false;
728  auto *StepC = dyn_cast_or_null<ConstantInt>(StepVPV->getLiveInIRValue());
729  return StepC && StepC->isOne();
730 }
731 
732 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
734  VPSlotTracker &SlotTracker) const {
735  O << Indent;
737  O << Indent << "= SCALAR-STEPS ";
739 }
740 #endif
741 
743  auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
744  // Construct a vector GEP by widening the operands of the scalar GEP as
745  // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
746  // results in a vector of pointers when at least one operand of the GEP
747  // is vector-typed. Thus, to keep the representation compact, we only use
748  // vector-typed operands for loop-varying values.
749 
750  if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
751  // If we are vectorizing, but the GEP has only loop-invariant operands,
752  // the GEP we build (by only using vector-typed operands for
753  // loop-varying values) would be a scalar pointer. Thus, to ensure we
754  // produce a vector of pointers, we need to either arbitrarily pick an
755  // operand to broadcast, or broadcast a clone of the original GEP.
756  // Here, we broadcast a clone of the original.
757  //
758  // TODO: If at some point we decide to scalarize instructions having
759  // loop-invariant operands, this special case will no longer be
760  // required. We would add the scalarization decision to
761  // collectLoopScalars() and teach getVectorValue() to broadcast
762  // the lane-zero scalar value.
763  auto *Clone = State.Builder.Insert(GEP->clone());
764  for (unsigned Part = 0; Part < State.UF; ++Part) {
765  Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone);
766  State.set(this, EntryPart, Part);
767  State.addMetadata(EntryPart, GEP);
768  }
769  } else {
770  // If the GEP has at least one loop-varying operand, we are sure to
771  // produce a vector of pointers. But if we are only unrolling, we want
772  // to produce a scalar GEP for each unroll part. Thus, the GEP we
773  // produce with the code below will be scalar (if VF == 1) or vector
774  // (otherwise). Note that for the unroll-only case, we still maintain
775  // values in the vector mapping with initVector, as we do for other
776  // instructions.
777  for (unsigned Part = 0; Part < State.UF; ++Part) {
778  // The pointer operand of the new GEP. If it's loop-invariant, we
779  // won't broadcast it.
780  auto *Ptr = IsPtrLoopInvariant
781  ? State.get(getOperand(0), VPIteration(0, 0))
782  : State.get(getOperand(0), Part);
783 
784  // Collect all the indices for the new GEP. If any index is
785  // loop-invariant, we won't broadcast it.
786  SmallVector<Value *, 4> Indices;
787  for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
788  VPValue *Operand = getOperand(I);
789  if (IsIndexLoopInvariant[I - 1])
790  Indices.push_back(State.get(Operand, VPIteration(0, 0)));
791  else
792  Indices.push_back(State.get(Operand, Part));
793  }
794 
795  // If the GEP instruction is vectorized and was in a basic block that
796  // needed predication, we can't propagate the poison-generating 'inbounds'
797  // flag. The control flow has been linearized and the GEP is no longer
798  // guarded by the predicate, which could make the 'inbounds' properties to
799  // no longer hold.
800  bool IsInBounds =
801  GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0;
802 
803  // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
804  // but it should be a vector, otherwise.
805  auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
806  Indices, "", IsInBounds);
807  assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
808  "NewGEP is not a pointer vector");
809  State.set(this, NewGEP, Part);
810  State.addMetadata(NewGEP, GEP);
811  }
812  }
813 }
814 
815 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
817  VPSlotTracker &SlotTracker) const {
818  O << Indent << "WIDEN-GEP ";
819  O << (IsPtrLoopInvariant ? "Inv" : "Var");
820  size_t IndicesNumber = IsIndexLoopInvariant.size();
821  for (size_t I = 0; I < IndicesNumber; ++I)
822  O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]";
823 
824  O << " ";
826  O << " = getelementptr ";
828 }
829 #endif
830 
832  State.setDebugLocFromInst(Phi);
833  // We know that all PHIs in non-header blocks are converted into
834  // selects, so we don't have to worry about the insertion order and we
835  // can just use the builder.
836  // At this point we generate the predication tree. There may be
837  // duplications since this is a simple recursive scan, but future
838  // optimizations will clean it up.
839 
840  unsigned NumIncoming = getNumIncomingValues();
841 
842  // Generate a sequence of selects of the form:
843  // SELECT(Mask3, In3,
844  // SELECT(Mask2, In2,
845  // SELECT(Mask1, In1,
846  // In0)))
847  // Note that Mask0 is never used: lanes for which no path reaches this phi and
848  // are essentially undef are taken from In0.
849  VectorParts Entry(State.UF);
850  for (unsigned In = 0; In < NumIncoming; ++In) {
851  for (unsigned Part = 0; Part < State.UF; ++Part) {
852  // We might have single edge PHIs (blocks) - use an identity
853  // 'select' for the first PHI operand.
854  Value *In0 = State.get(getIncomingValue(In), Part);
855  if (In == 0)
856  Entry[Part] = In0; // Initialize with the first incoming value.
857  else {
858  // Select between the current value and the previous incoming edge
859  // based on the incoming mask.
860  Value *Cond = State.get(getMask(In), Part);
861  Entry[Part] =
862  State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
863  }
864  }
865  }
866  for (unsigned Part = 0; Part < State.UF; ++Part)
867  State.set(this, Entry[Part], Part);
868 }
869 
870 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
872  VPSlotTracker &SlotTracker) const {
873  O << Indent << "BLEND ";
874  Phi->printAsOperand(O, false);
875  O << " =";
876  if (getNumIncomingValues() == 1) {
877  // Not a User of any mask: not really blending, this is a
878  // single-predecessor phi.
879  O << " ";
881  } else {
882  for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
883  O << " ";
885  O << "/";
887  }
888  }
889 }
890 
892  VPSlotTracker &SlotTracker) const {
893  O << Indent << "REDUCE ";
895  O << " = ";
897  O << " +";
898  if (isa<FPMathOperator>(getUnderlyingInstr()))
900  O << " reduce." << Instruction::getOpcodeName(RdxDesc->getOpcode()) << " (";
902  if (getCondOp()) {
903  O << ", ";
905  }
906  O << ")";
907  if (RdxDesc->IntermediateStore)
908  O << " (with final reduction value stored in invariant address sank "
909  "outside of loop)";
910 }
911 
913  VPSlotTracker &SlotTracker) const {
914  O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
915 
916  if (!getUnderlyingInstr()->getType()->isVoidTy()) {
918  O << " = ";
919  }
920  if (auto *CB = dyn_cast<CallBase>(getUnderlyingInstr())) {
921  O << "call @" << CB->getCalledFunction()->getName() << "(";
923  O, [&O, &SlotTracker](VPValue *Op) {
924  Op->printAsOperand(O, SlotTracker);
925  });
926  O << ")";
927  } else {
930  }
931 
932  if (AlsoPack)
933  O << " (S->V)";
934 }
935 #endif
936 
938  assert(State.Instance && "Branch on Mask works only on single instance.");
939 
940  unsigned Part = State.Instance->Part;
941  unsigned Lane = State.Instance->Lane.getKnownLane();
942 
943  Value *ConditionBit = nullptr;
944  VPValue *BlockInMask = getMask();
945  if (BlockInMask) {
946  ConditionBit = State.get(BlockInMask, Part);
947  if (ConditionBit->getType()->isVectorTy())
948  ConditionBit = State.Builder.CreateExtractElement(
949  ConditionBit, State.Builder.getInt32(Lane));
950  } else // Block in mask is all-one.
951  ConditionBit = State.Builder.getTrue();
952 
953  // Replace the temporary unreachable terminator with a new conditional branch,
954  // whose two destinations will be set later when they are created.
955  auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
956  assert(isa<UnreachableInst>(CurrentTerminator) &&
957  "Expected to replace unreachable terminator with conditional branch.");
958  auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
959  CondBr->setSuccessor(0, nullptr);
960  ReplaceInstWithInst(CurrentTerminator, CondBr);
961 }
962 
964  assert(State.Instance && "Predicated instruction PHI works per instance.");
965  Instruction *ScalarPredInst =
966  cast<Instruction>(State.get(getOperand(0), *State.Instance));
967  BasicBlock *PredicatedBB = ScalarPredInst->getParent();
968  BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
969  assert(PredicatingBB && "Predicated block has no single predecessor.");
970  assert(isa<VPReplicateRecipe>(getOperand(0)) &&
971  "operand must be VPReplicateRecipe");
972 
973  // By current pack/unpack logic we need to generate only a single phi node: if
974  // a vector value for the predicated instruction exists at this point it means
975  // the instruction has vector users only, and a phi for the vector value is
976  // needed. In this case the recipe of the predicated instruction is marked to
977  // also do that packing, thereby "hoisting" the insert-element sequence.
978  // Otherwise, a phi node for the scalar value is needed.
979  unsigned Part = State.Instance->Part;
980  if (State.hasVectorValue(getOperand(0), Part)) {
981  Value *VectorValue = State.get(getOperand(0), Part);
982  InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
983  PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
984  VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
985  VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
986  if (State.hasVectorValue(this, Part))
987  State.reset(this, VPhi, Part);
988  else
989  State.set(this, VPhi, Part);
990  // NOTE: Currently we need to update the value of the operand, so the next
991  // predicated iteration inserts its generated value in the correct vector.
992  State.reset(getOperand(0), VPhi, Part);
993  } else {
994  Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
995  PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
996  Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
997  PredicatingBB);
998  Phi->addIncoming(ScalarPredInst, PredicatedBB);
999  if (State.hasScalarValue(this, *State.Instance))
1000  State.reset(this, Phi, *State.Instance);
1001  else
1002  State.set(this, Phi, *State.Instance);
1003  // NOTE: Currently we need to update the value of the operand, so the next
1004  // predicated iteration inserts its generated value in the correct vector.
1005  State.reset(getOperand(0), Phi, *State.Instance);
1006  }
1007 }
1008 
1009 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1011  VPSlotTracker &SlotTracker) const {
1012  O << Indent << "PHI-PREDICATED-INSTRUCTION ";
1014  O << " = ";
1016 }
1017 
1019  VPSlotTracker &SlotTracker) const {
1020  O << Indent << "WIDEN ";
1021 
1022  if (!isStore()) {
1024  O << " = ";
1025  }
1026  O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
1027 
1029 }
1030 #endif
1031 
1033  Value *Start = getStartValue()->getLiveInIRValue();
1034  PHINode *EntryPart = PHINode::Create(
1035  Start->getType(), 2, "index", &*State.CFG.PrevBB->getFirstInsertionPt());
1036 
1037  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1038  EntryPart->addIncoming(Start, VectorPH);
1039  EntryPart->setDebugLoc(DL);
1040  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1041  State.set(this, EntryPart, Part);
1042 }
1043 
1044 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1046  VPSlotTracker &SlotTracker) const {
1047  O << Indent << "EMIT ";
1049  O << " = CANONICAL-INDUCTION";
1050 }
1051 #endif
1052 
1054  return IsScalarAfterVectorization &&
1055  (!VF.isScalable() || vputils::onlyFirstLaneUsed(this));
1056 }
1057 
1058 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1060  VPSlotTracker &SlotTracker) const {
1061  O << Indent << "EMIT ";
1063  O << " = WIDEN-POINTER-INDUCTION ";
1065  O << ", " << *IndDesc.getStep();
1066 }
1067 #endif
1068 
1070  assert(!State.Instance && "cannot be used in per-lane");
1071  const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout();
1072  SCEVExpander Exp(SE, DL, "induction");
1073 
1074  Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
1075  &*State.Builder.GetInsertPoint());
1076 
1077  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1078  State.set(this, Res, Part);
1079 }
1080 
1081 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1083  VPSlotTracker &SlotTracker) const {
1084  O << Indent << "EMIT ";
1086  O << " = EXPAND SCEV " << *Expr;
1087 }
1088 #endif
1089 
1091  Value *CanonicalIV = State.get(getOperand(0), 0);
1092  Type *STy = CanonicalIV->getType();
1094  ElementCount VF = State.VF;
1095  Value *VStart = VF.isScalar()
1096  ? CanonicalIV
1097  : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");
1098  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1099  Value *VStep = createStepForVF(Builder, STy, VF, Part);
1100  if (VF.isVector()) {
1101  VStep = Builder.CreateVectorSplat(VF, VStep);
1102  VStep =
1103  Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
1104  }
1105  Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
1106  State.set(this, CanonicalVectorIV, Part);
1107  }
1108 }
1109 
1110 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1112  VPSlotTracker &SlotTracker) const {
1113  O << Indent << "EMIT ";
1115  O << " = WIDEN-CANONICAL-INDUCTION ";
1117 }
1118 #endif
1119 
1121  auto &Builder = State.Builder;
1122  // Create a vector from the initial value.
1123  auto *VectorInit = getStartValue()->getLiveInIRValue();
1124 
1125  Type *VecTy = State.VF.isScalar()
1126  ? VectorInit->getType()
1127  : VectorType::get(VectorInit->getType(), State.VF);
1128 
1129  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1130  if (State.VF.isVector()) {
1131  auto *IdxTy = Builder.getInt32Ty();
1132  auto *One = ConstantInt::get(IdxTy, 1);
1134  Builder.SetInsertPoint(VectorPH->getTerminator());
1135  auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
1136  auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
1137  VectorInit = Builder.CreateInsertElement(
1138  PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
1139  }
1140 
1141  // Create a phi node for the new recurrence.
1142  PHINode *EntryPart = PHINode::Create(
1143  VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt());
1144  EntryPart->addIncoming(VectorInit, VectorPH);
1145  State.set(this, EntryPart, 0);
1146 }
1147 
1148 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1150  VPSlotTracker &SlotTracker) const {
1151  O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";
1153  O << " = phi ";
1155 }
1156 #endif
1157 
1159  PHINode *PN = cast<PHINode>(getUnderlyingValue());
1160  auto &Builder = State.Builder;
1161 
1162  // In order to support recurrences we need to be able to vectorize Phi nodes.
1163  // Phi nodes have cycles, so we need to vectorize them in two stages. This is
1164  // stage #1: We create a new vector PHI node with no incoming edges. We'll use
1165  // this value when we vectorize all of the instructions that use the PHI.
1166  bool ScalarPHI = State.VF.isScalar() || IsInLoop;
1167  Type *VecTy =
1168  ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
1169 
1170  BasicBlock *HeaderBB = State.CFG.PrevBB;
1171  assert(State.CurrentVectorLoop->getHeader() == HeaderBB &&
1172  "recipe must be in the vector loop header");
1173  unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
1174  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1175  Value *EntryPart =
1176  PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
1177  State.set(this, EntryPart, Part);
1178  }
1179 
1180  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1181 
1182  // Reductions do not have to start at zero. They can start with
1183  // any loop invariant values.
1184  VPValue *StartVPV = getStartValue();
1185  Value *StartV = StartVPV->getLiveInIRValue();
1186 
1187  Value *Iden = nullptr;
1188  RecurKind RK = RdxDesc.getRecurrenceKind();
1191  // MinMax reduction have the start value as their identify.
1192  if (ScalarPHI) {
1193  Iden = StartV;
1194  } else {
1196  Builder.SetInsertPoint(VectorPH->getTerminator());
1197  StartV = Iden =
1198  Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
1199  }
1200  } else {
1201  Iden = RdxDesc.getRecurrenceIdentity(RK, VecTy->getScalarType(),
1202  RdxDesc.getFastMathFlags());
1203 
1204  if (!ScalarPHI) {
1205  Iden = Builder.CreateVectorSplat(State.VF, Iden);
1207  Builder.SetInsertPoint(VectorPH->getTerminator());
1208  Constant *Zero = Builder.getInt32(0);
1209  StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
1210  }
1211  }
1212 
1213  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1214  Value *EntryPart = State.get(this, Part);
1215  // Make sure to add the reduction start value only to the
1216  // first unroll part.
1217  Value *StartVal = (Part == 0) ? StartV : Iden;
1218  cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1219  }
1220 }
1221 
1222 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1224  VPSlotTracker &SlotTracker) const {
1225  O << Indent << "WIDEN-REDUCTION-PHI ";
1226 
1228  O << " = phi ";
1230 }
1231 #endif
1232 
1235  "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1236 
1237  // Currently we enter here in the VPlan-native path for non-induction
1238  // PHIs where all control flow is uniform. We simply widen these PHIs.
1239  // Create a vector phi with no operands - the vector phi operands will be
1240  // set at the end of vector code generation.
1241  VPBasicBlock *Parent = getParent();
1242  VPRegionBlock *LoopRegion = Parent->getEnclosingLoopRegion();
1243  unsigned StartIdx = 0;
1244  // For phis in header blocks of loop regions, use the index of the value
1245  // coming from the preheader.
1246  if (LoopRegion->getEntryBasicBlock() == Parent) {
1247  for (unsigned I = 0; I < getNumOperands(); ++I) {
1248  if (getIncomingBlock(I) ==
1249  LoopRegion->getSinglePredecessor()->getExitingBasicBlock())
1250  StartIdx = I;
1251  }
1252  }
1253  Value *Op0 = State.get(getOperand(StartIdx), 0);
1254  Type *VecTy = Op0->getType();
1255  Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
1256  State.set(this, VecPhi, 0);
1257 }
1258 
1259 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1261  VPSlotTracker &SlotTracker) const {
1262  O << Indent << "WIDEN-PHI ";
1263 
1264  auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
1265  // Unless all incoming values are modeled in VPlan print the original PHI
1266  // directly.
1267  // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
1268  // values as VPValues.
1269  if (getNumOperands() != OriginalPhi->getNumOperands()) {
1270  O << VPlanIngredient(OriginalPhi);
1271  return;
1272  }
1273 
1275  O << " = phi ";
1277 }
1278 #endif
1279 
1280 // TODO: It would be good to use the existing VPWidenPHIRecipe instead and
1281 // remove VPActiveLaneMaskPHIRecipe.
1283  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1284  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1285  Value *StartMask = State.get(getOperand(0), Part);
1286  PHINode *EntryPart =
1287  State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
1288  EntryPart->addIncoming(StartMask, VectorPH);
1289  EntryPart->setDebugLoc(DL);
1290  State.set(this, EntryPart, Part);
1291  }
1292 }
1293 
1294 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1296  VPSlotTracker &SlotTracker) const {
1297  O << Indent << "ACTIVE-LANE-MASK-PHI ";
1298 
1300  O << " = phi ";
1302 }
1303 #endif
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2060
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:851
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlanRecipes.cpp:178
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:502
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:891
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:142
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1097
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1388
llvm::VPInstruction::FirstOrderRecurrenceSplice
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:781
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1391
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:720
llvm::Type::getInt1Ty
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:236
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1830
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlanRecipes.cpp:173
llvm::VPLane::getFirstLane
static VPLane getFirstLane()
Definition: VPlan.h:136
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlanRecipes.cpp:1090
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: VPlanRecipes.cpp:963
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:871
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1481
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:359
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
ScalarEvolutionExpander.h
llvm::ElementCount
Definition: TypeSize.h:404
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2013
llvm::Function
Definition: Function.h:60
llvm::SCEVExpander
This class uses information about analyze scalars to rewrite expressions in canonical form.
Definition: ScalarEvolutionExpander.h:50
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:693
llvm::VPValue::getLiveInIRValue
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:196
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:328
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:786
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::VPCanonicalIVPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1045
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:402
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9863
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:2238
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: VPlanRecipes.cpp:937
llvm::IRBuilderBase::setFastMathFlags
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:296
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2525
llvm::VPWidenPointerInductionRecipe::onlyScalarsGenerated
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
Definition: VPlanRecipes.cpp:1053
llvm::VPInstruction::setFastMathFlags
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
Definition: VPlanRecipes.cpp:427
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:264
llvm::isVectorIntrinsicWithScalarOpAtArg
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:101
llvm::RecurrenceDescriptor::getOpcode
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1130
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:233
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:831
llvm::interleaveComma
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:2072
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::VPInstruction::BranchOnCond
@ BranchOnCond
Definition: VPlan.h:796
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1102
llvm::VPInstruction::CanonicalIVIncrement
@ CanonicalIVIncrement
Definition: VPlan.h:789
llvm::isVectorIntrinsicWithOverloadTypeAtArg
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, unsigned OpdIdx)
Identifies if the vector form of the intrinsic has a operand that has an overloaded type.
Definition: VectorUtils.cpp:119
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:291
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1018
llvm::iplist_impl::remove
pointer remove(iterator &IT)
Definition: ilist.h:252
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:285
STLExtras.h
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:295
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2176
llvm::CmpInst::getPredicateName
static StringRef getPredicateName(Predicate P)
Definition: Instructions.cpp:4095
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
llvm::VPInstruction::CanonicalIVIncrementForPartNUW
@ CanonicalIVIncrementForPartNUW
Definition: VPlan.h:794
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:253
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:425
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:391
llvm::VPScalarIVStepsRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:719
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1233
llvm::VPScalarIVStepsRecipe::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition: VPlanRecipes.cpp:715
llvm::BranchInst::setSuccessor
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
Definition: Instructions.h:3231
llvm::iplist_impl::erase
iterator erase(iterator where)
Definition: ilist.h:268
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPLiveOut::fixPhi
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
Definition: VPlanRecipes.cpp:134
llvm::VPWidenIntOrFpInductionRecipe::getInductionDescriptor
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1112
llvm::RecurKind
RecurKind
These are the kinds of recurrences that we support.
Definition: IVDescriptors.h:35
llvm::VPRecipeBase::mayWriteToMemory
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
Definition: VPlanRecipes.cpp:42
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ElementCount::isScalar
bool isScalar() const
Counting predicates.
Definition: TypeSize.h:414
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1010
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
Instruction.h
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:205
CommandLine.h
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:912
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:164
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:788
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1272
llvm::VFDatabase
The Vector Function Database.
Definition: VectorUtils.h:208
llvm::createStepForVF
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
Definition: LoopVectorize.cpp:955
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1758
llvm::RecurrenceDescriptor::isSelectCmpRecurrenceKind
static bool isSelectCmpRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Definition: IVDescriptors.h:239
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
Twine.h
llvm::ToVectorTy
Type * ToVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
Definition: VectorUtils.h:299
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:784
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:787
getOpcode
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:190
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlanRecipes.cpp:183
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:559
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:246
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1935
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::SmallBitVector::all
bool all() const
Returns true if all bits are set.
Definition: SmallBitVector.h:216
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1111
llvm::Instruction::getOpcodeName
const char * getOpcodeName() const
Definition: Instruction.h:166
llvm::RecurrenceDescriptor::getRecurrenceIdentity
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1075
llvm::Instruction
Definition: Instruction.h:42
EnableVPlanNativePath
cl::opt< bool > EnableVPlanNativePath
llvm::RecurrenceDescriptor::getRecurrenceKind
RecurKind getRecurrenceKind() const
Definition: IVDescriptors.h:198
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::VPScalarIVStepsRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:1999
llvm::VPValue
Definition: VPlanValue.h:44
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::VPWidenCallRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
Definition: VPlanRecipes.cpp:437
llvm::BasicBlock::getModule
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:147
VPlan.h
llvm::VFShape
Contains the information about the kind of vectorization available.
Definition: VectorUtils.h:82
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
IVDescriptors.h
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:269
Type.h
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2137
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1758
llvm::VPTransformState::setDebugLocFromInst
void setDebugLocFromInst(const Value *V)
Set the debug location in the builder using the debug location in V.
Definition: VPlan.cpp:258
llvm::IRBuilderBase::FastMathFlagGuard
Definition: IRBuilder.h:380
llvm::tgtok::In
@ In
Definition: TGLexer.h:51
llvm::VPActiveLaneMaskPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
Definition: VPlanRecipes.cpp:1282
BasicBlock.h
llvm::cl::opt< bool >
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlanRecipes.cpp:167
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:264
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlanRecipes.cpp:104
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::IRBuilderBase::GetInsertPoint
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:174
llvm::VPExpandSCEVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
Definition: VPlanRecipes.cpp:1069
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1536
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: VPlanRecipes.cpp:742
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlanRecipes.cpp:349
llvm::VPReductionPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1223
llvm::IRBuilderBase::getInt32
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:469
llvm::PHINode::addIncoming
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Definition: Instructions.h:2848
llvm::VFDatabase::getVectorizedFunction
Function * getVectorizedFunction(const VFShape &Shape) const
Definition: VectorUtils.h:268
llvm::VPLane::getLastLaneForVF
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:138
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3189
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:214
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlanRecipes.cpp:363
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:188
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1164
llvm::IRBuilderBase::CreateSelect
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1123
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1394
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:816
llvm::Instruction::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:351
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::VPInstruction::CanonicalIVIncrementNUW
@ CanonicalIVIncrementNUW
Definition: VPlan.h:790
llvm::Instruction::getFastMathFlags
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition: Instruction.cpp:319
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2316
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:875
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition: Instruction.h:169
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::IRBuilderBase::getTrue
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:449
llvm::VPUser::printOperands
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1018
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:296
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:682
llvm::ElementCount::isVector
bool isVector() const
One or more elements.
Definition: TypeSize.h:416
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4758
llvm::VPWidenIntOrFpInductionRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:709
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:496
llvm::IRBuilderBase::CreatePHI
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2254
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:272
llvm::VPScalarIVStepsRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:733
llvm::VPTransformState::addMetadata
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:246
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:963
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1534
Mul
BinaryOperator * Mul
Definition: X86PartialReduction.cpp:70
llvm::RecurrenceDescriptor::IntermediateStore
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
Definition: IVDescriptors.h:276
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:795
llvm::RecurrenceDescriptor::getFastMathFlags
FastMathFlags getFastMathFlags() const
Definition: IVDescriptors.h:202
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::InductionDescriptor::getStep
const SCEV * getStep() const
Definition: IVDescriptors.h:322
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:682
llvm::IRBuilderBase::InsertPointGuard
Definition: IRBuilder.h:358
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::VPCanonicalIVPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Definition: VPlanRecipes.cpp:1032
llvm::VPScalarIVStepsRecipe::getStepValue
VPValue * getStepValue() const
Definition: VPlan.h:2000
llvm::Instruction::copyFastMathFlags
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
Definition: Instruction.cpp:274
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: VPlanRecipes.cpp:532
llvm::VPInstruction::CanonicalIVIncrementForPart
@ CanonicalIVIncrementForPart
Definition: VPlan.h:793
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:213
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:147
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1101
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:414
llvm::vputils::isUniformAfterVectorization
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:3057
llvm::VPFirstOrderRecurrencePHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1149
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:164
llvm::VPReductionPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1158
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:452
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:461
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:377
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::VPTransformState::MayGeneratePoisonRecipes
SmallPtrSet< VPRecipeBase *, 16 > MayGeneratePoisonRecipes
Holds recipes that may generate a poison value that is used after vectorization, even when their oper...
Definition: VPlan.h:374
llvm::PHINode::Create
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Definition: Instructions.h:2740
llvm::IRBuilderBase::Insert
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:144
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:663
llvm::IRBuilderBase::GetInsertBlock
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:173
Casting.h
llvm::LoopBase::getHeader
BlockT * getHeader() const
Definition: LoopInfo.h:105
llvm::DebugLoc::print
void print(raw_ostream &OS) const
prints source location /path/to/file.exe:line:col @[inlined at]
Definition: DebugLoc.cpp:103
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:664
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:185
llvm::VPWidenPointerInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1059
llvm::iplist_impl::insertAfter
iterator insertAfter(iterator where, pointer New)
Definition: ilist.h:238
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
llvm::SmallBitVector::size
size_type size() const
Returns the number of bits in this bitvector.
Definition: SmallBitVector.h:195
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:788
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1968
Instructions.h
SmallVector.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:270
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:219
llvm::VPRecipeBase::mayReadFromMemory
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
Definition: VPlanRecipes.cpp:73
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::VFShape::get
static VFShape get(const CallInst &CI, ElementCount EC, bool HasGlobalPred)
Definition: VectorUtils.h:108
llvm::VPActiveLaneMaskPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1295
llvm::PHINode
Definition: Instructions.h:2698
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::VPBlockBase::getExitingBasicBlock
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:168
llvm::VPFirstOrderRecurrencePHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition: VPlanRecipes.cpp:1120
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:343
llvm::IRBuilderBase::CreateVectorSplat
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1246
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
llvm::SCEV::getType
Type * getType() const
Return the LLVM type of this SCEV expression.
Definition: ScalarEvolution.cpp:402
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1473
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1646
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1260
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::RecurrenceDescriptor::isMinMaxRecurrenceKind
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Definition: IVDescriptors.h:233
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3133
raw_ostream.h
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1532
llvm::VPExpandSCEVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1082
BasicBlockUtils.h
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: VPlanRecipes.cpp:553
llvm::VPDef::getVPDefID
unsigned getVPDefID() const
Definition: VPlanValue.h:434
llvm::VPInstruction::dump
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlanRecipes.cpp:358
Value.h
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2492
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlanRecipes.cpp:143
Debug.h
llvm::VectorType::get
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:668
llvm::VPReductionPHIRecipe::isOrdered
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1357
llvm::VPlanIngredient
Definition: VPlan.h:2778
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:518
llvm::VPValue::printAsOperand
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:1003
llvm::IRBuilderBase::CreateCall
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2269
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:1093
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:335
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlanRecipes.cpp:159
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:243
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:785
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1732