LLVM  15.0.0git
VPlanRecipes.cpp
Go to the documentation of this file.
1 //===- VPlanRecipes.cpp - Implementations for VPlan recipes ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file contains implementations for different VPlan recipes.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlan.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/Twine.h"
19 #include "llvm/IR/BasicBlock.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Instruction.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/Type.h"
24 #include "llvm/IR/Value.h"
25 #include "llvm/Support/Casting.h"
27 #include "llvm/Support/Debug.h"
31 #include <cassert>
32 
33 using namespace llvm;
34 
36 
38 
39 #define LV_NAME "loop-vectorize"
40 #define DEBUG_TYPE LV_NAME
41 
43  switch (getVPDefID()) {
44  case VPWidenMemoryInstructionSC: {
45  return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
46  }
47  case VPReplicateSC:
48  case VPWidenCallSC:
49  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
50  ->mayWriteToMemory();
51  case VPBranchOnMaskSC:
52  return false;
53  case VPWidenIntOrFpInductionSC:
54  case VPWidenCanonicalIVSC:
55  case VPWidenPHISC:
56  case VPBlendSC:
57  case VPWidenSC:
58  case VPWidenGEPSC:
59  case VPReductionSC:
60  case VPWidenSelectSC: {
61  const Instruction *I =
62  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
63  (void)I;
64  assert((!I || !I->mayWriteToMemory()) &&
65  "underlying instruction may write to memory");
66  return false;
67  }
68  default:
69  return true;
70  }
71 }
72 
74  switch (getVPDefID()) {
75  case VPWidenMemoryInstructionSC: {
76  return !cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
77  }
78  case VPReplicateSC:
79  case VPWidenCallSC:
80  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
81  ->mayReadFromMemory();
82  case VPBranchOnMaskSC:
83  return false;
84  case VPWidenIntOrFpInductionSC:
85  case VPWidenCanonicalIVSC:
86  case VPWidenPHISC:
87  case VPBlendSC:
88  case VPWidenSC:
89  case VPWidenGEPSC:
90  case VPReductionSC:
91  case VPWidenSelectSC: {
92  const Instruction *I =
93  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
94  (void)I;
95  assert((!I || !I->mayReadFromMemory()) &&
96  "underlying instruction may read from memory");
97  return false;
98  }
99  default:
100  return true;
101  }
102 }
103 
105  switch (getVPDefID()) {
106  case VPWidenIntOrFpInductionSC:
107  case VPWidenPointerInductionSC:
108  case VPWidenCanonicalIVSC:
109  case VPWidenPHISC:
110  case VPBlendSC:
111  case VPWidenSC:
112  case VPWidenGEPSC:
113  case VPReductionSC:
114  case VPWidenSelectSC:
115  case VPScalarIVStepsSC: {
116  const Instruction *I =
117  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
118  (void)I;
119  assert((!I || !I->mayHaveSideEffects()) &&
120  "underlying instruction has side-effects");
121  return false;
122  }
123  case VPReplicateSC: {
124  auto *R = cast<VPReplicateRecipe>(this);
125  return R->getUnderlyingInstr()->mayHaveSideEffects();
126  }
127  default:
128  return true;
129  }
130 }
131 
133  auto Lane = VPLane::getLastLaneForVF(State.VF);
134  VPValue *ExitValue = getOperand(0);
135  if (Plan.isUniformAfterVectorization(ExitValue))
136  Lane = VPLane::getFirstLane();
137  Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)),
138  State.Builder.GetInsertBlock());
139 }
140 
142  assert(!Parent && "Recipe already in some VPBasicBlock");
143  assert(InsertPos->getParent() &&
144  "Insertion position not in any VPBasicBlock");
145  Parent = InsertPos->getParent();
146  Parent->getRecipeList().insert(InsertPos->getIterator(), this);
147 }
148 
151  assert(!Parent && "Recipe already in some VPBasicBlock");
152  assert(I == BB.end() || I->getParent() == &BB);
153  Parent = &BB;
154  BB.getRecipeList().insert(I, this);
155 }
156 
158  assert(!Parent && "Recipe already in some VPBasicBlock");
159  assert(InsertPos->getParent() &&
160  "Insertion position not in any VPBasicBlock");
161  Parent = InsertPos->getParent();
162  Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
163 }
164 
166  assert(getParent() && "Recipe not in any VPBasicBlock");
168  Parent = nullptr;
169 }
170 
172  assert(getParent() && "Recipe not in any VPBasicBlock");
173  return getParent()->getRecipeList().erase(getIterator());
174 }
175 
178  insertAfter(InsertPos);
179 }
180 
184  insertBefore(BB, I);
185 }
186 
187 void VPInstruction::generateInstruction(VPTransformState &State,
188  unsigned Part) {
189  IRBuilderBase &Builder = State.Builder;
190  Builder.SetCurrentDebugLocation(DL);
191 
193  Value *A = State.get(getOperand(0), Part);
194  Value *B = State.get(getOperand(1), Part);
195  Value *V =
196  Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
197  State.set(this, V, Part);
198  return;
199  }
200 
201  switch (getOpcode()) {
202  case VPInstruction::Not: {
203  Value *A = State.get(getOperand(0), Part);
204  Value *V = Builder.CreateNot(A, Name);
205  State.set(this, V, Part);
206  break;
207  }
208  case VPInstruction::ICmpULE: {
209  Value *IV = State.get(getOperand(0), Part);
210  Value *TC = State.get(getOperand(1), Part);
211  Value *V = Builder.CreateICmpULE(IV, TC, Name);
212  State.set(this, V, Part);
213  break;
214  }
215  case Instruction::Select: {
216  Value *Cond = State.get(getOperand(0), Part);
217  Value *Op1 = State.get(getOperand(1), Part);
218  Value *Op2 = State.get(getOperand(2), Part);
219  Value *V = Builder.CreateSelect(Cond, Op1, Op2, Name);
220  State.set(this, V, Part);
221  break;
222  }
224  // Get first lane of vector induction variable.
225  Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
226  // Get the original loop tripcount.
227  Value *ScalarTC = State.get(getOperand(1), Part);
228 
229  auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
230  auto *PredTy = VectorType::get(Int1Ty, State.VF);
231  Instruction *Call = Builder.CreateIntrinsic(
232  Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()},
233  {VIVElem0, ScalarTC}, nullptr, Name);
234  State.set(this, Call, Part);
235  break;
236  }
238  // Generate code to combine the previous and current values in vector v3.
239  //
240  // vector.ph:
241  // v_init = vector(..., ..., ..., a[-1])
242  // br vector.body
243  //
244  // vector.body
245  // i = phi [0, vector.ph], [i+4, vector.body]
246  // v1 = phi [v_init, vector.ph], [v2, vector.body]
247  // v2 = a[i, i+1, i+2, i+3];
248  // v3 = vector(v1(3), v2(0, 1, 2))
249 
250  // For the first part, use the recurrence phi (v1), otherwise v2.
251  auto *V1 = State.get(getOperand(0), 0);
252  Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1);
253  if (!PartMinus1->getType()->isVectorTy()) {
254  State.set(this, PartMinus1, Part);
255  } else {
256  Value *V2 = State.get(getOperand(1), Part);
257  State.set(this, Builder.CreateVectorSplice(PartMinus1, V2, -1, Name),
258  Part);
259  }
260  break;
261  }
264  Value *Next = nullptr;
265  if (Part == 0) {
267  auto *Phi = State.get(getOperand(0), 0);
268  // The loop step is equal to the vectorization factor (num of SIMD
269  // elements) times the unroll factor (num of SIMD instructions).
270  Value *Step =
271  createStepForVF(Builder, Phi->getType(), State.VF, State.UF);
272  Next = Builder.CreateAdd(Phi, Step, Name, IsNUW, false);
273  } else {
274  Next = State.get(this, 0);
275  }
276 
277  State.set(this, Next, Part);
278  break;
279  }
280 
284  auto *IV = State.get(getOperand(0), VPIteration(0, 0));
285  if (Part == 0) {
286  State.set(this, IV, Part);
287  break;
288  }
289 
290  // The canonical IV is incremented by the vectorization factor (num of SIMD
291  // elements) times the unroll part.
292  Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
293  Value *Next = Builder.CreateAdd(IV, Step, Name, IsNUW, false);
294  State.set(this, Next, Part);
295  break;
296  }
298  if (Part != 0)
299  break;
300 
301  Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
302  VPRegionBlock *ParentRegion = getParent()->getParent();
303  VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
304 
305  // Replace the temporary unreachable terminator with a new conditional
306  // branch, hooking it up to backward destination for exiting blocks now and
307  // to forward destination(s) later when they are created.
308  BranchInst *CondBr =
309  Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr);
310 
311  if (getParent()->isExiting())
312  CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
313 
314  CondBr->setSuccessor(0, nullptr);
315  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
316  break;
317  }
319  if (Part != 0)
320  break;
321  // First create the compare.
322  Value *IV = State.get(getOperand(0), Part);
323  Value *TC = State.get(getOperand(1), Part);
324  Value *Cond = Builder.CreateICmpEQ(IV, TC);
325 
326  // Now create the branch.
327  auto *Plan = getParent()->getPlan();
328  VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
329  VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
330 
331  // Replace the temporary unreachable terminator with a new conditional
332  // branch, hooking it up to backward destination (the header) now and to the
333  // forward destination (the exit/middle block) later when it is created.
334  // Note that CreateCondBr expects a valid BB as first argument, so we need
335  // to set it to nullptr later.
336  BranchInst *CondBr = Builder.CreateCondBr(Cond, Builder.GetInsertBlock(),
337  State.CFG.VPBB2IRBB[Header]);
338  CondBr->setSuccessor(0, nullptr);
339  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
340  break;
341  }
342  default:
343  llvm_unreachable("Unsupported opcode for instruction");
344  }
345 }
346 
348  assert(!State.Instance && "VPInstruction executing an Instance");
350  State.Builder.setFastMathFlags(FMF);
351  for (unsigned Part = 0; Part < State.UF; ++Part)
352  generateInstruction(State, Part);
353 }
354 
355 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
356 void VPInstruction::dump() const {
357  VPSlotTracker SlotTracker(getParent()->getPlan());
358  print(dbgs(), "", SlotTracker);
359 }
360 
362  VPSlotTracker &SlotTracker) const {
363  O << Indent << "EMIT ";
364 
365  if (hasResult()) {
367  O << " = ";
368  }
369 
370  switch (getOpcode()) {
371  case VPInstruction::Not:
372  O << "not";
373  break;
375  O << "icmp ule";
376  break;
378  O << "combined load";
379  break;
381  O << "combined store";
382  break;
384  O << "active lane mask";
385  break;
387  O << "first-order splice";
388  break;
390  O << "VF * UF + ";
391  break;
393  O << "VF * UF +(nuw) ";
394  break;
396  O << "branch-on-cond";
397  break;
399  O << "VF * Part + ";
400  break;
402  O << "VF * Part +(nuw) ";
403  break;
405  O << "branch-on-count ";
406  break;
407  default:
409  }
410 
411  O << FMF;
412 
413  for (const VPValue *Operand : operands()) {
414  O << " ";
415  Operand->printAsOperand(O, SlotTracker);
416  }
417 
418  if (DL) {
419  O << ", !dbg ";
420  DL.print(O);
421  }
422 }
423 #endif
424 
426  // Make sure the VPInstruction is a floating-point operation.
427  assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
428  Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
429  Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
430  Opcode == Instruction::FCmp) &&
431  "this op can't take fast-math flags");
432  FMF = FMFNew;
433 }
434 
435 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
437  VPSlotTracker &SlotTracker) const {
438  O << Indent << "WIDEN-CALL ";
439 
440  auto *CI = cast<CallInst>(getUnderlyingInstr());
441  if (CI->getType()->isVoidTy())
442  O << "void ";
443  else {
445  O << " = ";
446  }
447 
448  O << "call @" << CI->getCalledFunction()->getName() << "(";
450  O << ")";
451 }
452 
454  VPSlotTracker &SlotTracker) const {
455  O << Indent << "WIDEN-SELECT ";
457  O << " = select ";
459  O << ", ";
461  O << ", ";
463  O << (InvariantCond ? " (condition is loop invariant)" : "");
464 }
465 #endif
466 
468  auto &I = *cast<SelectInst>(getUnderlyingInstr());
469  State.setDebugLocFromInst(&I);
470 
471  // The condition can be loop invariant but still defined inside the
472  // loop. This means that we can't just use the original 'cond' value.
473  // We have to take the 'vectorized' value and pick the first lane.
474  // Instcombine will make this a no-op.
475  auto *InvarCond =
476  InvariantCond ? State.get(getOperand(0), VPIteration(0, 0)) : nullptr;
477 
478  for (unsigned Part = 0; Part < State.UF; ++Part) {
479  Value *Cond = InvarCond ? InvarCond : State.get(getOperand(0), Part);
480  Value *Op0 = State.get(getOperand(1), Part);
481  Value *Op1 = State.get(getOperand(2), Part);
482  Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
483  State.set(this, Sel, Part);
484  State.addMetadata(Sel, &I);
485  }
486 }
487 
489  auto &I = *cast<Instruction>(getUnderlyingValue());
490  auto &Builder = State.Builder;
491  switch (I.getOpcode()) {
492  case Instruction::Call:
493  case Instruction::Br:
494  case Instruction::PHI:
495  case Instruction::GetElementPtr:
496  case Instruction::Select:
497  llvm_unreachable("This instruction is handled by a different recipe.");
498  case Instruction::UDiv:
499  case Instruction::SDiv:
500  case Instruction::SRem:
501  case Instruction::URem:
502  case Instruction::Add:
503  case Instruction::FAdd:
504  case Instruction::Sub:
505  case Instruction::FSub:
506  case Instruction::FNeg:
507  case Instruction::Mul:
508  case Instruction::FMul:
509  case Instruction::FDiv:
510  case Instruction::FRem:
511  case Instruction::Shl:
512  case Instruction::LShr:
513  case Instruction::AShr:
514  case Instruction::And:
515  case Instruction::Or:
516  case Instruction::Xor: {
517  // Just widen unops and binops.
518  State.setDebugLocFromInst(&I);
519 
520  for (unsigned Part = 0; Part < State.UF; ++Part) {
522  for (VPValue *VPOp : operands())
523  Ops.push_back(State.get(VPOp, Part));
524 
525  Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
526 
527  if (auto *VecOp = dyn_cast<Instruction>(V)) {
528  VecOp->copyIRFlags(&I);
529 
530  // If the instruction is vectorized and was in a basic block that needed
531  // predication, we can't propagate poison-generating flags (nuw/nsw,
532  // exact, etc.). The control flow has been linearized and the
533  // instruction is no longer guarded by the predicate, which could make
534  // the flag properties to no longer hold.
535  if (State.MayGeneratePoisonRecipes.contains(this))
536  VecOp->dropPoisonGeneratingFlags();
537  }
538 
539  // Use this vector value for all users of the original instruction.
540  State.set(this, V, Part);
541  State.addMetadata(V, &I);
542  }
543 
544  break;
545  }
546  case Instruction::Freeze: {
547  State.setDebugLocFromInst(&I);
548 
549  for (unsigned Part = 0; Part < State.UF; ++Part) {
550  Value *Op = State.get(getOperand(0), Part);
551 
552  Value *Freeze = Builder.CreateFreeze(Op);
553  State.set(this, Freeze, Part);
554  }
555  break;
556  }
557  case Instruction::ICmp:
558  case Instruction::FCmp: {
559  // Widen compares. Generate vector compares.
560  bool FCmp = (I.getOpcode() == Instruction::FCmp);
561  auto *Cmp = cast<CmpInst>(&I);
562  State.setDebugLocFromInst(Cmp);
563  for (unsigned Part = 0; Part < State.UF; ++Part) {
564  Value *A = State.get(getOperand(0), Part);
565  Value *B = State.get(getOperand(1), Part);
566  Value *C = nullptr;
567  if (FCmp) {
568  // Propagate fast math flags.
570  Builder.setFastMathFlags(Cmp->getFastMathFlags());
571  C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
572  } else {
573  C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
574  }
575  State.set(this, C, Part);
576  State.addMetadata(C, &I);
577  }
578 
579  break;
580  }
581 
582  case Instruction::ZExt:
583  case Instruction::SExt:
584  case Instruction::FPToUI:
585  case Instruction::FPToSI:
586  case Instruction::FPExt:
587  case Instruction::PtrToInt:
588  case Instruction::IntToPtr:
589  case Instruction::SIToFP:
590  case Instruction::UIToFP:
591  case Instruction::Trunc:
592  case Instruction::FPTrunc:
593  case Instruction::BitCast: {
594  auto *CI = cast<CastInst>(&I);
595  State.setDebugLocFromInst(CI);
596 
597  /// Vectorize casts.
598  Type *DestTy = (State.VF.isScalar())
599  ? CI->getType()
600  : VectorType::get(CI->getType(), State.VF);
601 
602  for (unsigned Part = 0; Part < State.UF; ++Part) {
603  Value *A = State.get(getOperand(0), Part);
604  Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
605  State.set(this, Cast, Part);
606  State.addMetadata(Cast, &I);
607  }
608  break;
609  }
610  default:
611  // This instruction is not vectorized by simple widening.
612  LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
613  llvm_unreachable("Unhandled instruction!");
614  } // end of switch.
615 }
616 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
618  VPSlotTracker &SlotTracker) const {
619  O << Indent << "WIDEN ";
621  O << " = " << getUnderlyingInstr()->getOpcodeName() << " ";
623 }
624 
626  VPSlotTracker &SlotTracker) const {
627  O << Indent << "WIDEN-INDUCTION";
628  if (getTruncInst()) {
629  O << "\\l\"";
630  O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
631  O << " +\n" << Indent << "\" ";
633  } else
634  O << " " << VPlanIngredient(IV);
635 
636  O << ", ";
638 }
639 #endif
640 
642  auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
643  auto *StepC = dyn_cast<SCEVConstant>(getInductionDescriptor().getStep());
644  return StartC && StartC->isZero() && StepC && StepC->isOne();
645 }
646 
648  return cast<VPCanonicalIVPHIRecipe>(getOperand(0));
649 }
650 
652  auto *CanIV = getCanonicalIV();
653  // The start value of the steps-recipe must match the start value of the
654  // canonical induction and it must step by 1.
655  if (CanIV->getStartValue() != getStartValue())
656  return false;
657  auto *StepVPV = getStepValue();
658  if (StepVPV->getDef())
659  return false;
660  auto *StepC = dyn_cast_or_null<ConstantInt>(StepVPV->getLiveInIRValue());
661  return StepC && StepC->isOne();
662 }
663 
664 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
666  VPSlotTracker &SlotTracker) const {
667  O << Indent;
669  O << Indent << "= SCALAR-STEPS ";
671 }
672 #endif
673 
675  auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
676  // Construct a vector GEP by widening the operands of the scalar GEP as
677  // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
678  // results in a vector of pointers when at least one operand of the GEP
679  // is vector-typed. Thus, to keep the representation compact, we only use
680  // vector-typed operands for loop-varying values.
681 
682  if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
683  // If we are vectorizing, but the GEP has only loop-invariant operands,
684  // the GEP we build (by only using vector-typed operands for
685  // loop-varying values) would be a scalar pointer. Thus, to ensure we
686  // produce a vector of pointers, we need to either arbitrarily pick an
687  // operand to broadcast, or broadcast a clone of the original GEP.
688  // Here, we broadcast a clone of the original.
689  //
690  // TODO: If at some point we decide to scalarize instructions having
691  // loop-invariant operands, this special case will no longer be
692  // required. We would add the scalarization decision to
693  // collectLoopScalars() and teach getVectorValue() to broadcast
694  // the lane-zero scalar value.
695  auto *Clone = State.Builder.Insert(GEP->clone());
696  for (unsigned Part = 0; Part < State.UF; ++Part) {
697  Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone);
698  State.set(this, EntryPart, Part);
699  State.addMetadata(EntryPart, GEP);
700  }
701  } else {
702  // If the GEP has at least one loop-varying operand, we are sure to
703  // produce a vector of pointers. But if we are only unrolling, we want
704  // to produce a scalar GEP for each unroll part. Thus, the GEP we
705  // produce with the code below will be scalar (if VF == 1) or vector
706  // (otherwise). Note that for the unroll-only case, we still maintain
707  // values in the vector mapping with initVector, as we do for other
708  // instructions.
709  for (unsigned Part = 0; Part < State.UF; ++Part) {
710  // The pointer operand of the new GEP. If it's loop-invariant, we
711  // won't broadcast it.
712  auto *Ptr = IsPtrLoopInvariant
713  ? State.get(getOperand(0), VPIteration(0, 0))
714  : State.get(getOperand(0), Part);
715 
716  // Collect all the indices for the new GEP. If any index is
717  // loop-invariant, we won't broadcast it.
718  SmallVector<Value *, 4> Indices;
719  for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
720  VPValue *Operand = getOperand(I);
721  if (IsIndexLoopInvariant[I - 1])
722  Indices.push_back(State.get(Operand, VPIteration(0, 0)));
723  else
724  Indices.push_back(State.get(Operand, Part));
725  }
726 
727  // If the GEP instruction is vectorized and was in a basic block that
728  // needed predication, we can't propagate the poison-generating 'inbounds'
729  // flag. The control flow has been linearized and the GEP is no longer
730  // guarded by the predicate, which could make the 'inbounds' properties to
731  // no longer hold.
732  bool IsInBounds =
733  GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0;
734 
735  // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
736  // but it should be a vector, otherwise.
737  auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
738  Indices, "", IsInBounds);
739  assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
740  "NewGEP is not a pointer vector");
741  State.set(this, NewGEP, Part);
742  State.addMetadata(NewGEP, GEP);
743  }
744  }
745 }
746 
747 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
749  VPSlotTracker &SlotTracker) const {
750  O << Indent << "WIDEN-GEP ";
751  O << (IsPtrLoopInvariant ? "Inv" : "Var");
752  size_t IndicesNumber = IsIndexLoopInvariant.size();
753  for (size_t I = 0; I < IndicesNumber; ++I)
754  O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]";
755 
756  O << " ";
758  O << " = getelementptr ";
760 }
761 #endif
762 
764  State.setDebugLocFromInst(Phi);
765  // We know that all PHIs in non-header blocks are converted into
766  // selects, so we don't have to worry about the insertion order and we
767  // can just use the builder.
768  // At this point we generate the predication tree. There may be
769  // duplications since this is a simple recursive scan, but future
770  // optimizations will clean it up.
771 
772  unsigned NumIncoming = getNumIncomingValues();
773 
774  // Generate a sequence of selects of the form:
775  // SELECT(Mask3, In3,
776  // SELECT(Mask2, In2,
777  // SELECT(Mask1, In1,
778  // In0)))
779  // Note that Mask0 is never used: lanes for which no path reaches this phi and
780  // are essentially undef are taken from In0.
781  VectorParts Entry(State.UF);
782  for (unsigned In = 0; In < NumIncoming; ++In) {
783  for (unsigned Part = 0; Part < State.UF; ++Part) {
784  // We might have single edge PHIs (blocks) - use an identity
785  // 'select' for the first PHI operand.
786  Value *In0 = State.get(getIncomingValue(In), Part);
787  if (In == 0)
788  Entry[Part] = In0; // Initialize with the first incoming value.
789  else {
790  // Select between the current value and the previous incoming edge
791  // based on the incoming mask.
792  Value *Cond = State.get(getMask(In), Part);
793  Entry[Part] =
794  State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
795  }
796  }
797  }
798  for (unsigned Part = 0; Part < State.UF; ++Part)
799  State.set(this, Entry[Part], Part);
800 }
801 
802 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
804  VPSlotTracker &SlotTracker) const {
805  O << Indent << "BLEND ";
806  Phi->printAsOperand(O, false);
807  O << " =";
808  if (getNumIncomingValues() == 1) {
809  // Not a User of any mask: not really blending, this is a
810  // single-predecessor phi.
811  O << " ";
813  } else {
814  for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
815  O << " ";
817  O << "/";
819  }
820  }
821 }
822 
824  VPSlotTracker &SlotTracker) const {
825  O << Indent << "REDUCE ";
827  O << " = ";
829  O << " +";
830  if (isa<FPMathOperator>(getUnderlyingInstr()))
832  O << " reduce." << Instruction::getOpcodeName(RdxDesc->getOpcode()) << " (";
834  if (getCondOp()) {
835  O << ", ";
837  }
838  O << ")";
839  if (RdxDesc->IntermediateStore)
840  O << " (with final reduction value stored in invariant address sank "
841  "outside of loop)";
842 }
843 
845  VPSlotTracker &SlotTracker) const {
846  O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
847 
848  if (!getUnderlyingInstr()->getType()->isVoidTy()) {
850  O << " = ";
851  }
852  if (auto *CB = dyn_cast<CallBase>(getUnderlyingInstr())) {
853  O << "call @" << CB->getCalledFunction()->getName() << "(";
855  O, [&O, &SlotTracker](VPValue *Op) {
856  Op->printAsOperand(O, SlotTracker);
857  });
858  O << ")";
859  } else {
862  }
863 
864  if (AlsoPack)
865  O << " (S->V)";
866 }
867 #endif
868 
870  assert(State.Instance && "Branch on Mask works only on single instance.");
871 
872  unsigned Part = State.Instance->Part;
873  unsigned Lane = State.Instance->Lane.getKnownLane();
874 
875  Value *ConditionBit = nullptr;
876  VPValue *BlockInMask = getMask();
877  if (BlockInMask) {
878  ConditionBit = State.get(BlockInMask, Part);
879  if (ConditionBit->getType()->isVectorTy())
880  ConditionBit = State.Builder.CreateExtractElement(
881  ConditionBit, State.Builder.getInt32(Lane));
882  } else // Block in mask is all-one.
883  ConditionBit = State.Builder.getTrue();
884 
885  // Replace the temporary unreachable terminator with a new conditional branch,
886  // whose two destinations will be set later when they are created.
887  auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
888  assert(isa<UnreachableInst>(CurrentTerminator) &&
889  "Expected to replace unreachable terminator with conditional branch.");
890  auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
891  CondBr->setSuccessor(0, nullptr);
892  ReplaceInstWithInst(CurrentTerminator, CondBr);
893 }
894 
896  assert(State.Instance && "Predicated instruction PHI works per instance.");
897  Instruction *ScalarPredInst =
898  cast<Instruction>(State.get(getOperand(0), *State.Instance));
899  BasicBlock *PredicatedBB = ScalarPredInst->getParent();
900  BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
901  assert(PredicatingBB && "Predicated block has no single predecessor.");
902  assert(isa<VPReplicateRecipe>(getOperand(0)) &&
903  "operand must be VPReplicateRecipe");
904 
905  // By current pack/unpack logic we need to generate only a single phi node: if
906  // a vector value for the predicated instruction exists at this point it means
907  // the instruction has vector users only, and a phi for the vector value is
908  // needed. In this case the recipe of the predicated instruction is marked to
909  // also do that packing, thereby "hoisting" the insert-element sequence.
910  // Otherwise, a phi node for the scalar value is needed.
911  unsigned Part = State.Instance->Part;
912  if (State.hasVectorValue(getOperand(0), Part)) {
913  Value *VectorValue = State.get(getOperand(0), Part);
914  InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
915  PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
916  VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
917  VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
918  if (State.hasVectorValue(this, Part))
919  State.reset(this, VPhi, Part);
920  else
921  State.set(this, VPhi, Part);
922  // NOTE: Currently we need to update the value of the operand, so the next
923  // predicated iteration inserts its generated value in the correct vector.
924  State.reset(getOperand(0), VPhi, Part);
925  } else {
926  Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
927  PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
928  Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
929  PredicatingBB);
930  Phi->addIncoming(ScalarPredInst, PredicatedBB);
931  if (State.hasScalarValue(this, *State.Instance))
932  State.reset(this, Phi, *State.Instance);
933  else
934  State.set(this, Phi, *State.Instance);
935  // NOTE: Currently we need to update the value of the operand, so the next
936  // predicated iteration inserts its generated value in the correct vector.
937  State.reset(getOperand(0), Phi, *State.Instance);
938  }
939 }
940 
941 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
943  VPSlotTracker &SlotTracker) const {
944  O << Indent << "PHI-PREDICATED-INSTRUCTION ";
946  O << " = ";
948 }
949 
951  VPSlotTracker &SlotTracker) const {
952  O << Indent << "WIDEN ";
953 
954  if (!isStore()) {
956  O << " = ";
957  }
958  O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
959 
961 }
962 #endif
963 
965  Value *Start = getStartValue()->getLiveInIRValue();
966  PHINode *EntryPart = PHINode::Create(
967  Start->getType(), 2, "index", &*State.CFG.PrevBB->getFirstInsertionPt());
968 
969  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
970  EntryPart->addIncoming(Start, VectorPH);
971  EntryPart->setDebugLoc(DL);
972  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
973  State.set(this, EntryPart, Part);
974 }
975 
976 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
978  VPSlotTracker &SlotTracker) const {
979  O << Indent << "EMIT ";
981  O << " = CANONICAL-INDUCTION";
982 }
983 #endif
984 
986  bool IsUniform = vputils::onlyFirstLaneUsed(this);
987  return all_of(users(),
988  [&](const VPUser *U) { return U->usesScalars(this); }) &&
989  (IsUniform || !VF.isScalable());
990 }
991 
992 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
994  VPSlotTracker &SlotTracker) const {
995  O << Indent << "EMIT ";
997  O << " = WIDEN-POINTER-INDUCTION ";
999  O << ", " << *IndDesc.getStep();
1000 }
1001 #endif
1002 
1004  assert(!State.Instance && "cannot be used in per-lane");
1005  const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout();
1006  SCEVExpander Exp(SE, DL, "induction");
1007 
1008  Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
1009  &*State.Builder.GetInsertPoint());
1010 
1011  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1012  State.set(this, Res, Part);
1013 }
1014 
1015 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1017  VPSlotTracker &SlotTracker) const {
1018  O << Indent << "EMIT ";
1020  O << " = EXPAND SCEV " << *Expr;
1021 }
1022 #endif
1023 
1025  Value *CanonicalIV = State.get(getOperand(0), 0);
1026  Type *STy = CanonicalIV->getType();
1028  ElementCount VF = State.VF;
1029  Value *VStart = VF.isScalar()
1030  ? CanonicalIV
1031  : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");
1032  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1033  Value *VStep = createStepForVF(Builder, STy, VF, Part);
1034  if (VF.isVector()) {
1035  VStep = Builder.CreateVectorSplat(VF, VStep);
1036  VStep =
1037  Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
1038  }
1039  Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
1040  State.set(this, CanonicalVectorIV, Part);
1041  }
1042 }
1043 
1044 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1046  VPSlotTracker &SlotTracker) const {
1047  O << Indent << "EMIT ";
1049  O << " = WIDEN-CANONICAL-INDUCTION ";
1051 }
1052 #endif
1053 
1055  auto &Builder = State.Builder;
1056  // Create a vector from the initial value.
1057  auto *VectorInit = getStartValue()->getLiveInIRValue();
1058 
1059  Type *VecTy = State.VF.isScalar()
1060  ? VectorInit->getType()
1061  : VectorType::get(VectorInit->getType(), State.VF);
1062 
1063  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1064  if (State.VF.isVector()) {
1065  auto *IdxTy = Builder.getInt32Ty();
1066  auto *One = ConstantInt::get(IdxTy, 1);
1068  Builder.SetInsertPoint(VectorPH->getTerminator());
1069  auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
1070  auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
1071  VectorInit = Builder.CreateInsertElement(
1072  PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
1073  }
1074 
1075  // Create a phi node for the new recurrence.
1076  PHINode *EntryPart = PHINode::Create(
1077  VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt());
1078  EntryPart->addIncoming(VectorInit, VectorPH);
1079  State.set(this, EntryPart, 0);
1080 }
1081 
1082 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1084  VPSlotTracker &SlotTracker) const {
1085  O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";
1087  O << " = phi ";
1089 }
1090 #endif
1091 
1093  PHINode *PN = cast<PHINode>(getUnderlyingValue());
1094  auto &Builder = State.Builder;
1095 
1096  // In order to support recurrences we need to be able to vectorize Phi nodes.
1097  // Phi nodes have cycles, so we need to vectorize them in two stages. This is
1098  // stage #1: We create a new vector PHI node with no incoming edges. We'll use
1099  // this value when we vectorize all of the instructions that use the PHI.
1100  bool ScalarPHI = State.VF.isScalar() || IsInLoop;
1101  Type *VecTy =
1102  ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
1103 
1104  BasicBlock *HeaderBB = State.CFG.PrevBB;
1105  assert(State.CurrentVectorLoop->getHeader() == HeaderBB &&
1106  "recipe must be in the vector loop header");
1107  unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
1108  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1109  Value *EntryPart =
1110  PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
1111  State.set(this, EntryPart, Part);
1112  }
1113 
1114  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1115 
1116  // Reductions do not have to start at zero. They can start with
1117  // any loop invariant values.
1118  VPValue *StartVPV = getStartValue();
1119  Value *StartV = StartVPV->getLiveInIRValue();
1120 
1121  Value *Iden = nullptr;
1122  RecurKind RK = RdxDesc.getRecurrenceKind();
1125  // MinMax reduction have the start value as their identify.
1126  if (ScalarPHI) {
1127  Iden = StartV;
1128  } else {
1130  Builder.SetInsertPoint(VectorPH->getTerminator());
1131  StartV = Iden =
1132  Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
1133  }
1134  } else {
1135  Iden = RdxDesc.getRecurrenceIdentity(RK, VecTy->getScalarType(),
1136  RdxDesc.getFastMathFlags());
1137 
1138  if (!ScalarPHI) {
1139  Iden = Builder.CreateVectorSplat(State.VF, Iden);
1141  Builder.SetInsertPoint(VectorPH->getTerminator());
1142  Constant *Zero = Builder.getInt32(0);
1143  StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
1144  }
1145  }
1146 
1147  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1148  Value *EntryPart = State.get(this, Part);
1149  // Make sure to add the reduction start value only to the
1150  // first unroll part.
1151  Value *StartVal = (Part == 0) ? StartV : Iden;
1152  cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1153  }
1154 }
1155 
1156 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1158  VPSlotTracker &SlotTracker) const {
1159  O << Indent << "WIDEN-REDUCTION-PHI ";
1160 
1162  O << " = phi ";
1164 }
1165 #endif
1166 
1169  "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1170 
1171  // Currently we enter here in the VPlan-native path for non-induction
1172  // PHIs where all control flow is uniform. We simply widen these PHIs.
1173  // Create a vector phi with no operands - the vector phi operands will be
1174  // set at the end of vector code generation.
1175  VPBasicBlock *Parent = getParent();
1176  VPRegionBlock *LoopRegion = Parent->getEnclosingLoopRegion();
1177  unsigned StartIdx = 0;
1178  // For phis in header blocks of loop regions, use the index of the value
1179  // coming from the preheader.
1180  if (LoopRegion->getEntryBasicBlock() == Parent) {
1181  for (unsigned I = 0; I < getNumOperands(); ++I) {
1182  if (getIncomingBlock(I) ==
1183  LoopRegion->getSinglePredecessor()->getExitingBasicBlock())
1184  StartIdx = I;
1185  }
1186  }
1187  Value *Op0 = State.get(getOperand(StartIdx), 0);
1188  Type *VecTy = Op0->getType();
1189  Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
1190  State.set(this, VecPhi, 0);
1191 }
1192 
1193 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1195  VPSlotTracker &SlotTracker) const {
1196  O << Indent << "WIDEN-PHI ";
1197 
1198  auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
1199  // Unless all incoming values are modeled in VPlan print the original PHI
1200  // directly.
1201  // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
1202  // values as VPValues.
1203  if (getNumOperands() != OriginalPhi->getNumOperands()) {
1204  O << VPlanIngredient(OriginalPhi);
1205  return;
1206  }
1207 
1209  O << " = phi ";
1211 }
1212 #endif
1213 
1214 // TODO: It would be good to use the existing VPWidenPHIRecipe instead and
1215 // remove VPActiveLaneMaskPHIRecipe.
1217  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1218  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1219  Value *StartMask = State.get(getOperand(0), Part);
1220  PHINode *EntryPart =
1221  State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
1222  EntryPart->addIncoming(StartMask, VectorPH);
1223  EntryPart->setDebugLoc(DL);
1224  State.set(this, EntryPart, Part);
1225  }
1226 }
1227 
1228 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1230  VPSlotTracker &SlotTracker) const {
1231  O << Indent << "ACTIVE-LANE-MASK-PHI ";
1232 
1234  O << " = phi ";
1236 }
1237 #endif
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2062
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:847
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlanRecipes.cpp:176
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:498
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:823
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:134
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1088
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1388
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1391
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:716
llvm::Type::getInt1Ty
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:236
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1832
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlanRecipes.cpp:171
llvm::VPLane::getFirstLane
static VPLane getFirstLane()
Definition: VPlan.h:132
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlanRecipes.cpp:1024
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: VPlanRecipes.cpp:895
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:803
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:355
ScalarEvolutionExpander.h
llvm::ElementCount
Definition: TypeSize.h:404
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2015
llvm::SCEVExpander
This class uses information about analyze scalars to rewrite expressions in canonical form.
Definition: ScalarEvolutionExpander.h:50
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:625
llvm::VPValue::getLiveInIRValue
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:188
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:309
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::VPCanonicalIVPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:977
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:783
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:388
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9755
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: VPlanRecipes.cpp:869
llvm::IRBuilderBase::setFastMathFlags
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:288
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2485
llvm::VPWidenPointerInductionRecipe::onlyScalarsGenerated
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
Definition: VPlanRecipes.cpp:985
llvm::VPInstruction::setFastMathFlags
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
Definition: VPlanRecipes.cpp:425
llvm::VPInstruction::CanonicalIVIncrement
@ CanonicalIVIncrement
Definition: VPlan.h:785
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:260
llvm::RecurrenceDescriptor::getOpcode
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1102
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:225
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:763
llvm::interleaveComma
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:1907
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1093
llvm::VPInstruction::CanonicalIVIncrementForPart
@ CanonicalIVIncrementForPart
Definition: VPlan.h:789
llvm::VPInstruction::BranchOnCond
@ BranchOnCond
Definition: VPlan.h:792
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:278
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:950
llvm::iplist_impl::remove
pointer remove(iterator &IT)
Definition: ilist.h:252
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:261
STLExtras.h
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:282
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2178
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:249
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:782
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:417
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:391
llvm::VPScalarIVStepsRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:651
llvm::VPValue::users
user_range users()
Definition: VPlanValue.h:163
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1167
llvm::VPScalarIVStepsRecipe::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition: VPlanRecipes.cpp:647
llvm::BranchInst::setSuccessor
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
Definition: Instructions.h:3195
llvm::iplist_impl::erase
iterator erase(iterator where)
Definition: ilist.h:268
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPLiveOut::fixPhi
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
Definition: VPlanRecipes.cpp:132
llvm::VPWidenIntOrFpInductionRecipe::getInductionDescriptor
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1103
llvm::RecurKind
RecurKind
These are the kinds of recurrences that we support.
Definition: IVDescriptors.h:35
llvm::VPRecipeBase::mayWriteToMemory
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
Definition: VPlanRecipes.cpp:42
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ElementCount::isScalar
bool isScalar() const
Counting predicates.
Definition: TypeSize.h:414
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:942
Instruction.h
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:201
CommandLine.h
llvm::VPInstruction::CanonicalIVIncrementNUW
@ CanonicalIVIncrementNUW
Definition: VPlan.h:786
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:844
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:157
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1617
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1272
llvm::createStepForVF
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
Definition: LoopVectorize.cpp:944
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1725
llvm::RecurrenceDescriptor::isSelectCmpRecurrenceKind
static bool isSelectCmpRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Definition: IVDescriptors.h:236
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
Twine.h
getOpcode
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:190
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlanRecipes.cpp:181
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:478
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:227
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1934
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::SmallBitVector::all
bool all() const
Returns true if all bits are set.
Definition: SmallBitVector.h:216
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1045
llvm::Instruction::getOpcodeName
const char * getOpcodeName() const
Definition: Instruction.h:159
llvm::RecurrenceDescriptor::getRecurrenceIdentity
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1051
llvm::Instruction
Definition: Instruction.h:42
EnableVPlanNativePath
cl::opt< bool > EnableVPlanNativePath
llvm::RecurrenceDescriptor::getRecurrenceKind
RecurKind getRecurrenceKind() const
Definition: IVDescriptors.h:195
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:54
llvm::VPScalarIVStepsRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:2001
llvm::VPValue
Definition: VPlanValue.h:44
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::BasicBlock::getModule
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:147
VPlan.h
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
IVDescriptors.h
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:256
Type.h
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2139
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1760
llvm::VPInstruction::FirstOrderRecurrenceSplice
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:777
llvm::VPTransformState::setDebugLocFromInst
void setDebugLocFromInst(const Value *V)
Set the debug location in the builder using the debug location in V.
Definition: VPlan.cpp:250
llvm::IRBuilderBase::FastMathFlagGuard
Definition: IRBuilder.h:372
llvm::tgtok::In
@ In
Definition: TGLexer.h:51
llvm::VPActiveLaneMaskPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
Definition: VPlanRecipes.cpp:1216
BasicBlock.h
llvm::cl::opt< bool >
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlanRecipes.cpp:165
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:240
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlanRecipes.cpp:104
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::IRBuilderBase::GetInsertPoint
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:174
llvm::VPExpandSCEVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
Definition: VPlanRecipes.cpp:1003
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1538
llvm::VPlan::isUniformAfterVectorization
bool isUniformAfterVectorization(VPValue *VPV) const
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:2684
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: VPlanRecipes.cpp:674
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlanRecipes.cpp:347
llvm::VPReductionPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1157
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:780
llvm::IRBuilderBase::getInt32
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:461
llvm::PHINode::addIncoming
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Definition: Instructions.h:2812
llvm::VPLane::getLastLaneForVF
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:134
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3153
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:210
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlanRecipes.cpp:361
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:184
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1163
llvm::IRBuilderBase::CreateSelect
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1020
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1394
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:748
llvm::Instruction::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:349
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2276
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:871
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition: Instruction.h:162
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::IRBuilderBase::getTrue
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:441
llvm::VPUser::printOperands
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1010
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:617
llvm::ElementCount::isVector
bool isVector() const
One or more elements.
Definition: TypeSize.h:416
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4669
llvm::VPWidenIntOrFpInductionRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:641
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:436
llvm::IRBuilderBase::CreatePHI
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2221
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:268
llvm::VPScalarIVStepsRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:665
llvm::VPTransformState::addMetadata
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:238
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:952
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1536
llvm::Instruction::getFastMathFlags
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition: Instruction.cpp:289
Mul
BinaryOperator * Mul
Definition: X86PartialReduction.cpp:70
llvm::RecurrenceDescriptor::IntermediateStore
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
Definition: IVDescriptors.h:273
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm::RecurrenceDescriptor::getFastMathFlags
FastMathFlags getFastMathFlags() const
Definition: IVDescriptors.h:199
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::InductionDescriptor::getStep
const SCEV * getStep() const
Definition: IVDescriptors.h:319
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:678
llvm::IRBuilderBase::InsertPointGuard
Definition: IRBuilder.h:350
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::VPCanonicalIVPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Definition: VPlanRecipes.cpp:964
llvm::VPScalarIVStepsRecipe::getStepValue
VPValue * getStepValue() const
Definition: VPlan.h:2002
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: VPlanRecipes.cpp:467
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:209
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:139
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1093
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:400
llvm::VPFirstOrderRecurrencePHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1083
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:164
llvm::VPReductionPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1092
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:438
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:457
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:373
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::VPTransformState::MayGeneratePoisonRecipes
SmallPtrSet< VPRecipeBase *, 16 > MayGeneratePoisonRecipes
Holds recipes that may generate a poison value that is used after vectorization, even when their oper...
Definition: VPlan.h:370
llvm::PHINode::Create
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Definition: Instructions.h:2704
llvm::IRBuilderBase::Insert
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:144
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:659
llvm::IRBuilderBase::GetInsertBlock
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:173
Casting.h
llvm::LoopBase::getHeader
BlockT * getHeader() const
Definition: LoopInfo.h:104
llvm::DebugLoc::print
void print(raw_ostream &OS) const
prints source location /path/to/file.exe:line:col @[inlined at]
Definition: DebugLoc.cpp:103
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:656
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:185
llvm::VPWidenPointerInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:993
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:784
llvm::VPInstruction::CanonicalIVIncrementForPartNUW
@ CanonicalIVIncrementForPartNUW
Definition: VPlan.h:790
llvm::iplist_impl::insertAfter
iterator insertAfter(iterator where, pointer New)
Definition: ilist.h:238
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:207
llvm::SmallBitVector::size
size_type size() const
Returns the number of bits in this bitvector.
Definition: SmallBitVector.h:195
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:771
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1967
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:791
Instructions.h
SmallVector.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:257
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:215
llvm::VPRecipeBase::mayReadFromMemory
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
Definition: VPlanRecipes.cpp:73
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::VPActiveLaneMaskPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1229
llvm::PHINode
Definition: Instructions.h:2662
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::VPBlockBase::getExitingBasicBlock
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:160
llvm::VPFirstOrderRecurrencePHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition: VPlanRecipes.cpp:1054
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:339
llvm::IRBuilderBase::CreateVectorSplat
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1143
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
llvm::SCEV::getType
Type * getType() const
Return the LLVM type of this SCEV expression.
Definition: ScalarEvolution.cpp:392
llvm::VPUser::usesScalars
virtual bool usesScalars(const VPValue *Op) const
Returns true if the VPUser uses scalars of operand Op.
Definition: VPlanValue.h:292
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1648
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1194
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::RecurrenceDescriptor::isMinMaxRecurrenceKind
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Definition: IVDescriptors.h:230
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3097
raw_ostream.h
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1534
llvm::VPExpandSCEVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1016
BasicBlockUtils.h
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: VPlanRecipes.cpp:488
llvm::VPDef::getVPDefID
unsigned getVPDefID() const
Definition: VPlanValue.h:420
llvm::VPInstruction::dump
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlanRecipes.cpp:356
Value.h
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2494
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlanRecipes.cpp:141
Debug.h
llvm::VectorType::get
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:668
llvm::VPReductionPHIRecipe::isOrdered
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1357
llvm::VPlanIngredient
Definition: VPlan.h:2786
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:453
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:781
llvm::VPValue::printAsOperand
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:995
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:1084
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:331
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlanRecipes.cpp:157
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:239
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1729