LLVM 18.0.0git
VPlanRecipes.cpp
Go to the documentation of this file.
1//===- VPlanRecipes.cpp - Implementations for VPlan recipes ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file contains implementations for different VPlan recipes.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlan.h"
15#include "VPlanAnalysis.h"
16#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/Twine.h"
20#include "llvm/IR/BasicBlock.h"
21#include "llvm/IR/IRBuilder.h"
22#include "llvm/IR/Instruction.h"
24#include "llvm/IR/Type.h"
25#include "llvm/IR/Value.h"
28#include "llvm/Support/Debug.h"
32#include <cassert>
33
34using namespace llvm;
35
37
38namespace llvm {
40}
41
42#define LV_NAME "loop-vectorize"
43#define DEBUG_TYPE LV_NAME
44
46 switch (getVPDefID()) {
47 case VPWidenMemoryInstructionSC: {
48 return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
49 }
50 case VPReplicateSC:
51 case VPWidenCallSC:
52 return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
53 ->mayWriteToMemory();
54 case VPBranchOnMaskSC:
55 case VPScalarIVStepsSC:
56 case VPPredInstPHISC:
57 return false;
58 case VPBlendSC:
59 case VPReductionSC:
60 case VPWidenCanonicalIVSC:
61 case VPWidenCastSC:
62 case VPWidenGEPSC:
63 case VPWidenIntOrFpInductionSC:
64 case VPWidenPHISC:
65 case VPWidenSC:
66 case VPWidenSelectSC: {
67 const Instruction *I =
68 dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
69 (void)I;
70 assert((!I || !I->mayWriteToMemory()) &&
71 "underlying instruction may write to memory");
72 return false;
73 }
74 default:
75 return true;
76 }
77}
78
80 switch (getVPDefID()) {
81 case VPWidenMemoryInstructionSC: {
82 return !cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
83 }
84 case VPReplicateSC:
85 case VPWidenCallSC:
86 return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
87 ->mayReadFromMemory();
88 case VPBranchOnMaskSC:
89 case VPScalarIVStepsSC:
90 case VPPredInstPHISC:
91 return false;
92 case VPBlendSC:
93 case VPReductionSC:
94 case VPWidenCanonicalIVSC:
95 case VPWidenCastSC:
96 case VPWidenGEPSC:
97 case VPWidenIntOrFpInductionSC:
98 case VPWidenPHISC:
99 case VPWidenSC:
100 case VPWidenSelectSC: {
101 const Instruction *I =
102 dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
103 (void)I;
104 assert((!I || !I->mayReadFromMemory()) &&
105 "underlying instruction may read from memory");
106 return false;
107 }
108 default:
109 return true;
110 }
111}
112
114 switch (getVPDefID()) {
115 case VPDerivedIVSC:
116 case VPPredInstPHISC:
117 return false;
118 case VPInstructionSC:
119 switch (cast<VPInstruction>(this)->getOpcode()) {
120 case Instruction::ICmp:
125 return false;
126 default:
127 return true;
128 }
129 case VPWidenCallSC:
130 return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
131 ->mayHaveSideEffects();
132 case VPBlendSC:
133 case VPReductionSC:
134 case VPScalarIVStepsSC:
135 case VPWidenCanonicalIVSC:
136 case VPWidenCastSC:
137 case VPWidenGEPSC:
138 case VPWidenIntOrFpInductionSC:
139 case VPWidenPHISC:
140 case VPWidenPointerInductionSC:
141 case VPWidenSC:
142 case VPWidenSelectSC: {
143 const Instruction *I =
144 dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
145 (void)I;
146 assert((!I || !I->mayHaveSideEffects()) &&
147 "underlying instruction has side-effects");
148 return false;
149 }
150 case VPWidenMemoryInstructionSC:
151 assert(cast<VPWidenMemoryInstructionRecipe>(this)
152 ->getIngredient()
154 "mayHaveSideffects result for ingredient differs from this "
155 "implementation");
156 return mayWriteToMemory();
157 case VPReplicateSC: {
158 auto *R = cast<VPReplicateRecipe>(this);
159 return R->getUnderlyingInstr()->mayHaveSideEffects();
160 }
161 default:
162 return true;
163 }
164}
165
167 auto Lane = VPLane::getLastLaneForVF(State.VF);
168 VPValue *ExitValue = getOperand(0);
170 Lane = VPLane::getFirstLane();
171 VPBasicBlock *MiddleVPBB =
172 cast<VPBasicBlock>(Plan.getVectorLoopRegion()->getSingleSuccessor());
173 assert(MiddleVPBB->getNumSuccessors() == 0 &&
174 "the middle block must not have any successors");
175 BasicBlock *MiddleBB = State.CFG.VPBB2IRBB[MiddleVPBB];
176 Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)),
177 MiddleBB);
178}
179
180#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
182 O << "Live-out ";
184 O << " = ";
186 O << "\n";
187}
188#endif
189
191 assert(!Parent && "Recipe already in some VPBasicBlock");
192 assert(InsertPos->getParent() &&
193 "Insertion position not in any VPBasicBlock");
194 Parent = InsertPos->getParent();
195 Parent->getRecipeList().insert(InsertPos->getIterator(), this);
196}
197
200 assert(!Parent && "Recipe already in some VPBasicBlock");
201 assert(I == BB.end() || I->getParent() == &BB);
202 Parent = &BB;
203 BB.getRecipeList().insert(I, this);
204}
205
207 assert(!Parent && "Recipe already in some VPBasicBlock");
208 assert(InsertPos->getParent() &&
209 "Insertion position not in any VPBasicBlock");
210 Parent = InsertPos->getParent();
211 Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
212}
213
215 assert(getParent() && "Recipe not in any VPBasicBlock");
217 Parent = nullptr;
218}
219
221 assert(getParent() && "Recipe not in any VPBasicBlock");
223}
224
227 insertAfter(InsertPos);
228}
229
233 insertBefore(BB, I);
234}
235
237 assert(OpType == OperationType::FPMathOp &&
238 "recipe doesn't have fast math flags");
239 FastMathFlags Res;
240 Res.setAllowReassoc(FMFs.AllowReassoc);
241 Res.setNoNaNs(FMFs.NoNaNs);
242 Res.setNoInfs(FMFs.NoInfs);
243 Res.setNoSignedZeros(FMFs.NoSignedZeros);
244 Res.setAllowReciprocal(FMFs.AllowReciprocal);
245 Res.setAllowContract(FMFs.AllowContract);
246 Res.setApproxFunc(FMFs.ApproxFunc);
247 return Res;
248}
249
252 const Twine &Name)
253 : VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef<VPValue *>({A, B}),
254 Pred, DL),
255 VPValue(this), Opcode(Opcode), Name(Name.str()) {
256 assert(Opcode == Instruction::ICmp &&
257 "only ICmp predicates supported at the moment");
258}
259
261 std::initializer_list<VPValue *> Operands,
262 FastMathFlags FMFs, DebugLoc DL, const Twine &Name)
263 : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL),
264 VPValue(this), Opcode(Opcode), Name(Name.str()) {
265 // Make sure the VPInstruction is a floating-point operation.
266 assert(isFPMathOp() && "this op can't take fast-math flags");
267}
268
269Value *VPInstruction::generateInstruction(VPTransformState &State,
270 unsigned Part) {
271 IRBuilderBase &Builder = State.Builder;
273
275 Value *A = State.get(getOperand(0), Part);
276 Value *B = State.get(getOperand(1), Part);
277 return Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
278 }
279
280 switch (getOpcode()) {
281 case VPInstruction::Not: {
282 Value *A = State.get(getOperand(0), Part);
283 return Builder.CreateNot(A, Name);
284 }
285 case Instruction::ICmp: {
286 Value *A = State.get(getOperand(0), Part);
287 Value *B = State.get(getOperand(1), Part);
288 return Builder.CreateCmp(getPredicate(), A, B, Name);
289 }
290 case Instruction::Select: {
291 Value *Cond = State.get(getOperand(0), Part);
292 Value *Op1 = State.get(getOperand(1), Part);
293 Value *Op2 = State.get(getOperand(2), Part);
294 return Builder.CreateSelect(Cond, Op1, Op2, Name);
295 }
297 // Get first lane of vector induction variable.
298 Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
299 // Get the original loop tripcount.
300 Value *ScalarTC = State.get(getOperand(1), VPIteration(Part, 0));
301
302 auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
303 auto *PredTy = VectorType::get(Int1Ty, State.VF);
304 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
305 {PredTy, ScalarTC->getType()},
306 {VIVElem0, ScalarTC}, nullptr, Name);
307 }
309 // Generate code to combine the previous and current values in vector v3.
310 //
311 // vector.ph:
312 // v_init = vector(..., ..., ..., a[-1])
313 // br vector.body
314 //
315 // vector.body
316 // i = phi [0, vector.ph], [i+4, vector.body]
317 // v1 = phi [v_init, vector.ph], [v2, vector.body]
318 // v2 = a[i, i+1, i+2, i+3];
319 // v3 = vector(v1(3), v2(0, 1, 2))
320
321 // For the first part, use the recurrence phi (v1), otherwise v2.
322 auto *V1 = State.get(getOperand(0), 0);
323 Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1);
324 if (!PartMinus1->getType()->isVectorTy())
325 return PartMinus1;
326 Value *V2 = State.get(getOperand(1), Part);
327 return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name);
328 }
330 Value *ScalarTC = State.get(getOperand(0), {0, 0});
331 Value *Step =
332 createStepForVF(Builder, ScalarTC->getType(), State.VF, State.UF);
333 Value *Sub = Builder.CreateSub(ScalarTC, Step);
334 Value *Cmp = Builder.CreateICmp(CmpInst::Predicate::ICMP_UGT, ScalarTC, Step);
335 Value *Zero = ConstantInt::get(ScalarTC->getType(), 0);
336 return Builder.CreateSelect(Cmp, Sub, Zero);
337 }
339 if (Part == 0) {
340 auto *Phi = State.get(getOperand(0), 0);
341 // The loop step is equal to the vectorization factor (num of SIMD
342 // elements) times the unroll factor (num of SIMD instructions).
343 Value *Step =
344 createStepForVF(Builder, Phi->getType(), State.VF, State.UF);
345 return Builder.CreateAdd(Phi, Step, Name, hasNoUnsignedWrap(),
347 }
348 return State.get(this, 0);
349 }
350
352 auto *IV = State.get(getOperand(0), VPIteration(0, 0));
353 if (Part == 0)
354 return IV;
355
356 // The canonical IV is incremented by the vectorization factor (num of SIMD
357 // elements) times the unroll part.
358 Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
359 return Builder.CreateAdd(IV, Step, Name, hasNoUnsignedWrap(),
361 }
363 if (Part != 0)
364 return nullptr;
365
366 Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
367 VPRegionBlock *ParentRegion = getParent()->getParent();
368 VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
369
370 // Replace the temporary unreachable terminator with a new conditional
371 // branch, hooking it up to backward destination for exiting blocks now and
372 // to forward destination(s) later when they are created.
373 BranchInst *CondBr =
374 Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr);
375
376 if (getParent()->isExiting())
377 CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
378
379 CondBr->setSuccessor(0, nullptr);
381 return CondBr;
382 }
384 if (Part != 0)
385 return nullptr;
386 // First create the compare.
387 Value *IV = State.get(getOperand(0), Part);
388 Value *TC = State.get(getOperand(1), Part);
389 Value *Cond = Builder.CreateICmpEQ(IV, TC);
390
391 // Now create the branch.
392 auto *Plan = getParent()->getPlan();
393 VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
394 VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
395
396 // Replace the temporary unreachable terminator with a new conditional
397 // branch, hooking it up to backward destination (the header) now and to the
398 // forward destination (the exit/middle block) later when it is created.
399 // Note that CreateCondBr expects a valid BB as first argument, so we need
400 // to set it to nullptr later.
401 BranchInst *CondBr = Builder.CreateCondBr(Cond, Builder.GetInsertBlock(),
402 State.CFG.VPBB2IRBB[Header]);
403 CondBr->setSuccessor(0, nullptr);
405 return CondBr;
406 }
407 default:
408 llvm_unreachable("Unsupported opcode for instruction");
409 }
410}
411
412#if !defined(NDEBUG)
413bool VPInstruction::isFPMathOp() const {
414 // Inspired by FPMathOperator::classof. Notable differences are that we don't
415 // support Call, PHI and Select opcodes here yet.
416 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
417 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
418 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
419 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
420}
421#endif
422
424 assert(!State.Instance && "VPInstruction executing an Instance");
426 assert((hasFastMathFlags() == isFPMathOp() ||
427 getOpcode() == Instruction::Select) &&
428 "Recipe not a FPMathOp but has fast-math flags?");
429 if (hasFastMathFlags())
431 for (unsigned Part = 0; Part < State.UF; ++Part) {
432 Value *GeneratedValue = generateInstruction(State, Part);
433 if (!hasResult())
434 continue;
435 assert(GeneratedValue && "generateInstruction must produce a value");
436 State.set(this, GeneratedValue, Part);
437 }
438}
439
440#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
442 VPSlotTracker SlotTracker(getParent()->getPlan());
443 print(dbgs(), "", SlotTracker);
444}
445
447 VPSlotTracker &SlotTracker) const {
448 O << Indent << "EMIT ";
449
450 if (hasResult()) {
452 O << " = ";
453 }
454
455 switch (getOpcode()) {
457 O << "not";
458 break;
460 O << "combined load";
461 break;
463 O << "combined store";
464 break;
466 O << "active lane mask";
467 break;
469 O << "first-order splice";
470 break;
472 O << "VF * UF +";
473 break;
475 O << "branch-on-cond";
476 break;
478 O << "TC > VF ? TC - VF : 0";
479 break;
481 O << "VF * Part +";
482 break;
484 O << "branch-on-count";
485 break;
486 default:
488 }
489
490 printFlags(O);
492
493 if (auto DL = getDebugLoc()) {
494 O << ", !dbg ";
495 DL.print(O);
496 }
497}
498#endif
499
501 assert(State.VF.isVector() && "not widening");
502 auto &CI = *cast<CallInst>(getUnderlyingInstr());
503 assert(!isa<DbgInfoIntrinsic>(CI) &&
504 "DbgInfoIntrinsic should have been dropped during VPlan construction");
505 State.setDebugLocFrom(CI.getDebugLoc());
506
507 FunctionType *VFTy = nullptr;
508 if (Variant)
509 VFTy = Variant->getFunctionType();
510 for (unsigned Part = 0; Part < State.UF; ++Part) {
511 SmallVector<Type *, 2> TysForDecl;
512 // Add return type if intrinsic is overloaded on it.
513 if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1)) {
514 TysForDecl.push_back(
515 VectorType::get(CI.getType()->getScalarType(), State.VF));
516 }
518 for (const auto &I : enumerate(operands())) {
519 // Some intrinsics have a scalar argument - don't replace it with a
520 // vector.
521 // Some vectorized function variants may also take a scalar argument,
522 // e.g. linear parameters for pointers.
523 Value *Arg;
524 if ((VFTy && !VFTy->getParamType(I.index())->isVectorTy()) ||
525 (VectorIntrinsicID != Intrinsic::not_intrinsic &&
526 isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())))
527 Arg = State.get(I.value(), VPIteration(0, 0));
528 else
529 Arg = State.get(I.value(), Part);
530 if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
531 TysForDecl.push_back(Arg->getType());
532 Args.push_back(Arg);
533 }
534
535 Function *VectorF;
536 if (VectorIntrinsicID != Intrinsic::not_intrinsic) {
537 // Use vector version of the intrinsic.
538 Module *M = State.Builder.GetInsertBlock()->getModule();
539 VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
540 assert(VectorF && "Can't retrieve vector intrinsic.");
541 } else {
542#ifndef NDEBUG
543 assert(Variant != nullptr && "Can't create vector function.");
544#endif
545 VectorF = Variant;
546 }
547
549 CI.getOperandBundlesAsDefs(OpBundles);
550 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
551
552 if (isa<FPMathOperator>(V))
553 V->copyFastMathFlags(&CI);
554
555 State.set(this, V, Part);
556 State.addMetadata(V, &CI);
557 }
558}
559
560#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
562 VPSlotTracker &SlotTracker) const {
563 O << Indent << "WIDEN-CALL ";
564
565 auto *CI = cast<CallInst>(getUnderlyingInstr());
566 if (CI->getType()->isVoidTy())
567 O << "void ";
568 else {
570 O << " = ";
571 }
572
573 O << "call @" << CI->getCalledFunction()->getName() << "(";
575 O << ")";
576
577 if (VectorIntrinsicID)
578 O << " (using vector intrinsic)";
579 else {
580 O << " (using library function";
581 if (Variant->hasName())
582 O << ": " << Variant->getName();
583 O << ")";
584 }
585}
586
588 VPSlotTracker &SlotTracker) const {
589 O << Indent << "WIDEN-SELECT ";
591 O << " = select ";
593 O << ", ";
595 O << ", ";
597 O << (isInvariantCond() ? " (condition is loop invariant)" : "");
598}
599#endif
600
603
604 // The condition can be loop invariant but still defined inside the
605 // loop. This means that we can't just use the original 'cond' value.
606 // We have to take the 'vectorized' value and pick the first lane.
607 // Instcombine will make this a no-op.
608 auto *InvarCond =
609 isInvariantCond() ? State.get(getCond(), VPIteration(0, 0)) : nullptr;
610
611 for (unsigned Part = 0; Part < State.UF; ++Part) {
612 Value *Cond = InvarCond ? InvarCond : State.get(getCond(), Part);
613 Value *Op0 = State.get(getOperand(1), Part);
614 Value *Op1 = State.get(getOperand(2), Part);
615 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
616 State.set(this, Sel, Part);
617 State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
618 }
619}
620
621VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
622 const FastMathFlags &FMF) {
623 AllowReassoc = FMF.allowReassoc();
624 NoNaNs = FMF.noNaNs();
625 NoInfs = FMF.noInfs();
626 NoSignedZeros = FMF.noSignedZeros();
627 AllowReciprocal = FMF.allowReciprocal();
628 AllowContract = FMF.allowContract();
629 ApproxFunc = FMF.approxFunc();
630}
631
632#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
634 switch (OpType) {
635 case OperationType::Cmp:
637 break;
638 case OperationType::PossiblyExactOp:
639 if (ExactFlags.IsExact)
640 O << " exact";
641 break;
642 case OperationType::OverflowingBinOp:
643 if (WrapFlags.HasNUW)
644 O << " nuw";
645 if (WrapFlags.HasNSW)
646 O << " nsw";
647 break;
648 case OperationType::FPMathOp:
650 break;
651 case OperationType::GEPOp:
652 if (GEPFlags.IsInBounds)
653 O << " inbounds";
654 break;
655 case OperationType::Other:
656 break;
657 }
658 if (getNumOperands() > 0)
659 O << " ";
660}
661#endif
662
665 auto &Builder = State.Builder;
666 switch (Opcode) {
667 case Instruction::Call:
668 case Instruction::Br:
669 case Instruction::PHI:
670 case Instruction::GetElementPtr:
671 case Instruction::Select:
672 llvm_unreachable("This instruction is handled by a different recipe.");
673 case Instruction::UDiv:
674 case Instruction::SDiv:
675 case Instruction::SRem:
676 case Instruction::URem:
677 case Instruction::Add:
678 case Instruction::FAdd:
679 case Instruction::Sub:
680 case Instruction::FSub:
681 case Instruction::FNeg:
682 case Instruction::Mul:
683 case Instruction::FMul:
684 case Instruction::FDiv:
685 case Instruction::FRem:
686 case Instruction::Shl:
687 case Instruction::LShr:
688 case Instruction::AShr:
689 case Instruction::And:
690 case Instruction::Or:
691 case Instruction::Xor: {
692 // Just widen unops and binops.
693 for (unsigned Part = 0; Part < State.UF; ++Part) {
695 for (VPValue *VPOp : operands())
696 Ops.push_back(State.get(VPOp, Part));
697
698 Value *V = Builder.CreateNAryOp(Opcode, Ops);
699
700 if (auto *VecOp = dyn_cast<Instruction>(V))
701 setFlags(VecOp);
702
703 // Use this vector value for all users of the original instruction.
704 State.set(this, V, Part);
705 State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
706 }
707
708 break;
709 }
710 case Instruction::Freeze: {
711 for (unsigned Part = 0; Part < State.UF; ++Part) {
712 Value *Op = State.get(getOperand(0), Part);
713
714 Value *Freeze = Builder.CreateFreeze(Op);
715 State.set(this, Freeze, Part);
716 }
717 break;
718 }
719 case Instruction::ICmp:
720 case Instruction::FCmp: {
721 // Widen compares. Generate vector compares.
722 bool FCmp = Opcode == Instruction::FCmp;
723 for (unsigned Part = 0; Part < State.UF; ++Part) {
724 Value *A = State.get(getOperand(0), Part);
725 Value *B = State.get(getOperand(1), Part);
726 Value *C = nullptr;
727 if (FCmp) {
728 // Propagate fast math flags.
729 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
730 if (auto *I = dyn_cast_or_null<Instruction>(getUnderlyingValue()))
731 Builder.setFastMathFlags(I->getFastMathFlags());
732 C = Builder.CreateFCmp(getPredicate(), A, B);
733 } else {
734 C = Builder.CreateICmp(getPredicate(), A, B);
735 }
736 State.set(this, C, Part);
737 State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
738 }
739
740 break;
741 }
742 default:
743 // This instruction is not vectorized by simple widening.
744 LLVM_DEBUG(dbgs() << "LV: Found an unhandled opcode : "
745 << Instruction::getOpcodeName(Opcode));
746 llvm_unreachable("Unhandled instruction!");
747 } // end of switch.
748
749#if !defined(NDEBUG)
750 // Verify that VPlan type inference results agree with the type of the
751 // generated values.
752 for (unsigned Part = 0; Part < State.UF; ++Part) {
754 State.VF) == State.get(this, Part)->getType() &&
755 "inferred type and type from generated instructions do not match");
756 }
757#endif
758}
759
760#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
762 VPSlotTracker &SlotTracker) const {
763 O << Indent << "WIDEN ";
765 O << " = " << Instruction::getOpcodeName(Opcode);
766 printFlags(O);
768}
769#endif
770
773 auto &Builder = State.Builder;
774 /// Vectorize casts.
775 assert(State.VF.isVector() && "Not vectorizing?");
776 Type *DestTy = VectorType::get(getResultType(), State.VF);
777 VPValue *Op = getOperand(0);
778 for (unsigned Part = 0; Part < State.UF; ++Part) {
779 if (Part > 0 && Op->isLiveIn()) {
780 // FIXME: Remove once explicit unrolling is implemented using VPlan.
781 State.set(this, State.get(this, 0), Part);
782 continue;
783 }
784 Value *A = State.get(Op, Part);
785 Value *Cast = Builder.CreateCast(Instruction::CastOps(Opcode), A, DestTy);
786 State.set(this, Cast, Part);
787 State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
788 }
789}
790
791#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
793 VPSlotTracker &SlotTracker) const {
794 O << Indent << "WIDEN-CAST ";
796 O << " = " << Instruction::getOpcodeName(Opcode) << " ";
798 O << " to " << *getResultType();
799}
800#endif
801
802/// This function adds
803/// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
804/// to each vector element of Val. The sequence starts at StartIndex.
805/// \p Opcode is relevant for FP induction variable.
806static Value *getStepVector(Value *Val, Value *StartIdx, Value *Step,
808 IRBuilderBase &Builder) {
809 assert(VF.isVector() && "only vector VFs are supported");
810
811 // Create and check the types.
812 auto *ValVTy = cast<VectorType>(Val->getType());
813 ElementCount VLen = ValVTy->getElementCount();
814
815 Type *STy = Val->getType()->getScalarType();
816 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
817 "Induction Step must be an integer or FP");
818 assert(Step->getType() == STy && "Step has wrong type");
819
821
822 // Create a vector of consecutive numbers from zero to VF.
823 VectorType *InitVecValVTy = ValVTy;
824 if (STy->isFloatingPointTy()) {
825 Type *InitVecValSTy =
827 InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
828 }
829 Value *InitVec = Builder.CreateStepVector(InitVecValVTy);
830
831 // Splat the StartIdx
832 Value *StartIdxSplat = Builder.CreateVectorSplat(VLen, StartIdx);
833
834 if (STy->isIntegerTy()) {
835 InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);
836 Step = Builder.CreateVectorSplat(VLen, Step);
837 assert(Step->getType() == Val->getType() && "Invalid step vec");
838 // FIXME: The newly created binary instructions should contain nsw/nuw
839 // flags, which can be found from the original scalar operations.
840 Step = Builder.CreateMul(InitVec, Step);
841 return Builder.CreateAdd(Val, Step, "induction");
842 }
843
844 // Floating point induction.
845 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
846 "Binary Opcode should be specified for FP induction");
847 InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
848 InitVec = Builder.CreateFAdd(InitVec, StartIdxSplat);
849
850 Step = Builder.CreateVectorSplat(VLen, Step);
851 Value *MulOp = Builder.CreateFMul(InitVec, Step);
852 return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
853}
854
855/// A helper function that returns an integer or floating-point constant with
856/// value C.
858 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
859 : ConstantFP::get(Ty, C);
860}
861
863 ElementCount VF) {
864 assert(FTy->isFloatingPointTy() && "Expected floating point type!");
865 Type *IntTy = IntegerType::get(FTy->getContext(), FTy->getScalarSizeInBits());
866 Value *RuntimeVF = getRuntimeVF(B, IntTy, VF);
867 return B.CreateUIToFP(RuntimeVF, FTy);
868}
869
871 assert(!State.Instance && "Int or FP induction being replicated.");
872
875 TruncInst *Trunc = getTruncInst();
876 IRBuilderBase &Builder = State.Builder;
877 assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");
878 assert(State.VF.isVector() && "must have vector VF");
879
880 // The value from the original loop to which we are mapping the new induction
881 // variable.
882 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
883
884 // Fast-math-flags propagate from the original induction instruction.
885 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
886 if (ID.getInductionBinOp() && isa<FPMathOperator>(ID.getInductionBinOp()))
887 Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());
888
889 // Now do the actual transformations, and start with fetching the step value.
890 Value *Step = State.get(getStepValue(), VPIteration(0, 0));
891
892 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
893 "Expected either an induction phi-node or a truncate of it!");
894
895 // Construct the initial value of the vector IV in the vector loop preheader
896 auto CurrIP = Builder.saveIP();
897 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
898 Builder.SetInsertPoint(VectorPH->getTerminator());
899 if (isa<TruncInst>(EntryVal)) {
900 assert(Start->getType()->isIntegerTy() &&
901 "Truncation requires an integer type");
902 auto *TruncType = cast<IntegerType>(EntryVal->getType());
903 Step = Builder.CreateTrunc(Step, TruncType);
904 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
905 }
906
907 Value *Zero = getSignedIntOrFpConstant(Start->getType(), 0);
908 Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start);
909 Value *SteppedStart = getStepVector(
910 SplatStart, Zero, Step, ID.getInductionOpcode(), State.VF, State.Builder);
911
912 // We create vector phi nodes for both integer and floating-point induction
913 // variables. Here, we determine the kind of arithmetic we will perform.
916 if (Step->getType()->isIntegerTy()) {
917 AddOp = Instruction::Add;
918 MulOp = Instruction::Mul;
919 } else {
920 AddOp = ID.getInductionOpcode();
921 MulOp = Instruction::FMul;
922 }
923
924 // Multiply the vectorization factor by the step using integer or
925 // floating-point arithmetic as appropriate.
926 Type *StepType = Step->getType();
927 Value *RuntimeVF;
928 if (Step->getType()->isFloatingPointTy())
929 RuntimeVF = getRuntimeVFAsFloat(Builder, StepType, State.VF);
930 else
931 RuntimeVF = getRuntimeVF(Builder, StepType, State.VF);
932 Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);
933
934 // Create a vector splat to use in the induction update.
935 //
936 // FIXME: If the step is non-constant, we create the vector splat with
937 // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
938 // handle a constant vector splat.
939 Value *SplatVF = isa<Constant>(Mul)
940 ? ConstantVector::getSplat(State.VF, cast<Constant>(Mul))
941 : Builder.CreateVectorSplat(State.VF, Mul);
942 Builder.restoreIP(CurrIP);
943
944 // We may need to add the step a number of times, depending on the unroll
945 // factor. The last of those goes into the PHI.
946 PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind");
947 VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
948 VecInd->setDebugLoc(EntryVal->getDebugLoc());
949 Instruction *LastInduction = VecInd;
950 for (unsigned Part = 0; Part < State.UF; ++Part) {
951 State.set(this, LastInduction, Part);
952
953 if (isa<TruncInst>(EntryVal))
954 State.addMetadata(LastInduction, EntryVal);
955
956 LastInduction = cast<Instruction>(
957 Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add"));
958 LastInduction->setDebugLoc(EntryVal->getDebugLoc());
959 }
960
961 LastInduction->setName("vec.ind.next");
962 VecInd->addIncoming(SteppedStart, VectorPH);
963 // Add induction update using an incorrect block temporarily. The phi node
964 // will be fixed after VPlan execution. Note that at this point the latch
965 // block cannot be used, as it does not exist yet.
966 // TODO: Model increment value in VPlan, by turning the recipe into a
967 // multi-def and a subclass of VPHeaderPHIRecipe.
968 VecInd->addIncoming(LastInduction, VectorPH);
969}
970
971#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
973 VPSlotTracker &SlotTracker) const {
974 O << Indent << "WIDEN-INDUCTION";
975 if (getTruncInst()) {
976 O << "\\l\"";
977 O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
978 O << " +\n" << Indent << "\" ";
980 } else
981 O << " " << VPlanIngredient(IV);
982
983 O << ", ";
985}
986#endif
987
989 // The step may be defined by a recipe in the preheader (e.g. if it requires
990 // SCEV expansion), but for the canonical induction the step is required to be
991 // 1, which is represented as live-in.
993 return false;
994 auto *StepC = dyn_cast<ConstantInt>(getStepValue()->getLiveInIRValue());
995 auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
996 return StartC && StartC->isZero() && StepC && StepC->isOne();
997}
998
999#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1001 VPSlotTracker &SlotTracker) const {
1002 O << Indent;
1004 O << Indent << "= DERIVED-IV ";
1006 O << " + ";
1008 O << " * ";
1010
1011 if (TruncResultTy)
1012 O << " (truncated to " << *TruncResultTy << ")";
1013}
1014#endif
1015
1017 // Fast-math-flags propagate from the original induction instruction.
1019 if (hasFastMathFlags())
1021
1022 /// Compute scalar induction steps. \p ScalarIV is the scalar induction
1023 /// variable on which to base the steps, \p Step is the size of the step.
1024
1025 Value *BaseIV = State.get(getOperand(0), VPIteration(0, 0));
1026 Value *Step = State.get(getStepValue(), VPIteration(0, 0));
1027 IRBuilderBase &Builder = State.Builder;
1028
1029 // Ensure step has the same type as that of scalar IV.
1030 Type *BaseIVTy = BaseIV->getType()->getScalarType();
1031 if (BaseIVTy != Step->getType()) {
1032 // TODO: Also use VPDerivedIVRecipe when only the step needs truncating, to
1033 // avoid separate truncate here.
1034 assert(Step->getType()->isIntegerTy() &&
1035 "Truncation requires an integer step");
1036 Step = State.Builder.CreateTrunc(Step, BaseIVTy);
1037 }
1038
1039 // We build scalar steps for both integer and floating-point induction
1040 // variables. Here, we determine the kind of arithmetic we will perform.
1043 if (BaseIVTy->isIntegerTy()) {
1044 AddOp = Instruction::Add;
1045 MulOp = Instruction::Mul;
1046 } else {
1047 AddOp = InductionOpcode;
1048 MulOp = Instruction::FMul;
1049 }
1050
1051 // Determine the number of scalars we need to generate for each unroll
1052 // iteration.
1053 bool FirstLaneOnly = vputils::onlyFirstLaneUsed(this);
1054 // Compute the scalar steps and save the results in State.
1055 Type *IntStepTy =
1056 IntegerType::get(BaseIVTy->getContext(), BaseIVTy->getScalarSizeInBits());
1057 Type *VecIVTy = nullptr;
1058 Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;
1059 if (!FirstLaneOnly && State.VF.isScalable()) {
1060 VecIVTy = VectorType::get(BaseIVTy, State.VF);
1061 UnitStepVec =
1062 Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF));
1063 SplatStep = Builder.CreateVectorSplat(State.VF, Step);
1064 SplatIV = Builder.CreateVectorSplat(State.VF, BaseIV);
1065 }
1066
1067 unsigned StartPart = 0;
1068 unsigned EndPart = State.UF;
1069 unsigned StartLane = 0;
1070 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue();
1071 if (State.Instance) {
1072 StartPart = State.Instance->Part;
1073 EndPart = StartPart + 1;
1074 StartLane = State.Instance->Lane.getKnownLane();
1075 EndLane = StartLane + 1;
1076 }
1077 for (unsigned Part = StartPart; Part < EndPart; ++Part) {
1078 Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part);
1079
1080 if (!FirstLaneOnly && State.VF.isScalable()) {
1081 auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0);
1082 auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
1083 if (BaseIVTy->isFloatingPointTy())
1084 InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
1085 auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
1086 auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
1087 State.set(this, Add, Part);
1088 // It's useful to record the lane values too for the known minimum number
1089 // of elements so we do those below. This improves the code quality when
1090 // trying to extract the first element, for example.
1091 }
1092
1093 if (BaseIVTy->isFloatingPointTy())
1094 StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy);
1095
1096 for (unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1097 Value *StartIdx = Builder.CreateBinOp(
1098 AddOp, StartIdx0, getSignedIntOrFpConstant(BaseIVTy, Lane));
1099 // The step returned by `createStepForVF` is a runtime-evaluated value
1100 // when VF is scalable. Otherwise, it should be folded into a Constant.
1101 assert((State.VF.isScalable() || isa<Constant>(StartIdx)) &&
1102 "Expected StartIdx to be folded to a constant when VF is not "
1103 "scalable");
1104 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
1105 auto *Add = Builder.CreateBinOp(AddOp, BaseIV, Mul);
1106 State.set(this, Add, VPIteration(Part, Lane));
1107 }
1108 }
1109}
1110
1111#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1113 VPSlotTracker &SlotTracker) const {
1114 O << Indent;
1116 O << " = SCALAR-STEPS ";
1118}
1119#endif
1120
1122 assert(State.VF.isVector() && "not widening");
1123 auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
1124 // Construct a vector GEP by widening the operands of the scalar GEP as
1125 // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
1126 // results in a vector of pointers when at least one operand of the GEP
1127 // is vector-typed. Thus, to keep the representation compact, we only use
1128 // vector-typed operands for loop-varying values.
1129
1130 if (areAllOperandsInvariant()) {
1131 // If we are vectorizing, but the GEP has only loop-invariant operands,
1132 // the GEP we build (by only using vector-typed operands for
1133 // loop-varying values) would be a scalar pointer. Thus, to ensure we
1134 // produce a vector of pointers, we need to either arbitrarily pick an
1135 // operand to broadcast, or broadcast a clone of the original GEP.
1136 // Here, we broadcast a clone of the original.
1137 //
1138 // TODO: If at some point we decide to scalarize instructions having
1139 // loop-invariant operands, this special case will no longer be
1140 // required. We would add the scalarization decision to
1141 // collectLoopScalars() and teach getVectorValue() to broadcast
1142 // the lane-zero scalar value.
1144 for (unsigned I = 0, E = getNumOperands(); I != E; I++)
1145 Ops.push_back(State.get(getOperand(I), VPIteration(0, 0)));
1146
1147 auto *NewGEP =
1148 State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0],
1149 ArrayRef(Ops).drop_front(), "", isInBounds());
1150 for (unsigned Part = 0; Part < State.UF; ++Part) {
1151 Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, NewGEP);
1152 State.set(this, EntryPart, Part);
1153 State.addMetadata(EntryPart, GEP);
1154 }
1155 } else {
1156 // If the GEP has at least one loop-varying operand, we are sure to
1157 // produce a vector of pointers. But if we are only unrolling, we want
1158 // to produce a scalar GEP for each unroll part. Thus, the GEP we
1159 // produce with the code below will be scalar (if VF == 1) or vector
1160 // (otherwise). Note that for the unroll-only case, we still maintain
1161 // values in the vector mapping with initVector, as we do for other
1162 // instructions.
1163 for (unsigned Part = 0; Part < State.UF; ++Part) {
1164 // The pointer operand of the new GEP. If it's loop-invariant, we
1165 // won't broadcast it.
1166 auto *Ptr = isPointerLoopInvariant()
1167 ? State.get(getOperand(0), VPIteration(0, 0))
1168 : State.get(getOperand(0), Part);
1169
1170 // Collect all the indices for the new GEP. If any index is
1171 // loop-invariant, we won't broadcast it.
1173 for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
1174 VPValue *Operand = getOperand(I);
1175 if (isIndexLoopInvariant(I - 1))
1176 Indices.push_back(State.get(Operand, VPIteration(0, 0)));
1177 else
1178 Indices.push_back(State.get(Operand, Part));
1179 }
1180
1181 // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
1182 // but it should be a vector, otherwise.
1183 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
1184 Indices, "", isInBounds());
1185 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
1186 "NewGEP is not a pointer vector");
1187 State.set(this, NewGEP, Part);
1188 State.addMetadata(NewGEP, GEP);
1189 }
1190 }
1191}
1192
1193#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1195 VPSlotTracker &SlotTracker) const {
1196 O << Indent << "WIDEN-GEP ";
1197 O << (isPointerLoopInvariant() ? "Inv" : "Var");
1198 for (size_t I = 0; I < getNumOperands() - 1; ++I)
1199 O << "[" << (isIndexLoopInvariant(I) ? "Inv" : "Var") << "]";
1200
1201 O << " ";
1203 O << " = getelementptr";
1204 printFlags(O);
1206}
1207#endif
1208
1211 // We know that all PHIs in non-header blocks are converted into
1212 // selects, so we don't have to worry about the insertion order and we
1213 // can just use the builder.
1214 // At this point we generate the predication tree. There may be
1215 // duplications since this is a simple recursive scan, but future
1216 // optimizations will clean it up.
1217
1218 unsigned NumIncoming = getNumIncomingValues();
1219
1220 // Generate a sequence of selects of the form:
1221 // SELECT(Mask3, In3,
1222 // SELECT(Mask2, In2,
1223 // SELECT(Mask1, In1,
1224 // In0)))
1225 // Note that Mask0 is never used: lanes for which no path reaches this phi and
1226 // are essentially undef are taken from In0.
1227 VectorParts Entry(State.UF);
1228 for (unsigned In = 0; In < NumIncoming; ++In) {
1229 for (unsigned Part = 0; Part < State.UF; ++Part) {
1230 // We might have single edge PHIs (blocks) - use an identity
1231 // 'select' for the first PHI operand.
1232 Value *In0 = State.get(getIncomingValue(In), Part);
1233 if (In == 0)
1234 Entry[Part] = In0; // Initialize with the first incoming value.
1235 else {
1236 // Select between the current value and the previous incoming edge
1237 // based on the incoming mask.
1238 Value *Cond = State.get(getMask(In), Part);
1239 Entry[Part] =
1240 State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
1241 }
1242 }
1243 }
1244 for (unsigned Part = 0; Part < State.UF; ++Part)
1245 State.set(this, Entry[Part], Part);
1246}
1247
1248#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1250 VPSlotTracker &SlotTracker) const {
1251 O << Indent << "BLEND ";
1253 O << " =";
1254 if (getNumIncomingValues() == 1) {
1255 // Not a User of any mask: not really blending, this is a
1256 // single-predecessor phi.
1257 O << " ";
1259 } else {
1260 for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
1261 O << " ";
1263 O << "/";
1265 }
1266 }
1267}
1268
1270 VPSlotTracker &SlotTracker) const {
1271 O << Indent << "REDUCE ";
1273 O << " = ";
1275 O << " +";
1276 if (isa<FPMathOperator>(getUnderlyingInstr()))
1278 O << " reduce." << Instruction::getOpcodeName(RdxDesc.getOpcode()) << " (";
1280 if (getCondOp()) {
1281 O << ", ";
1283 }
1284 O << ")";
1285 if (RdxDesc.IntermediateStore)
1286 O << " (with final reduction value stored in invariant address sank "
1287 "outside of loop)";
1288}
1289#endif
1290
1292 // Find if the recipe is used by a widened recipe via an intervening
1293 // VPPredInstPHIRecipe. In this case, also pack the scalar values in a vector.
1294 return any_of(users(), [](const VPUser *U) {
1295 if (auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
1296 return any_of(PredR->users(), [PredR](const VPUser *U) {
1297 return !U->usesScalars(PredR);
1298 });
1299 return false;
1300 });
1301}
1302
1303#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1305 VPSlotTracker &SlotTracker) const {
1306 O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
1307
1308 if (!getUnderlyingInstr()->getType()->isVoidTy()) {
1310 O << " = ";
1311 }
1312 if (auto *CB = dyn_cast<CallBase>(getUnderlyingInstr())) {
1313 O << "call";
1314 printFlags(O);
1315 O << "@" << CB->getCalledFunction()->getName() << "(";
1317 O, [&O, &SlotTracker](VPValue *Op) {
1318 Op->printAsOperand(O, SlotTracker);
1319 });
1320 O << ")";
1321 } else {
1323 printFlags(O);
1325 }
1326
1327 if (shouldPack())
1328 O << " (S->V)";
1329}
1330#endif
1331
1333 assert(State.Instance && "Branch on Mask works only on single instance.");
1334
1335 unsigned Part = State.Instance->Part;
1336 unsigned Lane = State.Instance->Lane.getKnownLane();
1337
1338 Value *ConditionBit = nullptr;
1339 VPValue *BlockInMask = getMask();
1340 if (BlockInMask) {
1341 ConditionBit = State.get(BlockInMask, Part);
1342 if (ConditionBit->getType()->isVectorTy())
1343 ConditionBit = State.Builder.CreateExtractElement(
1344 ConditionBit, State.Builder.getInt32(Lane));
1345 } else // Block in mask is all-one.
1346 ConditionBit = State.Builder.getTrue();
1347
1348 // Replace the temporary unreachable terminator with a new conditional branch,
1349 // whose two destinations will be set later when they are created.
1350 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
1351 assert(isa<UnreachableInst>(CurrentTerminator) &&
1352 "Expected to replace unreachable terminator with conditional branch.");
1353 auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
1354 CondBr->setSuccessor(0, nullptr);
1355 ReplaceInstWithInst(CurrentTerminator, CondBr);
1356}
1357
1359 assert(State.Instance && "Predicated instruction PHI works per instance.");
1360 Instruction *ScalarPredInst =
1361 cast<Instruction>(State.get(getOperand(0), *State.Instance));
1362 BasicBlock *PredicatedBB = ScalarPredInst->getParent();
1363 BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
1364 assert(PredicatingBB && "Predicated block has no single predecessor.");
1365 assert(isa<VPReplicateRecipe>(getOperand(0)) &&
1366 "operand must be VPReplicateRecipe");
1367
1368 // By current pack/unpack logic we need to generate only a single phi node: if
1369 // a vector value for the predicated instruction exists at this point it means
1370 // the instruction has vector users only, and a phi for the vector value is
1371 // needed. In this case the recipe of the predicated instruction is marked to
1372 // also do that packing, thereby "hoisting" the insert-element sequence.
1373 // Otherwise, a phi node for the scalar value is needed.
1374 unsigned Part = State.Instance->Part;
1375 if (State.hasVectorValue(getOperand(0), Part)) {
1376 Value *VectorValue = State.get(getOperand(0), Part);
1377 InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
1378 PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
1379 VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
1380 VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
1381 if (State.hasVectorValue(this, Part))
1382 State.reset(this, VPhi, Part);
1383 else
1384 State.set(this, VPhi, Part);
1385 // NOTE: Currently we need to update the value of the operand, so the next
1386 // predicated iteration inserts its generated value in the correct vector.
1387 State.reset(getOperand(0), VPhi, Part);
1388 } else {
1389 Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
1390 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
1391 Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
1392 PredicatingBB);
1393 Phi->addIncoming(ScalarPredInst, PredicatedBB);
1394 if (State.hasScalarValue(this, *State.Instance))
1395 State.reset(this, Phi, *State.Instance);
1396 else
1397 State.set(this, Phi, *State.Instance);
1398 // NOTE: Currently we need to update the value of the operand, so the next
1399 // predicated iteration inserts its generated value in the correct vector.
1400 State.reset(getOperand(0), Phi, *State.Instance);
1401 }
1402}
1403
1404#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1406 VPSlotTracker &SlotTracker) const {
1407 O << Indent << "PHI-PREDICATED-INSTRUCTION ";
1409 O << " = ";
1411}
1412
1414 VPSlotTracker &SlotTracker) const {
1415 O << Indent << "WIDEN ";
1416
1417 if (!isStore()) {
1419 O << " = ";
1420 }
1421 O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
1422
1424}
1425#endif
1426
1428 Value *Start = getStartValue()->getLiveInIRValue();
1429 PHINode *EntryPart = PHINode::Create(Start->getType(), 2, "index");
1430 EntryPart->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
1431
1432 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1433 EntryPart->addIncoming(Start, VectorPH);
1434 EntryPart->setDebugLoc(getDebugLoc());
1435 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1436 State.set(this, EntryPart, Part);
1437}
1438
1439#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1441 VPSlotTracker &SlotTracker) const {
1442 O << Indent << "EMIT ";
1444 O << " = CANONICAL-INDUCTION ";
1446}
1447#endif
1448
1451 Type *Ty) const {
1452 // The types must match and it must be an integer induction.
1454 return false;
1455 // Start must match the start value of this canonical induction.
1456 if (Start != getStartValue())
1457 return false;
1458
1459 // If the step is defined by a recipe, it is not a ConstantInt.
1460 if (Step->getDefiningRecipe())
1461 return false;
1462
1463 ConstantInt *StepC = dyn_cast<ConstantInt>(Step->getLiveInIRValue());
1464 return StepC && StepC->isOne();
1465}
1466
1468 return IsScalarAfterVectorization &&
1469 (!VF.isScalable() || vputils::onlyFirstLaneUsed(this));
1470}
1471
1472#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1474 VPSlotTracker &SlotTracker) const {
1475 O << Indent << "EMIT ";
1477 O << " = WIDEN-POINTER-INDUCTION ";
1479 O << ", " << *IndDesc.getStep();
1480}
1481#endif
1482
1484 assert(!State.Instance && "cannot be used in per-lane");
1485 const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout();
1486 SCEVExpander Exp(SE, DL, "induction");
1487
1488 Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
1489 &*State.Builder.GetInsertPoint());
1490 assert(!State.ExpandedSCEVs.contains(Expr) &&
1491 "Same SCEV expanded multiple times");
1492 State.ExpandedSCEVs[Expr] = Res;
1493 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1494 State.set(this, Res, {Part, 0});
1495}
1496
1497#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1499 VPSlotTracker &SlotTracker) const {
1500 O << Indent << "EMIT ";
1502 O << " = EXPAND SCEV " << *Expr;
1503}
1504#endif
1505
1507 Value *CanonicalIV = State.get(getOperand(0), 0);
1508 Type *STy = CanonicalIV->getType();
1509 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
1510 ElementCount VF = State.VF;
1511 Value *VStart = VF.isScalar()
1512 ? CanonicalIV
1513 : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");
1514 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1515 Value *VStep = createStepForVF(Builder, STy, VF, Part);
1516 if (VF.isVector()) {
1517 VStep = Builder.CreateVectorSplat(VF, VStep);
1518 VStep =
1519 Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
1520 }
1521 Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
1522 State.set(this, CanonicalVectorIV, Part);
1523 }
1524}
1525
1526#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1528 VPSlotTracker &SlotTracker) const {
1529 O << Indent << "EMIT ";
1531 O << " = WIDEN-CANONICAL-INDUCTION ";
1533}
1534#endif
1535
1537 auto &Builder = State.Builder;
1538 // Create a vector from the initial value.
1539 auto *VectorInit = getStartValue()->getLiveInIRValue();
1540
1541 Type *VecTy = State.VF.isScalar()
1542 ? VectorInit->getType()
1543 : VectorType::get(VectorInit->getType(), State.VF);
1544
1545 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1546 if (State.VF.isVector()) {
1547 auto *IdxTy = Builder.getInt32Ty();
1548 auto *One = ConstantInt::get(IdxTy, 1);
1549 IRBuilder<>::InsertPointGuard Guard(Builder);
1550 Builder.SetInsertPoint(VectorPH->getTerminator());
1551 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
1552 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
1553 VectorInit = Builder.CreateInsertElement(
1554 PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
1555 }
1556
1557 // Create a phi node for the new recurrence.
1558 PHINode *EntryPart = PHINode::Create(VecTy, 2, "vector.recur");
1559 EntryPart->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
1560 EntryPart->addIncoming(VectorInit, VectorPH);
1561 State.set(this, EntryPart, 0);
1562}
1563
1564#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1566 VPSlotTracker &SlotTracker) const {
1567 O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";
1569 O << " = phi ";
1571}
1572#endif
1573
1575 PHINode *PN = cast<PHINode>(getUnderlyingValue());
1576 auto &Builder = State.Builder;
1577
1578 // In order to support recurrences we need to be able to vectorize Phi nodes.
1579 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
1580 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
1581 // this value when we vectorize all of the instructions that use the PHI.
1582 bool ScalarPHI = State.VF.isScalar() || IsInLoop;
1583 Type *VecTy =
1584 ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
1585
1586 BasicBlock *HeaderBB = State.CFG.PrevBB;
1587 assert(State.CurrentVectorLoop->getHeader() == HeaderBB &&
1588 "recipe must be in the vector loop header");
1589 unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
1590 for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1591 Instruction *EntryPart = PHINode::Create(VecTy, 2, "vec.phi");
1592 EntryPart->insertBefore(HeaderBB->getFirstInsertionPt());
1593 State.set(this, EntryPart, Part);
1594 }
1595
1596 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1597
1598 // Reductions do not have to start at zero. They can start with
1599 // any loop invariant values.
1600 VPValue *StartVPV = getStartValue();
1601 Value *StartV = StartVPV->getLiveInIRValue();
1602
1603 Value *Iden = nullptr;
1604 RecurKind RK = RdxDesc.getRecurrenceKind();
1607 // MinMax and AnyOf reductions have the start value as their identity.
1608 if (ScalarPHI) {
1609 Iden = StartV;
1610 } else {
1611 IRBuilderBase::InsertPointGuard IPBuilder(Builder);
1612 Builder.SetInsertPoint(VectorPH->getTerminator());
1613 StartV = Iden =
1614 Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
1615 }
1616 } else {
1617 Iden = RdxDesc.getRecurrenceIdentity(RK, VecTy->getScalarType(),
1618 RdxDesc.getFastMathFlags());
1619
1620 if (!ScalarPHI) {
1621 Iden = Builder.CreateVectorSplat(State.VF, Iden);
1622 IRBuilderBase::InsertPointGuard IPBuilder(Builder);
1623 Builder.SetInsertPoint(VectorPH->getTerminator());
1624 Constant *Zero = Builder.getInt32(0);
1625 StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
1626 }
1627 }
1628
1629 for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1630 Value *EntryPart = State.get(this, Part);
1631 // Make sure to add the reduction start value only to the
1632 // first unroll part.
1633 Value *StartVal = (Part == 0) ? StartV : Iden;
1634 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1635 }
1636}
1637
1638#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1640 VPSlotTracker &SlotTracker) const {
1641 O << Indent << "WIDEN-REDUCTION-PHI ";
1642
1644 O << " = phi ";
1646}
1647#endif
1648
1651 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1652
1653 Value *Op0 = State.get(getOperand(0), 0);
1654 Type *VecTy = Op0->getType();
1655 Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
1656 State.set(this, VecPhi, 0);
1657}
1658
1659#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1661 VPSlotTracker &SlotTracker) const {
1662 O << Indent << "WIDEN-PHI ";
1663
1664 auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
1665 // Unless all incoming values are modeled in VPlan print the original PHI
1666 // directly.
1667 // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
1668 // values as VPValues.
1669 if (getNumOperands() != OriginalPhi->getNumOperands()) {
1670 O << VPlanIngredient(OriginalPhi);
1671 return;
1672 }
1673
1675 O << " = phi ";
1677}
1678#endif
1679
1680// TODO: It would be good to use the existing VPWidenPHIRecipe instead and
1681// remove VPActiveLaneMaskPHIRecipe.
1683 BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1684 for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1685 Value *StartMask = State.get(getOperand(0), Part);
1686 PHINode *EntryPart =
1687 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
1688 EntryPart->addIncoming(StartMask, VectorPH);
1689 EntryPart->setDebugLoc(getDebugLoc());
1690 State.set(this, EntryPart, Part);
1691 }
1692}
1693
1694#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1696 VPSlotTracker &SlotTracker) const {
1697 O << Indent << "ACTIVE-LANE-MASK-PHI ";
1698
1700 O << " = phi ";
1702}
1703#endif
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
Hexagon Common GEP
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
static Value * getStepVector(Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step,...
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static Value * getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy, ElementCount VF)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
This file contains the declarations of the Vectorization Plan base classes:
static constexpr uint32_t Opcode
Definition: aarch32.h:200
static const uint32_t IV[8]
Definition: blake3_impl.h:78
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:446
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:489
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:228
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:328
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:748
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:771
static StringRef getPredicateName(Predicate P)
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:927
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:203
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:114
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
Definition: Constants.cpp:1385
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
constexpr bool isVector() const
One or more elements.
Definition: TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:308
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:20
void setAllowContract(bool B=true)
Definition: FMF.h:91
bool noSignedZeros() const
Definition: FMF.h:68
bool noInfs() const
Definition: FMF.h:67
void setAllowReciprocal(bool B=true)
Definition: FMF.h:88
bool allowReciprocal() const
Definition: FMF.h:69
void print(raw_ostream &O) const
Print fast-math flags to O.
Definition: Operator.cpp:239
void setNoSignedZeros(bool B=true)
Definition: FMF.h:85
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
bool approxFunc() const
Definition: FMF.h:71
void setNoNaNs(bool B=true)
Definition: FMF.h:79
void setAllowReassoc(bool B=true)
Flag setters.
Definition: FMF.h:76
bool noNaNs() const
Definition: FMF.h:66
void setApproxFunc(bool B=true)
Definition: FMF.h:94
void setNoInfs(bool B=true)
Definition: FMF.h:82
bool allowContract() const
Definition: FMF.h:70
Class to represent function types.
Definition: DerivedTypes.h:103
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:200
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1996
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2334
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2445
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2067
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2433
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1527
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Definition: IRBuilder.cpp:1184
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1212
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:460
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Definition: IRBuilder.cpp:930
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1108
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:175
Value * CreateFreeze(Value *V, const Twine &Name="")
Definition: IRBuilder.h:2508
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:520
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:305
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Definition: IRBuilder.h:220
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2060
InsertPoint saveIP() const
Returns the current insert point.
Definition: IRBuilder.h:271
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:480
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2339
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2370
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1748
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2214
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1338
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition: IRBuilder.h:1114
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Definition: IRBuilder.cpp:1003
LLVMContext & getContext() const
Definition: IRBuilder.h:176
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1321
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1660
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2134
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
Definition: IRBuilder.h:283
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:180
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2385
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1865
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2324
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Definition: IRBuilder.h:1581
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Definition: IRBuilder.cpp:109
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1355
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2639
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
Definition: Instruction.cpp:98
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:438
bool isBinaryOp() const
Definition: Instruction.h:244
const BasicBlock * getParent() const
Definition: Instruction.h:139
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:93
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
Definition: Instruction.h:241
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:239
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:435
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:285
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:275
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1743
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:689
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition: Type.h:185
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:228
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:348
Value * getOperand(unsigned i) const
Definition: User.h:169
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2260
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2307
iterator end()
Definition: VPlan.h:2291
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1644
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1647
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1641
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
Definition: VPlan.h:492
size_t getNumSuccessors() const
Definition: VPlan.h:537
VPlan * getPlan()
Definition: VPlan.cpp:146
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:151
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:527
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1888
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, Type *Ty) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:2085
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:313
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:395
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:407
unsigned getVPDefID() const
Definition: VPlanValue.h:427
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
Definition: VPlan.h:2204
VPValue * getCanonicalIV() const
Definition: VPlan.h:2203
VPValue * getStartValue() const
Definition: VPlan.h:2202
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1371
bool hasResult() const
Definition: VPlan.h:1109
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
Definition: VPlan.h:1085
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:1024
@ CanonicalIVIncrementForPart
Definition: VPlan.h:1035
@ CalculateTripCountMinusVF
Definition: VPlan.h:1031
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:167
static VPLane getFirstLane()
Definition: VPlan.h:165
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
PHINode * getPhi() const
Definition: VPlan.h:692
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:707
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:767
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
Definition: VPlan.h:729
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition: VPlan.h:804
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
Definition: VPlan.h:825
ExactFlagsTy ExactFlags
Definition: VPlan.h:867
FastMathFlagsTy FMFs
Definition: VPlan.h:869
void setFlags(Instruction *I) const
Set the IR flags for I.
Definition: VPlan.h:953
bool isInBounds() const
Definition: VPlan.h:986
GEPFlagsTy GEPFlags
Definition: VPlan.h:868
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
Definition: VPlan.h:993
WrapFlagsTy WrapFlags
Definition: VPlan.h:866
bool hasNoUnsignedWrap() const
Definition: VPlan.h:997
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
Definition: VPlan.h:980
bool hasNoSignedWrap() const
Definition: VPlan.h:1003
FastMathFlags getFastMathFlags() const
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1615
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1786
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1788
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1784
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2384
const VPBlockBase * getEntry() const
Definition: VPlan.h:2423
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
Definition: VPlan.h:2247
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:445
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:211
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1163
operand_range operands()
Definition: VPlanValue.h:286
unsigned getNumOperands() const
Definition: VPlanValue.h:260
operand_iterator op_begin()
Definition: VPlanValue.h:282
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:261
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition: VPlan.cpp:116
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:1148
friend class VPInstruction
Definition: VPlanValue.h:47
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:187
user_range users()
Definition: VPlanValue.h:147
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
Definition: VPlan.h:1214
void execute(VPTransformState &State) override
Produce widened copies of the cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1450
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1445
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1456
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1993
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2481
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2677
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4915
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:683
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:172
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:169
self_iterator getIterator()
Definition: ilist_node.h:109
iterator erase(iterator where)
Definition: ilist.h:204
pointer remove(iterator &IT)
Definition: ilist.h:188
iterator insertAfter(iterator where, pointer New)
Definition: ilist.h:174
iterator insert(iterator where, pointer New)
Definition: ilist.h:165
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1444
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:3030
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1246
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2375
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:2134
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1733
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
Definition: VPlan.cpp:51
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
RecurKind
These are the kinds of recurrences that we support.
Definition: IVDescriptors.h:34
@ Mul
Product of integers.
@ Add
Sum of integers.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:217
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:363
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:371
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:329
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:234
DenseMap< const SCEV *, Value * > ExpandedSCEVs
Map SCEVs to their expanded values.
Definition: VPlan.h:412
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:342
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: VPlan.cpp:237
VPTypeAnalysis TypeAnalysis
VPlan-based type analysis.
Definition: VPlan.h:415
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:297
struct llvm::VPTransformState::CFGState CFG
std::optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:248
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:387
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:278
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:272
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:242
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:401
void setDebugLocFrom(DebugLoc DL)
Set the debug location in the builder using the debug location DL.
Definition: VPlan.cpp:362
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:289
bool isInvariantCond() const
Definition: VPlan.h:1274
VPValue * getCond() const
Definition: VPlan.h:1270
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.