62 WideCanIV->getDebugLoc(), Builder,
63 WideCanIV->getNoWrapFlags().withoutNoSignedWrap()));
64 WideCanIV->eraseFromParent();
81 WideCanIV->replaceAllUsesWith(WidenIV);
82 WideCanIV->eraseFromParent();
91 if (PHICost > BroadcastCost)
100 unsigned RegClass =
TTI.getRegisterClassForType(
true, VecTy);
111 nullptr, Plan.
getZero(CanIVTy), StepV, &Plan.
getVF(), ID,
112 WideCanIV->getNoWrapFlags(), WideCanIV->getDebugLoc());
113 NewWideIV->insertBefore(&*Header->getFirstNonPhi());
114 WideCanIV->replaceAllUsesWith(NewWideIV);
115 WideCanIV->eraseFromParent();
148 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
157 {StartV, TC, ALMMultiplier},
DL,
158 "active.lane.mask.entry");
161 "extract.entry.alm.part");
168 LaneMaskPhi->insertBefore(HeaderVPBB->begin());
172 VPValue *IncrementValue = CanonicalIVIncrement;
177 TC = Builder.createScalarIntrinsic(Intrinsic::usub_sat, {TC, &VFxUF},
179 "trip.count.minus.vf");
186 if (!IsBranchOnCount) {
188 CanonicalIVIncrement,
191 assert(LatchExit &&
"expected latch exit compare");
196 Builder.setInsertPoint(LatchExit);
198 {IncrementValue, TC, ALMMultiplier},
DL,
199 "active.lane.mask.next");
202 "extract.next.alm.part");
203 LaneMaskPhi->addBackedgeValue(ALM);
207 auto *NotMask = Builder.createNot(ALM,
DL);
211 assert(LatchExit->getSingleUser()->usesFirstLaneOnly(LatchExit) &&
212 "Single user of latch exit compare must only use the first lane");
213 LatchExit->replaceAllUsesWith(NotMask);
215 LatchExit->eraseFromParent();
220 VPlan &Plan,
bool UseActiveLaneMask,
bool UseActiveLaneMaskForControlFlow) {
226 if (UseActiveLaneMaskForControlFlow) {
232 VPBuilder Builder(Header->getFirstNonPhi());
237 if (UseActiveLaneMask) {
305 AddOp = Instruction::Add;
306 MulOp = Instruction::Mul;
308 AddOp = ID.getInductionOpcode();
309 MulOp = Instruction::FMul;
317 Step = Builder.createScalarCast(Instruction::Trunc, Step, Ty,
DL);
318 Start = Builder.createScalarCast(Instruction::Trunc, Start, Ty,
DL);
327 Init = Builder.createWidenCast(Instruction::UIToFP,
Init, StepTy);
332 Init = Builder.createNaryOp(MulOp, {
Init, SplatStep}, Flags);
333 Init = Builder.createNaryOp(AddOp, {SplatStart,
Init}, Flags,
351 if (R->getParent()->getEnclosingLoopRegion())
352 Builder.setInsertPoint(std::next(R->getIterator()));
357 VF = Builder.createScalarCast(Instruction::CastOps::UIToFP, VF, StepTy,
360 VF = Builder.createScalarZExtOrTrunc(VF, StepTy,
DL);
362 Inc = Builder.createNaryOp(MulOp, {Step, VF}, Flags);
369 auto *
Next = Builder.createNaryOp(AddOp, {Prev, Inc}, Flags,
372 WidePHI->addIncoming(
Next);
399 VPlan *Plan = R->getParent()->getPlan();
400 VPValue *Start = R->getStartValue();
401 VPValue *Step = R->getStepValue();
404 assert(R->getInductionDescriptor().getKind() ==
406 "Not a pointer induction according to InductionDescriptor!");
407 assert(R->getScalarType()->isPointerTy() &&
"Unexpected type.");
409 "Recipe should have been replaced");
415 VPPhi *ScalarPtrPhi = Builder.createScalarPhi(Start,
DL,
"pointer.phi");
419 Builder.setInsertPoint(R->getParent()->getFirstNonPhi());
422 Offset = Builder.createOverflowingOp(Instruction::Mul, {
Offset, Step});
424 Builder.createWidePtrAdd(ScalarPtrPhi,
Offset,
DL,
"vector.gep");
425 R->replaceAllUsesWith(PtrAdd);
430 VF = Builder.createScalarZExtOrTrunc(VF, StepTy,
DL);
431 VPValue *Inc = Builder.createOverflowingOp(Instruction::Mul, {Step, VF});
434 Builder.createPtrAdd(ScalarPtrPhi, Inc,
DL,
"ptr.ind");
441 VPValue *Start = R->getStartValue();
442 VPValue *Step = R->getStepValue();
443 VPValue *Index = R->getIndex();
446 ? Builder.createScalarZExtOrTrunc(
448 : Builder.createScalarCast(Instruction::UIToFP, Index, StepTy,
451 switch (R->getInductionKind()) {
453 assert(Index->getScalarType() == Start->getScalarType() &&
454 "Index type does not match StartValue type");
455 return R->replaceAllUsesWith(Builder.createAdd(
457 Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags),
461 return R->replaceAllUsesWith(Builder.createPtrAdd(
463 Builder.createOverflowingOp(Instruction::Mul, {Index, Step}, Flags)));
468 (FPBinOp->
getOpcode() == Instruction::FAdd ||
469 FPBinOp->
getOpcode() == Instruction::FSub) &&
470 "Original BinOp should be defined for FP induction");
472 VPValue *
FMul = Builder.createNaryOp(Instruction::FMul, {Step, Index}, FMF);
473 return R->replaceAllUsesWith(
474 Builder.createNaryOp(FPBinOp->
getOpcode(), {Start, FMul}, FMF));
487 if (!R->isReplicator())
491 R->dissolveToCFGLoop();
512 assert(Br->getNumOperands() == 2 &&
513 "BranchOnTwoConds must have exactly 2 conditions");
517 assert(Successors.size() == 3 &&
518 "BranchOnTwoConds must have exactly 3 successors");
523 VPValue *Cond0 = Br->getOperand(0);
524 VPValue *Cond1 = Br->getOperand(1);
531 if (Succ0 == Succ1) {
533 VPValue *Combined = Builder.createOr(Cond0, Cond1,
DL);
537 Br->eraseFromParent();
542 !BrOnTwoCondsBB->
getParent() &&
"regions must already be dissolved");
555 Br->eraseFromParent();
566 if (
Widen->getOpcode() == Instruction::Select &&
568 Widen->eraseMetadata(LLVMContext::MD_prof);
573 WidenIVR->eraseFromParent();
582 WidenIVR, Plan, Builder);
583 WidenIVR->replaceAllUsesWith(PtrAdd);
584 WidenIVR->eraseFromParent();
588 WidenIVR->eraseFromParent();
593 VPPhi *CondIVPhi = Builder.createScalarPhi(
594 CondIV->getStartValue(), CondIV->getDebugLoc(),
"conditional.iv");
595 CondIVPhi->
addIncoming(CondIV->getBackedgeValue());
596 CondIV->replaceAllUsesWith(CondIVPhi);
597 CondIV->eraseFromParent();
603 DerivedIVR->eraseFromParent();
608 VPValue *CanIV = WideCanIV->getCanonicalIV();
610 VPValue *Step = WideCanIV->getStepValue();
613 "Expected unroller to have materialized step for UF != 1");
618 Step = Builder.createAdd(
621 Builder.createAdd(CanIV, Step, WideCanIV->getDebugLoc(),
"vec.iv",
622 WideCanIV->getNoWrapFlags());
624 WideCanIV->eraseFromParent();
631 for (
unsigned I = 1;
I != Blend->getNumIncomingValues(); ++
I)
632 Select = Builder.createSelect(Blend->getMask(
I),
633 Blend->getIncomingValue(
I),
Select,
634 R.getDebugLoc(),
"predphi", *Blend);
635 Blend->replaceAllUsesWith(
Select);
636 Blend->eraseFromParent();
641 if (!VEPR->getOffset()) {
643 "Expected unroller to have materialized offset for UF != 1");
644 VEPR->materializeOffset();
651 Expr->eraseFromParent();
661 for (
VPValue *
Op : LastActiveL->operands()) {
662 VPValue *NotMask = Builder.createNot(
Op, LastActiveL->getDebugLoc());
667 VPValue *FirstInactiveLane = Builder.createFirstActiveLane(
668 NotMasks, LastActiveL->getDebugLoc(),
"first.inactive.lane");
674 Builder.createSub(FirstInactiveLane, One,
675 LastActiveL->getDebugLoc(),
"last.active.lane");
678 LastActiveL->eraseFromParent();
688 Instruction::Add, VPI->operands(), VPI->getNoWrapFlags(),
690 VPI->replaceAllUsesWith(
Add);
691 VPI->eraseFromParent();
699 DebugLoc DL = BranchOnCountInst->getDebugLoc();
702 BranchOnCountInst->eraseFromParent();
717 ? Instruction::UIToFP
718 : Instruction::Trunc;
719 VectorStep = Builder.createWidenCast(CastOp, VectorStep, IVTy);
725 Builder.createWidenCast(Instruction::Trunc, ScalarStep, IVTy);
731 MulOpc = Instruction::FMul;
732 Flags = VPI->getFastMathFlagsOrNone();
734 MulOpc = Instruction::Mul;
739 MulOpc, {VectorStep, ScalarStep}, Flags, R.getDebugLoc());
741 VPI->replaceAllUsesWith(VectorStep);
742 VPI->eraseFromParent();
761 for (
VPValue *VPV : VPValues) {
768 if (
User->usesScalars(VPV))
771 HoistPoint = VectorPreheader->getFirstNonPhi();
775 "All users must be in the vector preheader or dominated by it");
780 VPV->replaceUsesWithIf(Broadcast, [VPV, Broadcast](
VPUser *U) {
781 return Broadcast != U && !U->usesScalars(VPV);
789 assert(Plan.
hasVF(BestVF) &&
"BestVF is not available in Plan");
790 assert(Plan.
hasUF(BestUF) &&
"BestUF is not available in Plan");
834 for (
VPValue *Def : R.definedValues()) {
844 unsigned NumFirstLaneUsers =
count_if(Def->users(), [&Def](
VPUser *U) {
845 return U->usesFirstLaneOnly(Def);
847 if (!NumFirstLaneUsers || NumFirstLaneUsers == Def->getNumUsers())
854 Unpack->insertAfter(&R);
855 Def->replaceUsesWithIf(
856 Unpack, [&Def](
VPUser *U) {
return U->usesFirstLaneOnly(Def); });
864 bool RequiresScalarEpilogue,
VPValue *Step,
865 std::optional<uint64_t> MaxRuntimeStep) {
877 "Step VPBB must dominate VectorPHVPBB");
879 InsertPt = std::next(StepR->getIterator());
887 if (!RequiresScalarEpilogue &&
match(TC,
m_APInt(TCVal)) && MaxRuntimeStep &&
888 TCVal->
urem(*MaxRuntimeStep) == 0) {
900 TC = Builder.createAdd(
911 Builder.createNaryOp(Instruction::URem, {TC, Step},
920 if (RequiresScalarEpilogue) {
922 "requiring scalar epilogue is not supported with fail folding");
925 R = Builder.createSelect(IsZero, Step, R);
939 "VF and VFxUF must be materialized together");
951 Builder.createElementCount(TCTy, VFEC * Plan.
getConcreteUF());
958 VPValue *RuntimeVF = Builder.createElementCount(TCTy, VFEC);
966 Builder.createElementCount(TCTy, VFEC * Plan.
getConcreteUF());
979 assert(IncomingAliasMask &&
"Expected an alias mask!");
988 if (
Check.NeedsFreeze) {
989 Src = Builder.createFreeze(Src);
990 Sink = Builder.createFreeze(Sink);
998 Intrinsic::loop_dependence_war_mask,
1002 AliasMask = Builder.createAnd(AliasMask, WARMask);
1004 AliasMask = WARMask;
1009 VPValue *NumActive = Builder.createNaryOp(
1012 VPValue *ClampedVF = Builder.createScalarZExtOrTrunc(
1022 bool HasBranchWeights) {
1040 VPValue *DistanceToMax = Builder.createSub(MaxUIntTripCount, TripCount);
1048 VPValue *TripCountCheck = Builder.createICmp(
1051 VPValue *
Cond = Builder.createOr(IsScalar, TripCountCheck,
DL);
1062 "Clamped VF not supported with interleaving");
1080 if (ExpSCEV.user_empty())
1082 Builder.setInsertPoint(&ExpSCEV);
1084 if (Expanded == &ExpSCEV)
1091 ExpSCEV.eraseFromParent();
1103 BasicBlock *EntryBB = Entry->getIRBasicBlock();
1108 const SCEV *Expr = ExpSCEV.getSCEV();
1111 ExpandedSCEVs[Expr] = Res;
1116 ExpSCEV.eraseFromParent();
1119 "all VPExpandSCEVRecipes must have been expanded");
1122 auto EI = Entry->begin();
1132 return ExpandedSCEVs;
1147 "must have a BranchOnCond");
1150 if (VF.
isScalable() && VScaleForTuning.has_value())
1151 VectorStep *= *VScaleForTuning;
1152 assert(VectorStep > 0 &&
"trip count should not be zero");
1156 MiddleTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo MachineBasicBlock::iterator InsertPt
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
This file provides a LoopVectorizationPlanner class.
SmallVector< MachineOperand, 4 > Cond
static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A, const MachineInstr &B)
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
This file contains the declarations of different VPlan-related auxiliary helpers.
static VPActiveLaneMaskPHIRecipe * addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan)
static void expandVPDerivedIV(VPDerivedIVRecipe *R)
Expand a VPDerivedIVRecipe into executable recipes.
static void expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR)
Expand a VPWidenIntOrFpInduction into executable recipes, for the initial value, phi and backedge val...
static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R)
Expand a VPWidenPointerInductionRecipe into executable recipes, for the initial value,...
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
Class for arbitrary precision integers.
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
LLVM_ABI IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
static DebugLoc getCompilerGenerated()
static DebugLoc getUnknown()
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
Utility class for floating point operations which can have information about relaxed accuracy require...
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Convenience struct for specifying and reasoning about fast-math flags.
A struct for saving information about induction variables.
static LLVM_ABI InductionDescriptor getCanonicalIntInduction(Type *Ty, ScalarEvolution &SE)
Returns the canonical integer induction for type Ty with start = 0 and step = 1.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)
Return metadata containing two branch weights.
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
This class uses information about analyze scalars to rewrite expressions in canonical form.
LLVM_ABI Value * expandCodeFor(SCEVUse SH, Type *Ty, BasicBlock::iterator I)
Insert code to directly compute the specified SCEV expression into the program.
This class represents an analyzed expression in the program.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
LLVM_ABI const SCEV * getUDivExpr(SCEVUse LHS, SCEVUse RHS)
Get a canonical unsigned division expression, or something simpler if possible.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEVFlags Flags=SCEV::FlagNone)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI SCEVUse getMulExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy::iterator iterator
Instruction iterators...
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
const std::string & getName() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static auto blocksAs(T &&Range)
Return an iterator range over Range with each block cast to BlockTy.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static auto blocksOnly(T &&Range)
Return an iterator range over Range which only includes BlockTy blocks.
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", Type *ResultTy=nullptr, const VPIRAttributes &Attrs={})
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPInstruction * createSub(VPValue *LHS, VPValue *RHS, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", VPRecipeWithIRFlags::WrapFlagsTy WrapFlags={false, false})
VPWidenPHIRecipe * createWidenPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
A recipe for converting Current into Start + Current * Step.
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
Recipe to expand a SCEV expression.
Class to record and manage LLVM IR flags.
static LLVM_ABI_FOR_TEST VPIRFlags getDefaultFlags(unsigned Opcode, Type *ResultTy=nullptr)
Returns default flags for Opcode and scalar ResultTy for opcodes that support it, asserts otherwise.
void dropPoisonGeneratingFlags()
Drop all poison-generating flags.
static LLVM_ABI_FOR_TEST VPIRInstruction * create(Instruction &I)
Create a new VPIRPhi for \I , if it is a PHINode, otherwise create a VPIRInstruction.
This is a concrete Recipe that models a single VPlan-level instruction.
@ Unpack
Extracts all lanes from its (non-scalable) vector operand.
void addIncoming(VPValue *IncomingV)
Append IncomingV as an incoming value to the phi-like recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Type * getScalarType() const
Returns the scalar type of this VPRecipeValue.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
VPRegionValue * getUsedHeaderMask() const
Return the header mask if it exists and is used, or null otherwise.
VPInstruction * getOrCreateCanonicalIVIncrement()
Get the canonical IV increment instruction if it exists.
Type * getCanonicalIVType() const
Return the type of the canonical IV for loop regions.
bool hasCanonicalIVNUW() const
Indicates if NUW is set for the canonical IV increment, for loop regions.
VPRegionValue * getCanonicalIV()
Return the canonical induction variable of the region, null for replicating regions.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Lightweight SCEV-to-VPlan expander.
VPValue * expand(const SCEV *S)
Expand S into recipes and live-ins using the builder.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
A symbolic live-in VPValue, used for values like vector trip count, VF, and VFxUF.
bool isMaterialized() const
Returns true if this value has been materialized.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
Type * getScalarType() const
Returns the scalar type of this VPValue, dispatching based on the concrete subclass.
Value * getLiveInIRValue() const
Return the underlying IR value for a VPIRValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser *U)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
void setUnderlyingValue(Value *Val)
void replaceAllUsesWith(VPValue *New)
A Recipe for widening the canonical induction variable of the vector loop.
VPValue * getStepValue()
Returns the step value of the induction.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
VPValue * getSplatVFValue() const
If the recipe has been unrolled, return the VPValue for the induction increment, otherwise return nul...
VPValue * getLastUnrolledPartOperand()
Returns the VPValue representing the value of this induction at the last unrolled part,...
A recipe for widening vector intrinsics.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
bool hasVF(ElementCount VF) const
const DataLayout & getDataLayout() const
LLVMContext & getContext() const
VPBasicBlock * getEntry()
bool hasScalableVF() const
VPValue * getTripCount() const
The trip count of the original loop.
VPSymbolicValue & getVFxUF()
Returns VF * UF of the vector loop region.
auto getLiveIns() const
Return the list of live-in VPValues available in the VPlan.
bool hasUF(unsigned UF) const
VPIRValue * getPoison(Type *Ty)
Return a VPIRValue wrapping a poison value of type Ty.
VPSymbolicValue & getVectorTripCount()
The vector trip count.
VPIRValue * getOrAddLiveIn(Value *V)
Gets the live-in VPIRValue for V or adds a new live-in (if none exists yet) for V.
VPIRValue * getZero(Type *Ty)
Return a VPIRValue wrapping the null value of type Ty.
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
unsigned getConcreteUF() const
Returns the concrete UF of the plan, after unrolling.
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
VPBasicBlock * createVPBasicBlock(const Twine &Name, VPRecipeBase *Recipe=nullptr)
Create a new VPBasicBlock with Name and containing Recipe if present.
VPBasicBlock * getVectorPreheader() const
Returns the preheader of the vector loop region, if one exists, or null otherwise.
bool hasScalarVFOnly() const
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
bool hasTailFolded() const
Returns true if the vector loop region is tail-folded.
VPSymbolicValue & getVF()
Returns the VF of the vector loop region.
bool hasScalarTail() const
Returns true if the scalar tail may execute after the vector loop, i.e.
VPIRValue * getConstantInt(Type *Ty, uint64_t Val, bool IsSigned=false)
Return a VPIRValue wrapping a ConstantInt with the given type and value.
LLVM Value Representation.
iterator_range< user_iterator > users()
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
RecipeTy * findSingleUser(VPValue *V, const MatchT &P)
Return the sole user of V matching pattern P, casting it to RecipeTy, or nullptr.
VPInstruction_match< VPInstruction::BranchOnTwoConds > m_BranchOnTwoConds()
canonical_widen_iv_match m_CanonicalWidenIV()
VPInstruction_match< VPInstruction::BranchOnCount > m_BranchOnCount()
auto m_VPValue()
Match an arbitrary VPValue and ignore it.
match_bind< VPInstruction > m_VPInstruction(VPInstruction *&V)
Match a VPInstruction, capturing if we match.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used by all users.
VPValue * findIncomingAliasMask(const VPlan &Plan)
Finds the incoming alias-mask within the vector preheader.
VPScalarIVStepsRecipe * createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, Type *ResultTy, VPValue *StartV, VPValue *Step, DebugLoc DL, VPBuilder &Builder, const VPIRFlags::WrapFlagsTy &Flags={})
Create a scalar-iv-steps recipe over Plan's canonical IV for an induction of Kind with InductionOpcod...
bool onlyScalarValuesUsed(const VPValue *Def)
Returns true if only scalar values of Def are used by all users.
VPValue * scalarizeVPWidenPointerInduction(VPWidenPointerInductionRecipe *PtrIV, VPlan &Plan, VPBuilder &Builder)
Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd (IndStart,...
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
VPBuilderBase<> VPBuilder
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
auto make_isa_range(RangeT &&Range)
Return a range over Range containing only elements for which isa<T> holds, casting each of them to T.
auto dyn_cast_or_null(const Y &Val)
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
@ Mul
Product of integers.
DWARFExpression::Operation Op
auto make_second_range(ContainerTy &&c)
Given a container of pairs, return a range over the second elements.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
A struct that represents some properties of the register usage of a loop.
SmallMapVector< unsigned, unsigned, 4 > MaxLocalUsers
Holds the maximum number of concurrent live intervals in the loop.
InstructionCost spillCost(const TargetTransformInfo &TTI, TargetTransformInfo::TargetCostKind CostKind, unsigned OverrideMaxNumRegs=0) const
Calculate the estimated cost of any spills due to using more registers than the number available for ...