160#define LV_NAME "loop-vectorize"
161#define DEBUG_TYPE LV_NAME
171 "llvm.loop.vectorize.followup_vectorized";
173 "llvm.loop.vectorize.followup_epilogue";
176STATISTIC(LoopsVectorized,
"Number of loops vectorized");
177STATISTIC(LoopsAnalyzed,
"Number of loops analyzed for vectorization");
178STATISTIC(LoopsEpilogueVectorized,
"Number of epilogues vectorized");
182 cl::desc(
"Enable vectorization of epilogue loops."));
186 cl::desc(
"When epilogue vectorization is enabled, and a value greater than "
187 "1 is specified, forces the given VF for all applicable epilogue "
192 cl::desc(
"Only loops with vectorization factor equal to or larger than "
193 "the specified value are considered for epilogue vectorization."));
199 cl::desc(
"Loops with a constant trip count that is smaller than this "
200 "value are vectorized only if no scalar iteration overheads "
205 cl::desc(
"The maximum allowed number of runtime memory checks"));
221 "prefer-predicate-over-epilogue",
224 cl::desc(
"Tail-folding and predication preferences over creating a scalar "
228 "Don't tail-predicate loops, create scalar epilogue"),
230 "predicate-else-scalar-epilogue",
231 "prefer tail-folding, create scalar epilogue if tail "
234 "predicate-dont-vectorize",
235 "prefers tail-folding, don't attempt vectorization if "
236 "tail-folding fails.")));
239 "force-tail-folding-style",
cl::desc(
"Force the tail folding style"),
242 clEnumValN(TailFoldingStyle::None,
"none",
"Disable tail folding"),
244 TailFoldingStyle::Data,
"data",
245 "Create lane mask for data only, using active.lane.mask intrinsic"),
246 clEnumValN(TailFoldingStyle::DataWithoutLaneMask,
247 "data-without-lane-mask",
248 "Create lane mask with compare/stepvector"),
249 clEnumValN(TailFoldingStyle::DataAndControlFlow,
"data-and-control",
250 "Create lane mask using active.lane.mask intrinsic, and use "
251 "it for both data and control flow"),
252 clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck,
253 "data-and-control-without-rt-check",
254 "Similar to data-and-control, but remove the runtime check"),
255 clEnumValN(TailFoldingStyle::DataWithEVL,
"data-with-evl",
256 "Use predicated EVL instructions for tail folding. If EVL "
257 "is unsupported, fallback to data-without-lane-mask.")));
261 cl::desc(
"Maximize bandwidth when selecting vectorization factor which "
262 "will be determined by the smallest type in loop."));
266 cl::desc(
"Enable vectorization on interleaved memory accesses in a loop"));
272 cl::desc(
"Enable vectorization on masked interleaved memory accesses in a loop"));
276 cl::desc(
"A flag that overrides the target's number of scalar registers."));
280 cl::desc(
"A flag that overrides the target's number of vector registers."));
284 cl::desc(
"A flag that overrides the target's max interleave factor for "
289 cl::desc(
"A flag that overrides the target's max interleave factor for "
290 "vectorized loops."));
294 cl::desc(
"A flag that overrides the target's expected cost for "
295 "an instruction to a single constant value. Mostly "
296 "useful for getting consistent testing."));
301 "Pretend that scalable vectors are supported, even if the target does "
302 "not support them. This flag should only be used for testing."));
307 "The cost of a loop that is considered 'small' by the interleaver."));
311 cl::desc(
"Enable the use of the block frequency analysis to access PGO "
312 "heuristics minimizing code growth in cold regions and being more "
313 "aggressive in hot regions."));
319 "Enable runtime interleaving until load/store ports are saturated"));
324 cl::desc(
"Max number of stores to be predicated behind an if."));
328 cl::desc(
"Count the induction variable only once when interleaving"));
332 cl::desc(
"Enable if predication of stores during vectorization."));
336 cl::desc(
"The maximum interleave count to use when interleaving a scalar "
337 "reduction in a nested loop."));
342 cl::desc(
"Prefer in-loop vector reductions, "
343 "overriding the targets preference."));
347 cl::desc(
"Enable the vectorisation of loops with in-order (strict) "
353 "Prefer predicating a reduction operation over an after loop select."));
358 cl::desc(
"Enable VPlan-native vectorization path with "
359 "support for outer loop vectorization."));
369 "Build VPlan for every supported loop nest in the function and bail "
370 "out right after the build (stress test the VPlan H-CFG construction "
371 "in the VPlan-native vectorization path)."));
375 cl::desc(
"Enable loop interleaving in Loop vectorization passes"));
378 cl::desc(
"Run the Loop vectorization passes"));
382 cl::desc(
"Use dot format instead of plain text when dumping VPlans"));
385 "force-widen-divrem-via-safe-divisor",
cl::Hidden,
387 "Override cost based safe divisor widening for div/rem instructions"));
390 "vectorizer-maximize-bandwidth-for-vector-calls",
cl::init(
true),
392 cl::desc(
"Try wider VFs if they enable the use of vector variants"));
411 return DL.getTypeAllocSizeInBits(Ty) !=
DL.getTypeSizeInBits(Ty);
450 unsigned Factor = Vals.
size();
451 assert(Factor > 1 &&
"Tried to interleave invalid number of vectors");
455 for (
Value *Val : Vals)
456 assert(Val->getType() == VecTy &&
"Tried to interleave mismatched types");
461 if (VecTy->isScalableTy()) {
462 VectorType *WideVecTy = VectorType::getDoubleElementsVectorType(VecTy);
464 WideVecTy, Intrinsic::experimental_vector_interleave2, Vals,
472 const unsigned NumElts = VecTy->getElementCount().getFixedValue();
479class GeneratedRTChecks;
523 this->MinProfitableTripCount = VecWidth;
539 virtual std::pair<BasicBlock *, Value *>
566 VPValue *BlockInMask,
bool NeedsMaskForGaps);
581 std::pair<BasicBlock *, Value *> AdditionalBypass = {
nullptr,
nullptr});
651 const SCEV2ValueTy &ExpandedSCEVs,
652 std::pair<BasicBlock *, Value *> AdditionalBypass = {
nullptr,
nullptr});
799 "A high UF for the epilogue loop is likely not beneficial.");
819 GeneratedRTChecks &Checks)
821 EPI.MainLoopVF,
EPI.MainLoopVF,
EPI.MainLoopUF, LVL,
828 const SCEV2ValueTy &ExpandedSCEVs)
final {
835 virtual std::pair<BasicBlock *, Value *>
859 GeneratedRTChecks &Check)
864 std::pair<BasicBlock *, Value *>
888 GeneratedRTChecks &Checks)
895 std::pair<BasicBlock *, Value *>
917 if (
I->getDebugLoc() !=
Empty)
918 return I->getDebugLoc();
920 for (
Use &
Op :
I->operands()) {
922 if (OpInst->getDebugLoc() !=
Empty)
923 return OpInst->getDebugLoc();
926 return I->getDebugLoc();
935 dbgs() <<
"LV: " << Prefix << DebugMsg;
957 CodeRegion =
I->getParent();
960 if (
I->getDebugLoc())
961 DL =
I->getDebugLoc();
978 return B.CreateElementCount(Ty, VF);
984 assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&
"Invalid loop count");
998 <<
"loop not vectorized: " << OREMsg);
1016 "Vectorizing: ", TheLoop->
isInnermost() ?
"innermost loop" :
"outer loop",
1022 <<
"vectorized " << LoopType <<
"loop (vectorization width: "
1024 <<
", interleaved count: " <<
ore::NV(
"InterleaveCount", IC) <<
")";
1036 if (
const DebugLoc LoopDbgLoc = L->getStartLoc())
1037 LoopDbgLoc.print(
OS);
1040 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
1185 "Profitable to scalarize relevant only for VF > 1.");
1188 "cost-model should not be used for outer loops (in VPlan-native path)");
1190 auto Scalars = InstsToScalarize.find(VF);
1191 assert(Scalars != InstsToScalarize.end() &&
1192 "VF not yet analyzed for scalarization profitability");
1193 return Scalars->second.contains(
I);
1200 "cost-model should not be used for outer loops (in VPlan-native path)");
1204 if (isa<PseudoProbeInst>(
I))
1210 auto UniformsPerVF = Uniforms.find(VF);
1211 assert(UniformsPerVF != Uniforms.end() &&
1212 "VF not yet analyzed for uniformity");
1213 return UniformsPerVF->second.count(
I);
1220 "cost-model should not be used for outer loops (in VPlan-native path)");
1224 auto ScalarsPerVF = Scalars.find(VF);
1225 assert(ScalarsPerVF != Scalars.end() &&
1226 "Scalar values are not calculated for VF");
1227 return ScalarsPerVF->second.count(
I);
1233 return VF.
isVector() && MinBWs.contains(
I) &&
1255 WideningDecisions[std::make_pair(
I, VF)] = std::make_pair(W,
Cost);
1266 for (
unsigned i = 0; i < Grp->
getFactor(); ++i) {
1269 WideningDecisions[std::make_pair(
I, VF)] = std::make_pair(W,
Cost);
1271 WideningDecisions[std::make_pair(
I, VF)] = std::make_pair(W, 0);
1283 "cost-model should not be used for outer loops (in VPlan-native path)");
1285 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(
I, VF);
1286 auto Itr = WideningDecisions.
find(InstOnVF);
1287 if (Itr == WideningDecisions.
end())
1289 return Itr->second.first;
1296 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(
I, VF);
1298 "The cost is not calculated");
1299 return WideningDecisions[InstOnVF].second;
1312 std::optional<unsigned> MaskPos,
1315 CallWideningDecisions[std::make_pair(CI, VF)] = {Kind, Variant, IID,
1322 return CallWideningDecisions.
at(std::make_pair(CI, VF));
1330 auto *Trunc = dyn_cast<TruncInst>(
I);
1343 Value *
Op = Trunc->getOperand(0);
1363 if (VF.
isScalar() || Uniforms.contains(VF))
1367 collectLoopUniforms(VF);
1368 collectLoopScalars(VF);
1388 bool LI = isa<LoadInst>(V);
1389 bool SI = isa<StoreInst>(V);
1404 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1405 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1416 return ScalarCost < SafeDivisorCost;
1440 std::pair<InstructionCost, InstructionCost>
1481 auto RequiresScalarEpilogue = [
this](
ElementCount VF) {
1484 bool IsRequired =
all_of(Range, RequiresScalarEpilogue);
1486 (IsRequired ||
none_of(Range, RequiresScalarEpilogue)) &&
1487 "all VFs in range must agree on whether a scalar epilogue is required");
1499 if (!ChosenTailFoldingStyle)
1501 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1502 : ChosenTailFoldingStyle->second;
1510 assert(!ChosenTailFoldingStyle &&
"Tail folding must not be selected yet.");
1512 ChosenTailFoldingStyle =
1518 ChosenTailFoldingStyle = std::make_pair(
1533 IsScalableVF && UserIC <= 1 &&
1544 ChosenTailFoldingStyle =
1549 <<
"LV: Preference for VP intrinsics indicated. Will "
1550 "not try to generate VP Intrinsics "
1552 ?
"since interleave count specified is greater than 1.\n"
1553 :
"due to non-interleaving reasons.\n"));
1578 [](
const std::pair<std::pair<Instruction *, ElementCount>,
1579 std::pair<InstWidening, InstructionCost>>
1585 return InLoopReductions.contains(Phi);
1600 WideningDecisions.
clear();
1601 CallWideningDecisions.
clear();
1631 unsigned NumPredStores = 0;
1640 bool FoldTailByMasking);
1645 ElementCount getMaximizedVFForTarget(
unsigned MaxTripCount,
1646 unsigned SmallestType,
1647 unsigned WidestType,
1649 bool FoldTailByMasking);
1653 ElementCount getMaxLegalScalableVF(
unsigned MaxSafeElements);
1666 std::optional<InstructionCost>
1714 PredicatedBBsAfterVectorization;
1727 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1728 ChosenTailFoldingStyle;
1762 ScalarCostsTy &ScalarCosts,
1788 std::pair<InstWidening, InstructionCost>>;
1790 DecisionList WideningDecisions;
1792 using CallDecisionList =
1795 CallDecisionList CallWideningDecisions;
1818 Ops, [
this, VF](
Value *V) {
return this->needsExtract(V, VF); }));
1876class GeneratedRTChecks {
1882 Value *SCEVCheckCond =
nullptr;
1890 Value *MemRuntimeCheckCond =
nullptr;
1899 bool CostTooHigh =
false;
1900 const bool AddBranchWeights;
1902 Loop *OuterLoop =
nullptr;
1907 bool AddBranchWeights)
1908 : DT(DT), LI(LI),
TTI(
TTI), SCEVExp(SE,
DL,
"scev.check"),
1909 MemCheckExp(SE,
DL,
"scev.check"), AddBranchWeights(AddBranchWeights) {}
1937 nullptr,
"vector.scevcheck");
1944 if (RtPtrChecking.Need) {
1945 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1946 MemCheckBlock =
SplitBlock(Pred, Pred->getTerminator(), DT, LI,
nullptr,
1949 auto DiffChecks = RtPtrChecking.getDiffChecks();
1951 Value *RuntimeVF =
nullptr;
1956 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1962 MemCheckBlock->
getTerminator(), L, RtPtrChecking.getChecks(),
1965 assert(MemRuntimeCheckCond &&
1966 "no RT checks generated although RtPtrChecking "
1967 "claimed checks are required");
1970 if (!MemCheckBlock && !SCEVCheckBlock)
1980 if (SCEVCheckBlock) {
1985 if (MemCheckBlock) {
1992 if (MemCheckBlock) {
1996 if (SCEVCheckBlock) {
2002 OuterLoop =
L->getParentLoop();
2006 if (SCEVCheckBlock || MemCheckBlock)
2019 if (SCEVCheckBlock->getTerminator() == &
I)
2026 if (MemCheckBlock) {
2029 if (MemCheckBlock->getTerminator() == &
I)
2052 unsigned BestTripCount = 2;
2056 BestTripCount = SmallTC;
2060 BestTripCount = *EstimatedTC;
2063 BestTripCount = std::max(BestTripCount, 1U);
2067 NewMemCheckCost = std::max(*NewMemCheckCost.
getValue(),
2070 if (BestTripCount > 1)
2072 <<
"We expect runtime memory checks to be hoisted "
2073 <<
"out of the outer loop. Cost reduced from "
2074 << MemCheckCost <<
" to " << NewMemCheckCost <<
'\n');
2076 MemCheckCost = NewMemCheckCost;
2080 RTCheckCost += MemCheckCost;
2083 if (SCEVCheckBlock || MemCheckBlock)
2084 LLVM_DEBUG(
dbgs() <<
"Total cost of runtime checks: " << RTCheckCost
2092 ~GeneratedRTChecks() {
2096 SCEVCleaner.markResultUsed();
2098 if (!MemRuntimeCheckCond)
2099 MemCheckCleaner.markResultUsed();
2101 if (MemRuntimeCheckCond) {
2102 auto &SE = *MemCheckExp.
getSE();
2109 I.eraseFromParent();
2112 MemCheckCleaner.cleanup();
2113 SCEVCleaner.cleanup();
2116 SCEVCheckBlock->eraseFromParent();
2117 if (MemRuntimeCheckCond)
2118 MemCheckBlock->eraseFromParent();
2132 SCEVCheckCond =
nullptr;
2133 if (
auto *
C = dyn_cast<ConstantInt>(
Cond))
2144 SCEVCheckBlock->getTerminator()->eraseFromParent();
2145 SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
2146 Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
2153 if (AddBranchWeights)
2156 return SCEVCheckBlock;
2165 if (!MemRuntimeCheckCond)
2174 MemCheckBlock->moveBefore(LoopVectorPreHeader);
2181 if (AddBranchWeights) {
2185 MemCheckBlock->getTerminator()->setDebugLoc(
2186 Pred->getTerminator()->getDebugLoc());
2189 MemRuntimeCheckCond =
nullptr;
2190 return MemCheckBlock;
2196 return Style == TailFoldingStyle::Data ||
2197 Style == TailFoldingStyle::DataAndControlFlow ||
2198 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
2202 return Style == TailFoldingStyle::DataAndControlFlow ||
2203 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
2233 LLVM_DEBUG(
dbgs() <<
"LV: Loop hints prevent outer loop vectorization.\n");
2239 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Interleave is not supported for "
2259 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
2269 for (
Loop *InnerL : L)
2291 ?
B.CreateSExtOrTrunc(
Index, StepTy)
2292 :
B.CreateCast(Instruction::SIToFP,
Index, StepTy);
2293 if (CastedIndex !=
Index) {
2295 Index = CastedIndex;
2305 assert(
X->getType() ==
Y->getType() &&
"Types don't match!");
2306 if (
auto *CX = dyn_cast<ConstantInt>(
X))
2309 if (
auto *CY = dyn_cast<ConstantInt>(
Y))
2312 return B.CreateAdd(
X,
Y);
2318 assert(
X->getType()->getScalarType() ==
Y->getType() &&
2319 "Types don't match!");
2320 if (
auto *CX = dyn_cast<ConstantInt>(
X))
2323 if (
auto *CY = dyn_cast<ConstantInt>(
Y))
2326 VectorType *XVTy = dyn_cast<VectorType>(
X->getType());
2327 if (XVTy && !isa<VectorType>(
Y->getType()))
2328 Y =
B.CreateVectorSplat(XVTy->getElementCount(),
Y);
2329 return B.CreateMul(
X,
Y);
2332 switch (InductionKind) {
2335 "Vector indices not supported for integer inductions yet");
2337 "Index type does not match StartValue type");
2338 if (isa<ConstantInt>(Step) && cast<ConstantInt>(Step)->isMinusOne())
2339 return B.CreateSub(StartValue,
Index);
2347 "Vector indices not supported for FP inductions yet");
2350 (InductionBinOp->
getOpcode() == Instruction::FAdd ||
2351 InductionBinOp->
getOpcode() == Instruction::FSub) &&
2352 "Original bin op should be defined for FP induction");
2355 return B.CreateBinOp(InductionBinOp->
getOpcode(), StartValue, MulExp,
2369 if (
F.hasFnAttribute(Attribute::VScaleRange))
2370 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2372 return std::nullopt;
2381 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2383 unsigned MaxUF = UF ? *UF :
Cost->TTI.getMaxInterleaveFactor(VF);
2385 Type *IdxTy =
Cost->Legal->getWidestInductionType();
2386 APInt MaxUIntTripCount = cast<IntegerType>(IdxTy)->getMask();
2392 Cost->PSE.getSE()->getSmallConstantMaxTripCount(
Cost->TheLoop)) {
2395 std::optional<unsigned> MaxVScale =
2399 MaxVF *= *MaxVScale;
2402 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2450 VPValue *BlockInMask,
bool NeedsMaskForGaps) {
2452 const DataLayout &
DL = Instr->getModule()->getDataLayout();
2456 unsigned InterleaveFactor = Group->
getFactor();
2465 "Reversed masked interleave-group not supported.");
2483 for (
unsigned Part = 0; Part <
UF; Part++) {
2485 if (
auto *
I = dyn_cast<Instruction>(AddrPart))
2500 bool InBounds =
false;
2502 InBounds =
gep->isInBounds();
2510 auto CreateGroupMask = [
this, &BlockInMask, &State, &InterleaveFactor](
2511 unsigned Part,
Value *MaskForGaps) ->
Value * {
2513 assert(!MaskForGaps &&
"Interleaved groups with gaps are not supported.");
2514 assert(InterleaveFactor == 2 &&
2515 "Unsupported deinterleave factor for scalable vectors");
2516 auto *BlockInMaskPart = State.
get(BlockInMask, Part);
2521 MaskTy, Intrinsic::experimental_vector_interleave2, Ops,
2522 nullptr,
"interleaved.mask");
2528 Value *BlockInMaskPart = State.
get(BlockInMask, Part);
2532 "interleaved.mask");
2539 if (isa<LoadInst>(Instr)) {
2540 Value *MaskForGaps =
nullptr;
2541 if (NeedsMaskForGaps) {
2544 assert(MaskForGaps &&
"Mask for Gaps is required but it is null");
2549 for (
unsigned Part = 0; Part <
UF; Part++) {
2551 if (BlockInMask || MaskForGaps) {
2553 "masked interleaved groups are not allowed.");
2554 Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
2557 GroupMask, PoisonVec,
"wide.masked.vec");
2566 if (VecTy->isScalableTy()) {
2567 assert(InterleaveFactor == 2 &&
2568 "Unsupported deinterleave factor for scalable vectors");
2570 for (
unsigned Part = 0; Part <
UF; ++Part) {
2574 Intrinsic::experimental_vector_deinterleave2, VecTy, NewLoads[Part],
2575 nullptr,
"strided.vec");
2577 for (
unsigned I = 0;
I < InterleaveFactor; ++
I) {
2585 if (Member->getType() != ScalarTy) {
2593 State.
set(VPDefs[J], StridedVec, Part);
2604 for (
unsigned I = 0;
I < InterleaveFactor; ++
I) {
2613 for (
unsigned Part = 0; Part <
UF; Part++) {
2615 NewLoads[Part], StrideMask,
"strided.vec");
2618 if (Member->getType() != ScalarTy) {
2627 State.
set(VPDefs[J], StridedVec, Part);
2638 Value *MaskForGaps =
2641 "masked interleaved groups are not allowed.");
2643 "masking gaps for scalable vectors is not yet supported.");
2644 for (
unsigned Part = 0; Part <
UF; Part++) {
2647 unsigned StoredIdx = 0;
2648 for (
unsigned i = 0; i < InterleaveFactor; i++) {
2650 "Fail to get a member from an interleaved store group");
2660 Value *StoredVec = State.
get(StoredValues[StoredIdx], Part);
2668 if (StoredVec->
getType() != SubVT)
2677 if (BlockInMask || MaskForGaps) {
2678 Value *GroupMask = CreateGroupMask(Part, MaskForGaps);
2693 assert(!Instr->getType()->isAggregateType() &&
"Can't handle vectors");
2697 if (isa<NoAliasScopeDeclInst>(Instr))
2702 bool IsVoidRetTy = Instr->getType()->isVoidTy();
2706 Cloned->
setName(Instr->getName() +
".cloned");
2711 "inferred type and type from generated instructions do not match");
2717 if (
auto DL = Instr->getDebugLoc())
2723 auto InputInstance = Instance;
2727 Cloned->
setOperand(
I.index(), State.
get(Operand, InputInstance));
2734 State.
set(RepRecipe, Cloned, Instance);
2737 if (
auto *II = dyn_cast<AssumeInst>(Cloned))
2742 if (IfPredicateInstr)
2766 if (
Cost->foldTailByMasking()) {
2768 "VF*UF must be a power of 2 when folding tail by masking");
2800 auto *DstFVTy = cast<VectorType>(DstVTy);
2801 auto VF = DstFVTy->getElementCount();
2802 auto *SrcVecTy = cast<VectorType>(V->getType());
2803 assert(
VF == SrcVecTy->getElementCount() &&
"Vector dimensions do not match");
2804 Type *SrcElemTy = SrcVecTy->getElementType();
2805 Type *DstElemTy = DstFVTy->getElementType();
2806 assert((
DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) &&
2807 "Vector elements must have same size");
2818 "Only one type should be a pointer type");
2820 "Only one type should be a floating point type");
2846 auto CreateStep = [&]() ->
Value * {
2871 Value *MaxUIntTripCount =
2872 ConstantInt::get(CountTy, cast<IntegerType>(CountTy)->getMask());
2886 "TC check is expected to dominate Bypass");
2907 if (!SCEVCheckBlock)
2913 "Cannot SCEV check stride or overflow when optimizing for size");
2928 return SCEVCheckBlock;
2947 "Cannot emit memory checks when optimizing for size, unless forced "
2953 <<
"Code-size may be reduced by not forcing "
2954 "vectorization, or by source-code modifications "
2955 "eliminating the need for runtime checks "
2956 "(e.g., adding 'restrict').";
2964 return MemCheckBlock;
2973 "multiple exit loop without required epilogue?");
2977 LI,
nullptr,
Twine(Prefix) +
"middle.block");
2980 nullptr,
Twine(Prefix) +
"scalar.ph");
2997 BrInst->
setDebugLoc(ScalarLatchTerm->getDebugLoc());
3013 std::pair<BasicBlock *, Value *> AdditionalBypass) {
3019 Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
3020 if (OrigPhi == OldInduction) {
3035 if (AdditionalBypass.first) {
3036 B.SetInsertPoint(AdditionalBypass.first,
3037 AdditionalBypass.first->getFirstInsertionPt());
3038 EndValueFromAdditionalBypass =
3041 EndValueFromAdditionalBypass->
setName(
"ind.end");
3062 if (AdditionalBypass.first)
3064 EndValueFromAdditionalBypass);
3071 const SCEV2ValueTy &ExpandedSCEVs) {
3072 const SCEV *Step =
ID.getStep();
3073 if (
auto *
C = dyn_cast<SCEVConstant>(Step))
3074 return C->getValue();
3075 if (
auto *U = dyn_cast<SCEVUnknown>(Step))
3076 return U->getValue();
3077 auto I = ExpandedSCEVs.find(Step);
3078 assert(
I != ExpandedSCEVs.end() &&
"SCEV must be expanded at this point");
3083 const SCEV2ValueTy &ExpandedSCEVs,
3084 std::pair<BasicBlock *, Value *> AdditionalBypass) {
3085 assert(((AdditionalBypass.first && AdditionalBypass.second) ||
3086 (!AdditionalBypass.first && !AdditionalBypass.second)) &&
3087 "Inconsistent information about additional bypass.");
3096 PHINode *OrigPhi = InductionEntry.first;
3121 !
Cost->foldTailByMasking()) {
3130 B.SetCurrentDebugLocation(ScalarLatchTerm->getDebugLoc());
3143#ifdef EXPENSIVE_CHECKS
3150std::pair<BasicBlock *, Value *>
3152 const SCEV2ValueTy &ExpandedSCEVs) {
3237 assert(isa<PHINode>(UI) &&
"Expected LCSSA form");
3238 MissingVals[UI] = EndValue;
3246 auto *UI = cast<Instruction>(U);
3248 assert(isa<PHINode>(UI) &&
"Expected LCSSA form");
3255 Value *CountMinusOne =
B.CreateSub(
3257 CountMinusOne->
setName(
"cmo");
3260 assert(StepVPV &&
"step must have been expanded during VPlan execution");
3262 : State.
get(StepVPV, {0, 0});
3266 Escape->
setName(
"ind.escape");
3267 MissingVals[UI] = Escape;
3271 for (
auto &
I : MissingVals) {
3278 if (
PHI->getBasicBlockIndex(MiddleBlock) == -1) {
3279 PHI->addIncoming(
I.second, MiddleBlock);
3287struct CSEDenseMapInfo {
3289 return isa<InsertElementInst>(
I) || isa<ExtractElementInst>(
I) ||
3290 isa<ShuffleVectorInst>(
I) || isa<GetElementPtrInst>(
I);
3302 assert(canHandle(
I) &&
"Unknown instruction!");
3304 I->value_op_end()));
3308 if (
LHS == getEmptyKey() ||
RHS == getEmptyKey() ||
3309 LHS == getTombstoneKey() ||
RHS == getTombstoneKey())
3311 return LHS->isIdenticalTo(
RHS);
3322 if (!CSEDenseMapInfo::canHandle(&In))
3328 In.replaceAllUsesWith(V);
3329 In.eraseFromParent();
3343 return CallWideningDecisions.at(std::make_pair(CI, VF)).Cost;
3348 if (
auto RedCost = getReductionPatternCost(CI, VF,
RetTy,
CostKind))
3352 for (
auto &ArgOp : CI->
args())
3361 return std::min(ScalarCallCost, IntrinsicCost);
3363 return ScalarCallCost;
3376 assert(
ID &&
"Expected intrinsic call!");
3379 if (
auto *FPMO = dyn_cast<FPMathOperator>(CI))
3380 FMF = FPMO->getFastMathFlags();
3386 std::back_inserter(ParamTys),
3387 [&](
Type *Ty) { return MaybeVectorizeType(Ty, VF); });
3390 dyn_cast<IntrinsicInst>(CI));
3396 auto *I1 = cast<IntegerType>(cast<VectorType>(
T1)->getElementType());
3397 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3398 return I1->getBitWidth() < I2->getBitWidth() ?
T1 : T2;
3402 auto *I1 = cast<IntegerType>(cast<VectorType>(
T1)->getElementType());
3403 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3404 return I1->getBitWidth() > I2->getBitWidth() ?
T1 : T2;
3422 if (
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
3436 for (
PHINode &PN : Exit->phis())
3466 KV.second->fixPhi(Plan, State);
3549 Value *RuntimeVF =
nullptr;
3551 auto *One = ConstantInt::get(IdxTy, 1);
3559 auto RecurSplice = cast<VPInstruction>(*PhiR->
user_begin());
3561 RecurSplice->getOpcode() ==
3563 "recurrence phi must have a single user: FirstOrderRecurrenceSplice");
3565 for (
VPUser *U : RecurSplice->users())
3566 if (
auto *LiveOut = dyn_cast<VPLiveOut>(U))
3569 if (!LiveOuts.
empty()) {
3575 Value *ExtractForPhiUsedOutsideLoop =
nullptr;
3581 assert(
UF > 1 &&
"VF and UF cannot both be 1");
3586 ExtractForPhiUsedOutsideLoop = State.
get(PreviousDef,
UF - 2);
3591 PHINode *LCSSAPhi = LiveOut->getPhi();
3608 Phi->setName(
"scalar.recur");
3625 auto isBlockOfUsePredicated = [&](
Use &U) ->
bool {
3626 auto *
I = cast<Instruction>(U.getUser());
3628 if (
auto *Phi = dyn_cast<PHINode>(
I))
3629 BB = Phi->getIncomingBlock(
3631 return BB == PredBB;
3642 Worklist.
insert(InstsToReanalyze.
begin(), InstsToReanalyze.
end());
3643 InstsToReanalyze.
clear();
3646 while (!Worklist.
empty()) {
3652 if (!
I || isa<PHINode>(
I) || !VectorLoop->contains(
I) ||
3653 I->mayHaveSideEffects() ||
I->mayReadFromMemory())
3661 if (
I->getParent() == PredBB) {
3662 Worklist.
insert(
I->op_begin(),
I->op_end());
3676 I->moveBefore(&*PredBB->getFirstInsertionPt());
3677 Worklist.
insert(
I->op_begin(),
I->op_end());
3689 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
3694 PHINode *NewPhi = cast<PHINode>(State.
get(VPPhi, 0));
3706void LoopVectorizationCostModel::collectLoopScalars(
ElementCount VF) {
3711 "This function should not be visited twice for the same VF");
3717 Scalars[VF].
insert(Uniforms[VF].begin(), Uniforms[VF].end());
3736 "Widening decision should be ready at this moment");
3737 if (
auto *Store = dyn_cast<StoreInst>(MemAccess))
3738 if (
Ptr == Store->getValueOperand())
3741 "Ptr is neither a value or pointer operand");
3747 auto isLoopVaryingBitCastOrGEP = [&](
Value *
V) {
3748 return ((isa<BitCastInst>(V) &&
V->getType()->isPointerTy()) ||
3749 isa<GetElementPtrInst>(V)) &&
3760 if (!isLoopVaryingBitCastOrGEP(
Ptr))
3765 auto *
I = cast<Instruction>(
Ptr);
3773 return isa<LoadInst>(U) || isa<StoreInst>(U);
3777 PossibleNonScalarPtrs.
insert(
I);
3795 for (
auto &
I : *BB) {
3796 if (
auto *Load = dyn_cast<LoadInst>(&
I)) {
3797 evaluatePtrUse(Load,
Load->getPointerOperand());
3798 }
else if (
auto *Store = dyn_cast<StoreInst>(&
I)) {
3799 evaluatePtrUse(Store,
Store->getPointerOperand());
3800 evaluatePtrUse(Store,
Store->getValueOperand());
3803 for (
auto *
I : ScalarPtrs)
3804 if (!PossibleNonScalarPtrs.
count(
I)) {
3812 auto ForcedScalar = ForcedScalars.
find(VF);
3813 if (ForcedScalar != ForcedScalars.
end())
3814 for (
auto *
I : ForcedScalar->second) {
3815 LLVM_DEBUG(
dbgs() <<
"LV: Found (forced) scalar instruction: " << *
I <<
"\n");
3824 while (
Idx != Worklist.
size()) {
3826 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
3828 auto *Src = cast<Instruction>(Dst->getOperand(0));
3830 auto *J = cast<Instruction>(U);
3831 return !TheLoop->contains(J) || Worklist.count(J) ||
3832 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
3833 isScalarUse(J, Src));
3836 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Src <<
"\n");
3843 auto *Ind = Induction.first;
3844 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
3853 auto IsDirectLoadStoreFromPtrIndvar = [&](
Instruction *Indvar,
3855 return Induction.second.getKind() ==
3857 (isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
3864 auto *I = cast<Instruction>(U);
3865 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3866 IsDirectLoadStoreFromPtrIndvar(Ind, I);
3873 auto ScalarIndUpdate =
3875 auto *I = cast<Instruction>(U);
3876 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
3877 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
3879 if (!ScalarIndUpdate)
3884 Worklist.
insert(IndUpdate);
3885 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Ind <<
"\n");
3886 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *IndUpdate
3900 switch(
I->getOpcode()) {
3903 case Instruction::Call:
3906 return CallWideningDecisions.at(std::make_pair(cast<CallInst>(
I), VF))
3908 case Instruction::Load:
3909 case Instruction::Store: {
3921 case Instruction::UDiv:
3922 case Instruction::SDiv:
3923 case Instruction::SRem:
3924 case Instruction::URem: {
3940 switch(
I->getOpcode()) {
3943 case Instruction::Load:
3944 case Instruction::Store: {
3957 (isa<LoadInst>(
I) ||
3958 (isa<StoreInst>(
I) &&
3964 case Instruction::UDiv:
3965 case Instruction::SDiv:
3966 case Instruction::SRem:
3967 case Instruction::URem:
3971 case Instruction::Call:
3976std::pair<InstructionCost, InstructionCost>
3979 assert(
I->getOpcode() == Instruction::UDiv ||
3980 I->getOpcode() == Instruction::SDiv ||
3981 I->getOpcode() == Instruction::SRem ||
3982 I->getOpcode() == Instruction::URem);
3993 ScalarizationCost = 0;
4008 ScalarizationCost += getScalarizationOverhead(
I, VF,
CostKind);
4022 Instruction::Select, VecTy,
4028 Value *Op2 =
I->getOperand(1);
4037 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4039 return {ScalarizationCost, SafeDivisorCost};
4046 "Decision should not be set yet.");
4048 assert(Group &&
"Must have a group.");
4052 auto &
DL =
I->getModule()->getDataLayout();
4059 unsigned InterleaveFactor = Group->getFactor();
4060 bool ScalarNI =
DL.isNonIntegralPointerType(ScalarTy);
4061 for (
unsigned i = 0; i < InterleaveFactor; i++) {
4066 bool MemberNI =
DL.isNonIntegralPointerType(
MemberTy);
4068 if (MemberNI != ScalarNI) {
4071 }
else if (MemberNI && ScalarNI &&
4072 ScalarTy->getPointerAddressSpace() !=
4073 MemberTy->getPointerAddressSpace()) {
4083 bool PredicatedAccessRequiresMasking =
4086 bool LoadAccessWithGapsRequiresEpilogMasking =
4087 isa<LoadInst>(
I) && Group->requiresScalarEpilogue() &&
4089 bool StoreAccessWithGapsRequiresMasking =
4090 isa<StoreInst>(
I) && (Group->getNumMembers() < Group->getFactor());
4091 if (!PredicatedAccessRequiresMasking &&
4092 !LoadAccessWithGapsRequiresEpilogMasking &&
4093 !StoreAccessWithGapsRequiresMasking)
4100 "Masked interleave-groups for predicated accesses are not enabled.");
4102 if (Group->isReverse())
4114 assert((isa<LoadInst, StoreInst>(
I)) &&
"Invalid memory instruction");
4130 auto &
DL =
I->getModule()->getDataLayout();
4137void LoopVectorizationCostModel::collectLoopUniforms(
ElementCount VF) {
4144 "This function should not be visited twice for the same VF");
4148 Uniforms[VF].
clear();
4156 auto isOutOfScope = [&](
Value *V) ->
bool {
4170 auto addToWorklistIfAllowed = [&](
Instruction *
I) ->
void {
4171 if (isOutOfScope(
I)) {
4177 LLVM_DEBUG(
dbgs() <<
"LV: Found not uniform being ScalarWithPredication: "
4181 LLVM_DEBUG(
dbgs() <<
"LV: Found uniform instruction: " << *
I <<
"\n");
4190 addToWorklistIfAllowed(Cmp);
4198 if (PrevVF.isVector()) {
4199 auto Iter = Uniforms.
find(PrevVF);
4200 if (Iter != Uniforms.
end() && !Iter->second.contains(
I))
4205 if (isa<LoadInst>(
I))
4216 "Widening decision should be ready at this moment");
4218 if (isUniformMemOpUse(
I))
4221 return (WideningDecision ==
CM_Widen ||
4230 if (isa<StoreInst>(
I) &&
I->getOperand(0) ==
Ptr)
4246 for (
auto &
I : *BB) {
4248 switch (II->getIntrinsicID()) {
4249 case Intrinsic::sideeffect:
4250 case Intrinsic::experimental_noalias_scope_decl:
4251 case Intrinsic::assume:
4252 case Intrinsic::lifetime_start:
4253 case Intrinsic::lifetime_end:
4255 addToWorklistIfAllowed(&
I);
4264 if (
auto *EVI = dyn_cast<ExtractValueInst>(&
I)) {
4265 assert(isOutOfScope(EVI->getAggregateOperand()) &&
4266 "Expected aggregate value to be loop invariant");
4267 addToWorklistIfAllowed(EVI);
4276 if (isUniformMemOpUse(&
I))
4277 addToWorklistIfAllowed(&
I);
4279 if (isVectorizedMemAccessUse(&
I,
Ptr))
4286 for (
auto *V : HasUniformUse) {
4287 if (isOutOfScope(V))
4289 auto *
I = cast<Instruction>(V);
4290 auto UsersAreMemAccesses =
4292 return isVectorizedMemAccessUse(cast<Instruction>(U), V);
4294 if (UsersAreMemAccesses)
4295 addToWorklistIfAllowed(
I);
4302 while (idx != Worklist.
size()) {
4305 for (
auto *OV :
I->operand_values()) {
4307 if (isOutOfScope(OV))
4311 auto *
OP = dyn_cast<PHINode>(OV);
4316 auto *OI = cast<Instruction>(OV);
4318 auto *J = cast<Instruction>(U);
4319 return Worklist.count(J) || isVectorizedMemAccessUse(J, OI);
4321 addToWorklistIfAllowed(OI);
4332 auto *Ind = Induction.first;
4333 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4338 auto *I = cast<Instruction>(U);
4339 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
4340 isVectorizedMemAccessUse(I, Ind);
4347 auto UniformIndUpdate =
4349 auto *I = cast<Instruction>(U);
4350 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
4351 isVectorizedMemAccessUse(I, IndUpdate);
4353 if (!UniformIndUpdate)
4357 addToWorklistIfAllowed(Ind);
4358 addToWorklistIfAllowed(IndUpdate);
4369 "runtime pointer checks needed. Enable vectorization of this "
4370 "loop with '#pragma clang loop vectorize(enable)' when "
4371 "compiling with -Os/-Oz",
4372 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
4378 "runtime SCEV checks needed. Enable vectorization of this "
4379 "loop with '#pragma clang loop vectorize(enable)' when "
4380 "compiling with -Os/-Oz",
4381 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
4388 "runtime stride == 1 checks needed. Enable vectorization of "
4389 "this loop without such check by compiling with -Os/-Oz",
4390 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
4398LoopVectorizationCostModel::getMaxLegalScalableVF(
unsigned MaxSafeElements) {
4404 "ScalableVectorizationDisabled",
ORE,
TheLoop);
4408 LLVM_DEBUG(
dbgs() <<
"LV: Scalable vectorization is available\n");
4411 std::numeric_limits<ElementCount::ScalarTy>::max());
4422 "Scalable vectorization not supported for the reduction "
4423 "operations found in this loop.",
4435 "for all element types found in this loop.",
4441 return MaxScalableVF;
4451 "Max legal vector width too small, scalable vectorization "
4455 return MaxScalableVF;
4459 unsigned MaxTripCount,
ElementCount UserVF,
bool FoldTailByMasking) {
4461 unsigned SmallestType, WidestType;
4468 unsigned MaxSafeElements =
4472 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElements);
4474 LLVM_DEBUG(
dbgs() <<
"LV: The max safe fixed VF is: " << MaxSafeFixedVF
4476 LLVM_DEBUG(
dbgs() <<
"LV: The max safe scalable VF is: " << MaxSafeScalableVF
4481 auto MaxSafeUserVF =
4482 UserVF.
isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
4499 <<
" is unsafe, clamping to max safe VF="
4500 << MaxSafeFixedVF <<
".\n");
4505 <<
"User-specified vectorization factor "
4506 <<
ore::NV(
"UserVectorizationFactor", UserVF)
4507 <<
" is unsafe, clamping to maximum safe vectorization factor "
4508 <<
ore::NV(
"VectorizationFactor", MaxSafeFixedVF);
4510 return MaxSafeFixedVF;
4515 <<
" is ignored because scalable vectors are not "
4521 <<
"User-specified vectorization factor "
4522 <<
ore::NV(
"UserVectorizationFactor", UserVF)
4523 <<
" is ignored because the target does not support scalable "
4524 "vectors. The compiler will pick a more suitable value.";
4528 <<
" is unsafe. Ignoring scalable UserVF.\n");
4533 <<
"User-specified vectorization factor "
4534 <<
ore::NV(
"UserVectorizationFactor", UserVF)
4535 <<
" is unsafe. Ignoring the hint to let the compiler pick a "
4536 "more suitable value.";
4541 LLVM_DEBUG(
dbgs() <<
"LV: The Smallest and Widest types: " << SmallestType
4542 <<
" / " << WidestType <<
" bits.\n");
4547 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
4548 MaxSafeFixedVF, FoldTailByMasking))
4552 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
4553 MaxSafeScalableVF, FoldTailByMasking))
4554 if (MaxVF.isScalable()) {
4555 Result.ScalableVF = MaxVF;
4556 LLVM_DEBUG(
dbgs() <<
"LV: Found feasible scalable VF = " << MaxVF
4569 "Not inserting runtime ptr check for divergent target",
4570 "runtime pointer checks needed. Not enabled for divergent target",
4571 "CantVersionLoopWithDivergentTarget",
ORE,
TheLoop);
4580 "loop trip count is one, irrelevant for vectorization",
4585 switch (ScalarEpilogueStatus) {
4587 return computeFeasibleMaxVF(MaxTC, UserVF,
false);
4592 dbgs() <<
"LV: vector predicate hint/switch found.\n"
4593 <<
"LV: Not allowing scalar epilogue, creating predicated "
4594 <<
"vector loop.\n");
4601 dbgs() <<
"LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
4603 LLVM_DEBUG(
dbgs() <<
"LV: Not allowing scalar epilogue due to low trip "
4622 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with a "
4623 "scalar epilogue instead.\n");
4625 return computeFeasibleMaxVF(MaxTC, UserVF,
false);
4636 "No decisions should have been taken at this point");
4646 std::optional<unsigned> MaxPowerOf2RuntimeVF =
4651 MaxPowerOf2RuntimeVF = std::max<unsigned>(
4652 *MaxPowerOf2RuntimeVF,
4655 MaxPowerOf2RuntimeVF = std::nullopt;
4658 if (MaxPowerOf2RuntimeVF && *MaxPowerOf2RuntimeVF > 0) {
4660 "MaxFixedVF must be a power of 2");
4661 unsigned MaxVFtimesIC =
4662 UserIC ? *MaxPowerOf2RuntimeVF * UserIC : *MaxPowerOf2RuntimeVF;
4666 BackedgeTakenCount, SE->
getOne(BackedgeTakenCount->
getType()));
4672 LLVM_DEBUG(
dbgs() <<
"LV: No tail will remain for any chosen VF.\n");
4686 <<
"LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
4687 "try to generate VP Intrinsics with scalable vector "
4693 "Expected scalable vector factor.");
4703 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with a "
4704 "scalar epilogue instead.\n");
4710 LLVM_DEBUG(
dbgs() <<
"LV: Can't fold tail by masking: don't vectorize\n");
4716 "Unable to calculate the loop count due to complex control flow",
4717 "unable to calculate the loop count due to complex control flow",
4723 "Cannot optimize for size and vectorize at the same time.",
4724 "cannot optimize for size and vectorize at the same time. "
4725 "Enable vectorization of this loop with '#pragma clang loop "
4726 "vectorize(enable)' when compiling with -Os/-Oz",
4731ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
4732 unsigned MaxTripCount,
unsigned SmallestType,
unsigned WidestType,
4734 bool ComputeScalableMaxVF = MaxSafeVF.
isScalable();
4742 "Scalable flags must match");
4750 ComputeScalableMaxVF);
4751 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
4753 << (MaxVectorElementCount * WidestType) <<
" bits.\n");
4755 if (!MaxVectorElementCount) {
4757 << (ComputeScalableMaxVF ?
"scalable" :
"fixed")
4758 <<
" vector registers.\n");
4762 unsigned WidestRegisterMinEC = MaxVectorElementCount.getKnownMinValue();
4763 if (MaxVectorElementCount.isScalable() &&
4767 WidestRegisterMinEC *= Min;
4776 if (MaxTripCount && MaxTripCount <= WidestRegisterMinEC &&
4784 LLVM_DEBUG(
dbgs() <<
"LV: Clamping the MaxVF to maximum power of two not "
4785 "exceeding the constant trip count: "
4786 << ClampedUpperTripCount <<
"\n");
4788 ClampedUpperTripCount,
4789 FoldTailByMasking ? MaxVectorElementCount.isScalable() :
false);
4802 ComputeScalableMaxVF);
4803 MaxVectorElementCountMaxBW = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
4817 for (
int i = RUs.size() - 1; i >= 0; --i) {
4818 bool Selected =
true;
4819 for (
auto &pair : RUs[i].MaxLocalUsers) {
4821 if (pair.second > TargetNumRegisters)
4833 <<
") with target's minimum: " << MinVF <<
'\n');
4849static std::optional<unsigned>
4851 const Function *Fn = L->getHeader()->getParent();
4855 auto Max = Attr.getVScaleRangeMax();
4856 if (Max && Min == Max)
4863bool LoopVectorizationPlanner::isMoreProfitable(
4870 if (!
A.Width.isScalable() && !
B.Width.isScalable() && MaxTripCount) {
4879 auto GetCostForTC = [MaxTripCount,
this](
unsigned VF,
4883 : VectorCost * (MaxTripCount / VF) +
4884 ScalarCost * (MaxTripCount % VF);
4886 auto RTCostA = GetCostForTC(
A.Width.getFixedValue(), CostA,
A.ScalarCost);
4887 auto RTCostB = GetCostForTC(
B.Width.getFixedValue(), CostB,
B.ScalarCost);
4889 return RTCostA < RTCostB;
4893 unsigned EstimatedWidthA =
A.Width.getKnownMinValue();
4894 unsigned EstimatedWidthB =
B.Width.getKnownMinValue();
4896 if (
A.Width.isScalable())
4897 EstimatedWidthA *= *VScale;
4898 if (
B.Width.isScalable())
4899 EstimatedWidthB *= *VScale;
4905 if (
A.Width.isScalable() && !
B.Width.isScalable())
4906 return (CostA *
B.Width.getFixedValue()) <= (CostB * EstimatedWidthA);
4911 return (CostA * EstimatedWidthB) < (CostB * EstimatedWidthA);
4917 if (InvalidCosts.
empty())
4924 std::map<Instruction *, unsigned> Numbering;
4926 for (
auto &Pair : InvalidCosts)
4927 if (!Numbering.count(Pair.first))
4928 Numbering[Pair.first] =
I++;
4932 if (Numbering[
A.first] != Numbering[
B.first])
4933 return Numbering[
A.first] < Numbering[
B.first];
4935 return ECC(
A.second,
B.second);
4947 Subset =
Tail.take_front(1);
4956 if (Subset ==
Tail ||
Tail[Subset.size()].first !=
I) {
4957 std::string OutString;
4959 assert(!Subset.empty() &&
"Unexpected empty range");
4960 OS <<
"Instruction with invalid costs prevented vectorization at VF=(";
4961 for (
const auto &Pair : Subset)
4962 OS << (Pair.second == Subset.front().second ?
"" :
", ") << Pair.second;
4964 if (
auto *CI = dyn_cast<CallInst>(
I))
4965 OS <<
" call to " << CI->getCalledFunction()->getName();
4967 OS <<
" " <<
I->getOpcodeName();
4970 Tail =
Tail.drop_front(Subset.size());
4974 Subset =
Tail.take_front(Subset.size() + 1);
4975 }
while (!
Tail.empty());
4982 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ExpectedCost <<
".\n");
4983 assert(ExpectedCost.
isValid() &&
"Unexpected invalid cost for scalar loop");
4985 "Expected Scalar VF to be a candidate");
4992 if (ForceVectorization && VFCandidates.
size() > 1) {
5000 for (
const auto &i : VFCandidates) {
5010 unsigned AssumedMinimumVscale =
5013 Candidate.Width.isScalable()
5014 ? Candidate.Width.getKnownMinValue() * AssumedMinimumVscale
5015 : Candidate.Width.getFixedValue();
5017 <<
" costs: " << (Candidate.Cost / Width));
5020 << AssumedMinimumVscale <<
")");
5024 if (!
C.second && !ForceVectorization) {
5026 dbgs() <<
"LV: Not considering vector loop of width " << i
5027 <<
" because it will not generate any vector instructions.\n");
5032 if (isMoreProfitable(Candidate, ScalarCost))
5033 ProfitableVFs.push_back(Candidate);
5035 if (isMoreProfitable(Candidate, ChosenFactor))
5036 ChosenFactor = Candidate;
5043 "There are conditional stores.",
5044 "store that is conditionally executed prevents vectorization",
5045 "ConditionalStore", ORE, OrigLoop);
5046 ChosenFactor = ScalarCost;
5050 !isMoreProfitable(ChosenFactor, ScalarCost))
dbgs()
5051 <<
"LV: Vectorization seems to be not beneficial, "
5052 <<
"but was forced by a user.\n");
5054 return ChosenFactor;
5057bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
5062 [&](
PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
5070 Entry.first->getIncomingValueForBlock(OrigLoop->
getLoopLatch());
5072 if (!OrigLoop->
contains(cast<Instruction>(U)))
5075 for (
User *U : Entry.first->users())
5076 if (!OrigLoop->
contains(cast<Instruction>(U)))
5105 unsigned Multiplier = 1;
5117 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is disabled.\n");
5122 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because no "
5123 "epilogue is allowed.\n");
5129 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
5130 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because the loop "
5131 "is not a supported candidate.\n");
5136 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization factor is forced.\n");
5139 return {ForcedEC, 0, 0};
5141 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization forced factor is not "
5150 dbgs() <<
"LEV: Epilogue vectorization skipped due to opt for size.\n");
5155 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is not profitable for "
5167 EstimatedRuntimeVF *= *VScale;
5172 const SCEV *RemainingIterations =
nullptr;
5173 for (
auto &NextVF : ProfitableVFs) {
5180 if ((!NextVF.Width.isScalable() && MainLoopVF.
isScalable() &&
5187 if (!MainLoopVF.
isScalable() && !NextVF.Width.isScalable()) {
5189 if (!RemainingIterations) {
5196 SE.
getConstant(TCType, NextVF.Width.getKnownMinValue()),
5197 RemainingIterations))
5201 if (Result.Width.isScalar() || isMoreProfitable(NextVF, Result))
5207 << Result.Width <<
"\n");
5211std::pair<unsigned, unsigned>
5213 unsigned MinWidth = -1U;
5214 unsigned MaxWidth = 8;
5227 MaxWidth = std::min<unsigned>(
5228 MaxWidth, std::min<unsigned>(
5234 MinWidth = std::min<unsigned>(
5235 MinWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
5236 MaxWidth = std::max<unsigned>(
5237 MaxWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
5240 return {MinWidth, MaxWidth};
5248 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5256 if (!isa<LoadInst>(
I) && !isa<StoreInst>(
I) && !isa<PHINode>(
I))
5261 if (
auto *PN = dyn_cast<PHINode>(&
I)) {
5275 if (
auto *ST = dyn_cast<StoreInst>(&
I))
5276 T = ST->getValueOperand()->getType();
5279 "Expected the load/store/recurrence type to be sized");
5308 LLVM_DEBUG(
dbgs() <<
"LV: Preference for VP intrinsics indicated. "
5309 "Unroll factor forced to be 1.\n");
5322 if (LoopCost == 0) {
5324 assert(LoopCost.
isValid() &&
"Expected to have chosen a VF with valid cost");
5334 for (
auto& pair : R.MaxLocalUsers) {
5335 pair.second = std::max(pair.second, 1U);
5349 unsigned IC = UINT_MAX;
5351 for (
auto& pair : R.MaxLocalUsers) {
5363 unsigned MaxLocalUsers = pair.second;
5364 unsigned LoopInvariantRegs = 0;
5365 if (R.LoopInvariantRegs.find(pair.first) != R.LoopInvariantRegs.end())
5366 LoopInvariantRegs = R.LoopInvariantRegs[pair.first];
5368 unsigned TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
5372 TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
5373 std::max(1U, (MaxLocalUsers - 1)));
5376 IC = std::min(IC, TmpIC);
5394 EstimatedVF *= *VScale;
5396 assert(EstimatedVF >= 1 &&
"Estimated VF shouldn't be less than 1");
5402 unsigned AvailableTC =
5414 std::max(1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
5415 unsigned InterleaveCountLB =
bit_floor(std::max(
5416 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
5417 MaxInterleaveCount = InterleaveCountLB;
5419 if (InterleaveCountUB != InterleaveCountLB) {
5420 unsigned TailTripCountUB =
5421 (AvailableTC % (EstimatedVF * InterleaveCountUB));
5422 unsigned TailTripCountLB =
5423 (AvailableTC % (EstimatedVF * InterleaveCountLB));
5426 if (TailTripCountUB == TailTripCountLB)
5427 MaxInterleaveCount = InterleaveCountUB;
5429 }
else if (BestKnownTC && *BestKnownTC > 0) {
5433 ? (*BestKnownTC) - 1
5441 MaxInterleaveCount =
bit_floor(std::max(
5442 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
5445 assert(MaxInterleaveCount > 0 &&
5446 "Maximum interleave count must be greater than 0");
5450 if (IC > MaxInterleaveCount)
5451 IC = MaxInterleaveCount;
5454 IC = std::max(1u, IC);
5456 assert(IC > 0 &&
"Interleave count must be greater than 0.");
5460 if (VF.
isVector() && HasReductions) {
5461 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving because of reductions.\n");
5469 bool ScalarInterleavingRequiresPredication =
5471 return Legal->blockNeedsPredication(BB);
5473 bool ScalarInterleavingRequiresRuntimePointerCheck =
5479 <<
"LV: IC is " << IC <<
'\n'
5480 <<
"LV: VF is " << VF <<
'\n');
5481 const bool AggressivelyInterleaveReductions =
5483 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
5484 !ScalarInterleavingRequiresPredication && LoopCost <
SmallLoopCost) {
5488 unsigned SmallIC = std::min(IC, (
unsigned)llvm::bit_floor<uint64_t>(
5495 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
5496 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
5502 bool HasSelectCmpReductions =
5505 const RecurrenceDescriptor &RdxDesc = Reduction.second;
5506 return RecurrenceDescriptor::isAnyOfRecurrenceKind(
5507 RdxDesc.getRecurrenceKind());
5509 if (HasSelectCmpReductions) {
5510 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving select-cmp reductions.\n");
5520 bool HasOrderedReductions =
5522 const RecurrenceDescriptor &RdxDesc = Reduction.second;
5523 return RdxDesc.isOrdered();
5525 if (HasOrderedReductions) {
5527 dbgs() <<
"LV: Not interleaving scalar ordered reductions.\n");
5532 SmallIC = std::min(SmallIC,
F);
5533 StoresIC = std::min(StoresIC,
F);
5534 LoadsIC = std::min(LoadsIC,
F);
5538 std::max(StoresIC, LoadsIC) > SmallIC) {
5540 dbgs() <<
"LV: Interleaving to saturate store or load ports.\n");
5541 return std::max(StoresIC, LoadsIC);
5546 if (VF.
isScalar() && AggressivelyInterleaveReductions) {
5550 return std::max(IC / 2, SmallIC);
5552 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving to reduce branch cost.\n");
5559 if (AggressivelyInterleaveReductions) {
5609 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5613 for (
Value *U :
I.operands()) {
5614 auto *Instr = dyn_cast<Instruction>(U);
5625 LoopInvariants.
insert(Instr);
5630 EndPoint[Instr] = IdxToInstr.
size();
5648 LLVM_DEBUG(
dbgs() <<
"LV(REG): Calculating max register usage:\n");
5650 const auto &TTICapture =
TTI;
5657 for (
unsigned int i = 0, s = IdxToInstr.
size(); i < s; ++i) {
5661 InstrList &
List = TransposeEnds[i];
5676 for (
unsigned j = 0, e = VFs.
size(); j < e; ++j) {
5684 if (VFs[j].isScalar()) {
5685 for (
auto *Inst : OpenIntervals) {
5694 for (
auto *Inst : OpenIntervals) {
5707 RegUsage[ClassID] += GetRegUsage(Inst->getType(), VFs[j]);
5713 auto &Entry = MaxUsages[j][pair.first];
5714 Entry = std::max(Entry, pair.second);
5719 << OpenIntervals.
size() <<
'\n');
5725 for (
unsigned i = 0, e = VFs.
size(); i < e; ++i) {
5731 for (
auto *Inst : LoopInvariants) {
5734 bool IsScalar =
all_of(Inst->users(), [&](
User *U) {
5735 auto *I = cast<Instruction>(U);
5736 return TheLoop != LI->getLoopFor(I->getParent()) ||
5737 isScalarAfterVectorization(I, VFs[i]);
5743 Invariant[ClassID] += GetRegUsage(Inst->getType(), VF);
5747 dbgs() <<
"LV(REG): VF = " << VFs[i] <<
'\n';
5748 dbgs() <<
"LV(REG): Found max usage: " << MaxUsages[i].
size()
5750 for (
const auto &pair : MaxUsages[i]) {
5751 dbgs() <<
"LV(REG): RegisterClass: "
5755 dbgs() <<
"LV(REG): Found invariant usage: " << Invariant.
size()
5757 for (
const auto &pair : Invariant) {
5758 dbgs() <<
"LV(REG): RegisterClass: "
5772bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
Instruction *
I,
5783 "Expecting a scalar emulated instruction");
5784 return isa<LoadInst>(
I) ||
5785 (isa<StoreInst>(
I) &&
5802 PredicatedBBsAfterVectorization[VF].
clear();
5817 if (!VF.
isScalable() && !useEmulatedMaskMemRefHack(&
I, VF) &&
5818 computePredInstDiscount(&
I, ScalarCosts, VF) >= 0)
5821 PredicatedBBsAfterVectorization[VF].
insert(BB);
5829 "Instruction marked uniform-after-vectorization will be predicated");
5847 if (!
I->hasOneUse() || PredInst->
getParent() !=
I->getParent() ||
5866 for (
Use &U :
I->operands())
5867 if (
auto *J = dyn_cast<Instruction>(U.get()))
5879 while (!Worklist.
empty()) {
5883 if (ScalarCosts.contains(
I))
5914 for (
Use &U :
I->operands())
5915 if (
auto *J = dyn_cast<Instruction>(
U.get())) {
5917 "Instruction has non-scalar type");
5918 if (canBeScalarized(J))
5920 else if (needsExtract(J, VF)) {
5922 cast<VectorType>(
ToVectorTy(J->getType(), VF)),
5933 Discount += VectorCost - ScalarCost;
5934 ScalarCosts[
I] = ScalarCost;
5950 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5959 if (
C.first.isValid() &&
5967 BlockCost.first +=
C.first;
5968 BlockCost.second |=
C.second;
5970 <<
" for VF " << VF <<
" For instruction: " <<
I
5984 Cost.first += BlockCost.first;
5985 Cost.second |= BlockCost.second;
6000 const Loop *TheLoop) {
6002 auto *Gep = dyn_cast<GetElementPtrInst>(
Ptr);
6008 auto SE = PSE.
getSE();
6009 unsigned NumOperands = Gep->getNumOperands();
6010 for (
unsigned i = 1; i < NumOperands; ++i) {
6011 Value *Opd = Gep->getOperand(i);
6013 !
Legal->isInductionVariable(Opd))
6022LoopVectorizationCostModel::getMemInstScalarizationCost(
Instruction *
I,
6025 "Scalarization cost of instruction implies vectorization.");
6072 if (useEmulatedMaskMemRefHack(
I, VF))
6082LoopVectorizationCostModel::getConsecutiveMemOpCost(
Instruction *
I,
6085 auto *VectorTy = cast<VectorType>(
ToVectorTy(ValTy, VF));
6091 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
6092 "Stride should be 1 or -1 for consecutive memory access");
6104 bool Reverse = ConsecutiveStride < 0;
6112LoopVectorizationCostModel::getUniformMemOpCost(
Instruction *
I,
6117 auto *VectorTy = cast<VectorType>(
ToVectorTy(ValTy, VF));
6121 if (isa<LoadInst>(
I)) {
6133 (isLoopInvariantStoreValue
6140LoopVectorizationCostModel::getGatherScatterCost(
Instruction *
I,
6143 auto *VectorTy = cast<VectorType>(
ToVectorTy(ValTy, VF));
6154LoopVectorizationCostModel::getInterleaveGroupCost(
Instruction *
I,
6157 auto *VectorTy = cast<VectorType>(
ToVectorTy(ValTy, VF));
6162 assert(Group &&
"Fail to get an interleaved access group.");
6164 unsigned InterleaveFactor = Group->getFactor();
6169 for (
unsigned IF = 0;
IF < InterleaveFactor;
IF++)
6170 if (Group->getMember(IF))
6174 bool UseMaskForGaps =
6176 (isa<StoreInst>(
I) && (Group->getNumMembers() < Group->getFactor()));
6178 I->getOpcode(), WideVecTy, Group->getFactor(), Indices, Group->getAlign(),
6181 if (Group->isReverse()) {
6184 "Reverse masked interleaved access not supported.");
6185 Cost += Group->getNumMembers() *
6192std::optional<InstructionCost>
6193LoopVectorizationCostModel::getReductionPatternCost(
6198 if (InLoopReductions.
empty() || VF.
isScalar() || !isa<VectorType>(Ty))
6199 return std::nullopt;
6200 auto *VectorTy = cast<VectorType>(Ty);
6217 return std::nullopt;
6228 if (!InLoopReductionImmediateChains.
count(RetI))
6229 return std::nullopt;
6233 Instruction *LastChain = InLoopReductionImmediateChains.
at(RetI);
6235 while (!isa<PHINode>(ReductionPhi))
6236 ReductionPhi = InLoopReductionImmediateChains.
at(ReductionPhi);
6265 if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
6278 bool IsUnsigned = isa<ZExtInst>(Op0);
6295 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
6296 return I == RetI ? RedCost : 0;
6300 bool IsUnsigned = isa<ZExtInst>(RedOp);
6309 if (RedCost.
isValid() && RedCost < BaseCost + ExtCost)
6310 return I == RetI ? RedCost : 0;
6311 }
else if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
6316 bool IsUnsigned = isa<ZExtInst>(Op0);
6339 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
6340 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
6348 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
6349 return I == RetI ? RedCost : 0;
6358 if (RedCost.
isValid() && RedCost < MulCost + BaseCost)
6359 return I == RetI ? RedCost : 0;
6363 return I == RetI ? std::optional<InstructionCost>(BaseCost) :
std::nullopt;
6367LoopVectorizationCostModel::getMemoryInstructionCost(
Instruction *
I,
6385LoopVectorizationCostModel::getInstructionCost(
Instruction *
I,
6396 auto ForcedScalar = ForcedScalars.
find(VF);
6397 if (VF.
isVector() && ForcedScalar != ForcedScalars.
end()) {
6398 auto InstSet = ForcedScalar->second;
6399 if (InstSet.count(
I))
6409 bool TypeNotScalarized =
false;
6440 if (!
RetTy->isVoidTy() &&
6462 for (
auto *V : filterExtractingOperands(Ops, VF))
6465 filterExtractingOperands(Ops, VF), Tys,
CostKind);
6487 auto isLegalToScalarize = [&]() {
6501 if (isa<LoadInst>(
I))
6506 auto &SI = cast<StoreInst>(
I);
6524 if (GatherScatterCost < ScalarizationCost)
6536 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
6537 "Expected consecutive stride.");
6546 unsigned NumAccesses = 1;
6549 assert(Group &&
"Fail to get an interleaved access group.");
6555 NumAccesses = Group->getNumMembers();
6557 InterleaveCost = getInterleaveGroupCost(&
I, VF);
6562 ? getGatherScatterCost(&
I, VF) * NumAccesses
6566 getMemInstScalarizationCost(&
I, VF) * NumAccesses;
6572 if (InterleaveCost <= GatherScatterCost &&
6573 InterleaveCost < ScalarizationCost) {
6575 Cost = InterleaveCost;
6576 }
else if (GatherScatterCost < ScalarizationCost) {
6578 Cost = GatherScatterCost;
6581 Cost = ScalarizationCost;
6615 while (!Worklist.
empty()) {
6617 for (
auto &
Op :
I->operands())
6618 if (
auto *InstOp = dyn_cast<Instruction>(
Op))
6619 if ((InstOp->getParent() ==
I->getParent()) && !isa<PHINode>(InstOp) &&
6620 AddrDefs.
insert(InstOp).second)
6624 for (
auto *
I : AddrDefs) {
6625 if (isa<LoadInst>(
I)) {
6639 for (
unsigned I = 0;
I < Group->getFactor(); ++
I) {
6656 "Trying to set a vectorization decision for a scalar VF");
6675 for (
auto &ArgOp : CI->
args())
6680 for (
Type *ScalarTy : ScalarTys)
6686 if (
auto RedCost = getReductionPatternCost(CI, VF,
RetTy,
CostKind)) {
6689 std::nullopt, *RedCost);
6703 getScalarizationOverhead(CI, VF,
CostKind);
6709 bool UsesMask =
false;
6715 if (
Info.Shape.VF != VF)
6719 if (MaskRequired && !
Info.isMasked())
6723 bool ParamsOk =
true;
6725 switch (Param.ParamKind) {
6744 dyn_cast<SCEVAddRecExpr>(SE->
getSCEV(ScalarParam));
6746 if (!SAR || SAR->getLoop() !=
TheLoop) {
6752 dyn_cast<SCEVConstant>(SAR->getStepRecurrence(*SE));
6780 if (VecFunc && UsesMask && !MaskRequired)
6800 if (VectorCost <=
Cost) {
6805 if (IntrinsicCost <=
Cost) {
6806 Cost = IntrinsicCost;
6825 auto hasSingleCopyAfterVectorization = [
this](
Instruction *
I,
6830 auto Scalarized = InstsToScalarize.
find(VF);
6831 assert(Scalarized != InstsToScalarize.
end() &&
6832 "VF not yet analyzed for scalarization profitability");
6833 return !Scalarized->second.count(
I) &&
6835 auto *UI = cast<Instruction>(U);
6836 return !Scalarized->second.count(UI);
6839 (void) hasSingleCopyAfterVectorization;
6847 assert(
I->getOpcode() == Instruction::GetElementPtr ||
6848 I->getOpcode() == Instruction::PHI ||
6849 (
I->getOpcode() == Instruction::BitCast &&
6850 I->getType()->isPointerTy()) ||
6851 hasSingleCopyAfterVectorization(
I, VF));
6857 switch (
I->getOpcode()) {
6858 case Instruction::GetElementPtr:
6864 case Instruction::Br: {
6868 bool ScalarPredicatedBB =
false;
6871 (PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(0)) ||
6872 PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(1))))
6873 ScalarPredicatedBB =
true;
6875 if (ScalarPredicatedBB) {
6897 case Instruction::PHI: {
6898 auto *
Phi = cast<PHINode>(
I);
6905 cast<VectorType>(VectorTy), Mask,
CostKind,
6913 return (
Phi->getNumIncomingValues() - 1) *
6921 case Instruction::UDiv:
6922 case Instruction::SDiv:
6923 case Instruction::URem:
6924 case Instruction::SRem:
6928 ScalarCost : SafeDivisorCost;
6932 case Instruction::Add:
6933 case Instruction::FAdd:
6934 case Instruction::Sub:
6935 case Instruction::FSub:
6936 case Instruction::Mul:
6937 case Instruction::FMul:
6938 case Instruction::FDiv:
6939 case Instruction::FRem:
6940 case Instruction::Shl:
6941 case Instruction::LShr:
6942 case Instruction::AShr:
6943 case Instruction::And:
6944 case Instruction::Or:
6945 case Instruction::Xor: {
6949 if (
I->getOpcode() == Instruction::Mul &&
6955 if (
auto RedCost = getReductionPatternCost(
I, VF, VectorTy,
CostKind))
6960 Value *Op2 =
I->getOperand(1);
6969 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6972 case Instruction::FNeg: {
6975 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6976 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6977 I->getOperand(0),
I);
6979 case Instruction::Select: {
6984 const Value *Op0, *Op1;
7001 Type *CondTy =
SI->getCondition()->getType();
7006 if (
auto *Cmp = dyn_cast<CmpInst>(
SI->getCondition()))
7007 Pred =
Cmp->getPredicate();
7011 case Instruction::ICmp:
7012 case Instruction::FCmp: {
7013 Type *ValTy =
I->getOperand(0)->getType();
7014 Instruction *Op0AsInstruction = dyn_cast<Instruction>(
I->getOperand(0));
7019 cast<CmpInst>(
I)->getPredicate(),
CostKind,
7022 case Instruction::Store:
7023 case Instruction::Load: {
7028 "CM decision should be taken at this point");
7035 return getMemoryInstructionCost(
I, VF);
7037 case Instruction::BitCast:
7038 if (
I->getType()->isPointerTy())
7041 case Instruction::ZExt:
7042 case Instruction::SExt:
7043 case Instruction::FPToUI:
7044 case Instruction::FPToSI:
7045 case Instruction::FPExt:
7046 case Instruction::PtrToInt:
7047 case Instruction::IntToPtr:
7048 case Instruction::SIToFP:
7049 case Instruction::UIToFP:
7050 case Instruction::Trunc:
7051 case Instruction::FPTrunc: {
7054 assert((isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
7055 "Expected a load or a store!");
7081 unsigned Opcode =
I->getOpcode();
7084 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
7086 if (
StoreInst *Store = dyn_cast<StoreInst>(*
I->user_begin()))
7087 CCH = ComputeCCH(Store);
7090 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
7091 Opcode == Instruction::FPExt) {
7092 if (
LoadInst *Load = dyn_cast<LoadInst>(
I->getOperand(0)))
7093 CCH = ComputeCCH(Load);
7100 auto *Trunc = cast<TruncInst>(
I);
7102 Trunc->getSrcTy(), CCH,
CostKind, Trunc);
7106 if (
auto RedCost = getReductionPatternCost(
I, VF, VectorTy,
CostKind))
7109 Type *SrcScalarTy =
I->getOperand(0)->getType();
7118 Type *MinVecTy = VectorTy;
7119 if (Opcode == Instruction::Trunc) {
7123 }
else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
7133 case Instruction::Call:
7135 case Instruction::ExtractValue:
7137 case Instruction::Alloca:
7158 if ((SI = dyn_cast<StoreInst>(&
I)) &&
7201 bool InLoop = !ReductionOperations.
empty();
7204 InLoopReductions.
insert(Phi);
7207 for (
auto *
I : ReductionOperations) {
7208 InLoopReductionImmediateChains[
I] = LastChain;
7212 LLVM_DEBUG(
dbgs() <<
"LV: Using " << (InLoop ?
"inloop" :
"out of loop")
7213 <<
" reduction for phi: " << *Phi <<
"\n");
7221 return tryInsertInstruction(
7234 unsigned WidestType;
7243 unsigned N =
RegSize.getKnownMinValue() / WidestType;
7264 <<
"overriding computed VF.\n");
7269 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing. Scalable VF requested, but "
7270 <<
"not supported by the target.\n");
7272 "Scalable vectorization requested but not supported by the target",
7273 "the scalable user-specified vectorization width for outer-loop "
7274 "vectorization cannot be used because the target does not support "
7275 "scalable vectors.",
7276 "ScalableVFUnfeasible", ORE, OrigLoop);
7281 "VF needs to be a power of two");
7283 <<
"VF " << VF <<
" to build VPlans.\n");
7290 return {VF, 0 , 0 };
7294 dbgs() <<
"LV: Not vectorizing. Inner loops aren't supported in the "
7295 "VPlan-native path.\n");
7299std::optional<VectorizationFactor>
7307 return std::nullopt;
7314 <<
"LV: Invalidate all interleaved groups due to fold-tail by masking "
7315 "which requires masked-interleaved support.\n");
7326 if (!UserVF.
isZero() && UserVFIsLegal) {
7328 "VF needs to be a power of two");
7334 buildVPlansWithVPRecipes(UserVF, UserVF);
7336 LLVM_DEBUG(
dbgs() <<
"LV: No VPlan could be built for " << UserVF
7338 return std::nullopt;
7342 return {{UserVF, 0, 0}};
7345 "InvalidCost", ORE, OrigLoop);
7358 for (
const auto &VF : VFCandidates) {
7381 return std::nullopt;
7388 [VF](
const VPlanPtr &Plan) {
return Plan->hasVF(VF); }) ==
7390 "Best VF has not a single VPlan.");
7392 for (
const VPlanPtr &Plan : VPlans) {
7393 if (Plan->hasVF(VF))
7403 bool IsUnrollMetadata =
false;
7404 MDNode *LoopID = L->getLoopID();
7407 for (
unsigned i = 1, ie = LoopID->
getNumOperands(); i < ie; ++i) {
7408 auto *MD = dyn_cast<MDNode>(LoopID->
getOperand(i));
7410 const auto *S = dyn_cast<MDString>(MD->getOperand(0));
7412 S && S->getString().starts_with(
"llvm.loop.unroll.disable");
7418 if (!IsUnrollMetadata) {
7429 L->setLoopID(NewLoopID);
7443 auto *PhiR = cast<VPReductionPHIRecipe>(RedResult->
getOperand(0));
7450 dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
7466 BCBlockPhi->addIncoming(FinalValue,
Incoming);
7468 BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(
Incoming),
7471 BCBlockPhi->addIncoming(ReductionStartValue,
Incoming);
7474 auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
7478 int IncomingEdgeBlockIdx =
7480 assert(IncomingEdgeBlockIdx >= 0 &&
"Invalid block index");
7482 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
7483 OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
7485 OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
7487 ReductionResumeValues[&RdxDesc] = BCBlockPhi;
7490std::pair<DenseMap<const SCEV *, Value *>,
7497 "Trying to execute plan with unsupported VF");
7499 "Trying to execute plan with unsupported UF");
7501 (IsEpilogueVectorization || !ExpandedSCEVs) &&
7502 "expanded SCEVs to reuse can only be used during epilogue vectorization");
7504 if (!IsEpilogueVectorization)
7508 <<
", UF=" << BestUF <<
'\n');
7509 BestVPlan.
setName(
"Final VPlan");
7526 assert(IsEpilogueVectorization &&
"should only re-use the existing trip "
7527 "count during epilogue vectorization");
7531 Value *CanonicalIVStartValue;
7532 std::tie(State.
CFG.
PrevBB, CanonicalIVStartValue) =
7539 std::unique_ptr<LoopVersioning> LVer =
nullptr;
7547 LVer = std::make_unique<LoopVersioning>(
7550 State.
LVer = &*LVer;
7567 CanonicalIVStartValue, State);
7577 ReductionResumeValues, State, OrigLoop,
7586 std::optional<MDNode *> VectorizedLoopID =
7593 if (VectorizedLoopID)
7594 L->setLoopID(*VectorizedLoopID);
7618#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
7620 for (
const auto &Plan : VPlans)
7634std::pair<BasicBlock *, Value *>
7636 const SCEV2ValueTy &ExpandedSCEVs) {
7676 dbgs() <<
"Create Skeleton for epilogue vectorized loop (first pass)\n"
7686 dbgs() <<
"intermediate fn:\n"
7694 assert(Bypass &&
"Expected valid bypass basic block.");
7715 TCCheckBlock->
setName(
"vector.main.loop.iter.check");
7719 DT,
LI,
nullptr,
"vector.ph");
7724 "TC check is expected to dominate Bypass");
7748 return TCCheckBlock;
7757std::pair<BasicBlock *, Value *>
7759 const SCEV2ValueTy &ExpandedSCEVs) {
7765 VecEpilogueIterationCountCheck->
setName(
"vec.epilog.iter.check");
7768 LI,
nullptr,
"vec.epilog.ph");
7770 VecEpilogueIterationCountCheck);
7775 "expected this to be saved from the previous pass.");
7793 VecEpilogueIterationCountCheck,
7817 for (
PHINode &Phi : VecEpilogueIterationCountCheck->
phis())
7820 for (
PHINode *Phi : PhisInBlock) {
7822 Phi->replaceIncomingBlockWith(
7824 VecEpilogueIterationCountCheck);
7831 return EPI.EpilogueIterationCountCheck == IncB;
7843 Type *IdxTy =
Legal->getWidestInductionType();
7847 EPResumeVal->
addIncoming(ConstantInt::get(IdxTy, 0),
7858 {VecEpilogueIterationCountCheck,
7869 "Expected trip count to have been safed in the first pass.");
7873 "saved trip count does not dominate insertion point.");
7884 Value *CheckMinIters =
7888 "min.epilog.iters.check");
7894 unsigned EpilogueLoopStep =
7900 unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep);
7901 const uint32_t Weights[] = {EstimatedSkipCount,
7902 MainLoopStep - EstimatedSkipCount};
7913 dbgs() <<
"Create Skeleton for epilogue vectorized loop (second pass)\n"
7927 assert(!Range.isEmpty() &&
"Trying to test an empty VF range.");
7928 bool PredicateAtRangeStart = Predicate(Range.Start);
7931 if (Predicate(TmpVF) != PredicateAtRangeStart) {
7936 return PredicateAtRangeStart;
7946 auto MaxVFTimes2 = MaxVF * 2;
7948 VFRange SubRange = {VF, MaxVFTimes2};
7957 if (
auto *
I = dyn_cast<Instruction>(
Op)) {
7958 if (
auto *R = Ingredient2Recipe.lookup(
I))
7959 return R->getVPSingleValue();
7961 return Plan.getOrAddLiveIn(
Op);
7970 std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
7972 if (ECEntryIt != EdgeMaskCache.
end())
7973 return ECEntryIt->second;
7978 BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator());
7979 assert(BI &&
"Unexpected terminator found");
7982 return EdgeMaskCache[Edge] = SrcMask;
7988 return EdgeMaskCache[Edge] = SrcMask;
7991 assert(EdgeMask &&
"No Edge Mask found for condition");
8007 return EdgeMaskCache[Edge] = EdgeMask;
8014 std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
8016 assert(ECEntryIt != EdgeMaskCache.
end() &&
8017 "looking up mask for edge which has not been created");
8018 return ECEntryIt->second;
8026 BlockMaskCache[Header] =
nullptr;
8038 HeaderVPBB->
insert(
IV, NewInsertionPoint);
8045 BlockMaskCache[Header] = BlockMask;
8051 assert(BCEntryIt != BlockMaskCache.
end() &&
8052 "Trying to access mask for block without one.");
8053 return BCEntryIt->second;
8057 assert(OrigLoop->
contains(BB) &&
"Block is not a part of a loop");
8058 assert(BlockMaskCache.
count(BB) == 0 &&
"Mask for block already computed");
8060 "Loop header must have cached block mask");
8069 BlockMaskCache[BB] = EdgeMask;
8074 BlockMask = EdgeMask;
8078 BlockMask = Builder.
createOr(BlockMask, EdgeMask, {});
8081 BlockMaskCache[BB] = BlockMask;
8087 assert((isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
8088 "Must be called with either a load or store");
8094 "CM decision should be taken at this point.");
8120 auto *
GEP = dyn_cast<GetElementPtrInst>(
8121 Ptr->getUnderlyingValue()->stripPointerCasts());
8128 if (
LoadInst *Load = dyn_cast<LoadInst>(
I))
8147 "step must be loop invariant");
8151 if (
auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
8154 assert(isa<PHINode>(PhiOrTrunc) &&
"must be a phi node here");
8165 *PSE.
getSE(), *OrigLoop, Range);
8191 auto isOptimizableIVTruncate =
8199 isOptimizableIVTruncate(
I),
Range)) {
8201 auto *
Phi = cast<PHINode>(
I->getOperand(0));
8212 unsigned NumIncoming =
Phi->getNumIncomingValues();
8223 for (
unsigned In = 0;
In < NumIncoming;
In++) {
8228 assert(In == 0 &&
"Both null and non-null edge masks found");
8230 "Distinct incoming values with one having a full mask");
8253 if (
ID && (
ID == Intrinsic::assume ||
ID == Intrinsic::lifetime_end ||
8254 ID == Intrinsic::lifetime_start ||
ID == Intrinsic::sideeffect ||
8255 ID == Intrinsic::pseudoprobe ||
8256 ID == Intrinsic::experimental_noalias_scope_decl))
8262 bool ShouldUseVectorIntrinsic =
8269 if (ShouldUseVectorIntrinsic)
8274 std::optional<unsigned> MaskPos;
8296 Variant = Decision.Variant;
8297 MaskPos = Decision.MaskPos;
8304 if (ShouldUseVectorCall) {
8305 if (MaskPos.has_value()) {
8320 Ops.insert(Ops.
begin() + *MaskPos, Mask);
8332 assert(!isa<BranchInst>(
I) && !isa<PHINode>(
I) && !isa<LoadInst>(
I) &&
8333 !isa<StoreInst>(
I) &&
"Instruction should have been handled earlier");
8348 switch (
I->getOpcode()) {
8351 case Instruction::SDiv:
8352 case Instruction::UDiv:
8353 case Instruction::SRem:
8354 case Instruction::URem: {
8371 case Instruction::Add:
8372 case Instruction::And:
8373 case Instruction::AShr:
8374 case Instruction::FAdd:
8375 case Instruction::FCmp:
8376 case Instruction::FDiv:
8377 case Instruction::FMul:
8378 case Instruction::FNeg:
8379 case Instruction::FRem:
8380 case Instruction::FSub:
8381 case Instruction::ICmp:
8382 case Instruction::LShr:
8383 case Instruction::Mul:
8384 case Instruction::Or:
8385 case Instruction::Select:
8386 case Instruction::Shl:
8387 case Instruction::Sub:
8388 case Instruction::Xor:
8389 case Instruction::Freeze:
8397 auto *PN = cast<PHINode>(R->getUnderlyingValue());
8399 getRecipe(cast<Instruction>(PN->getIncomingValueForBlock(OrigLatch)));
8416 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(
I)) {
8418 case Intrinsic::assume:
8419 case Intrinsic::lifetime_start:
8420 case Intrinsic::lifetime_end:
8442 VPValue *BlockInMask =
nullptr;
8443 if (!IsPredicated) {
8447 LLVM_DEBUG(
dbgs() <<
"LV: Scalarizing and predicating:" << *
I <<
"\n");
8456 IsUniform, BlockInMask);
8467 if (
auto Phi = dyn_cast<PHINode>(Instr)) {
8468 if (Phi->getParent() != OrigLoop->
getHeader())
8471 if ((Recipe = tryToOptimizeInductionPHI(Phi,
Operands, Range)))
8477 "can only widen reductions and fixed-order recurrences here");
8495 PhisToFix.push_back(PhiRecipe);
8499 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8500 cast<TruncInst>(Instr),
Operands, Range)))
8508 if (
auto *CI = dyn_cast<CallInst>(Instr))
8509 return tryToWidenCall(CI,
Operands, Range);
8511 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
8512 return tryToWidenMemory(Instr,
Operands, Range);
8514 if (!shouldWiden(Instr, Range))
8517 if (
auto GEP = dyn_cast<GetElementPtrInst>(Instr))
8521 if (
auto *SI = dyn_cast<SelectInst>(Instr)) {
8526 if (
auto *CI = dyn_cast<CastInst>(Instr)) {
8531 return tryToWiden(Instr,
Operands, VPBB);
8534void LoopVectorizationPlanner::buildVPlansWithVPRecipes(
ElementCount MinVF,
8538 auto MaxVFTimes2 = MaxVF * 2;
8540 VFRange SubRange = {VF, MaxVFTimes2};
8541 if (
auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
8561 Value *StartIdx = ConstantInt::get(IdxTy, 0);
8568 Header->insert(CanonicalIVPHI, Header->begin());
8573 Instruction::Add, {CanonicalIVPHI, &Plan.
getVFxUF()}, {HasNUW,
false},
DL,
8575 CanonicalIVPHI->
addOperand(CanonicalIVIncrement);
8594 Value *IncomingValue =
8595 ExitPhi.getIncomingValueForBlock(ExitingBB);
8602LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
VFRange &Range) {
8622 Plan->getVectorLoopRegion()->setEntry(HeaderVPBB);
8623 Plan->getVectorLoopRegion()->setExiting(LatchVPBB);
8629 bool IVUpdateMayOverflow =
false;
8640 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, Legal, CM, PSE, Builder);
8660 "Unsupported interleave factor for scalable vectors");
8665 InterleaveGroups.
insert(IG);
8682 bool NeedsBlends = BB != HeaderBB && !BB->phis().empty();
8683 return Legal->blockNeedsPredication(BB) || NeedsBlends;
8688 if (VPBB != HeaderVPBB)
8692 if (VPBB == HeaderVPBB)
8693 RecipeBuilder.createHeaderMask();
8694 else if (NeedsMasks)
8695 RecipeBuilder.createBlockInMask(BB);
8702 auto *
Phi = dyn_cast<PHINode>(Instr);
8703 if (Phi &&
Phi->getParent() == HeaderBB) {
8704 Operands.push_back(Plan->getOrAddLiveIn(
8707 auto OpRange = RecipeBuilder.mapToVPValues(
Instr->operands());
8708 Operands = {OpRange.begin(), OpRange.end()};
8714 if ((SI = dyn_cast<StoreInst>(&
I)) &&
8719 RecipeBuilder.tryToCreateWidenRecipe(Instr,
Operands, Range, VPBB);
8721 Recipe = RecipeBuilder.handleReplication(Instr, Range);
8723 RecipeBuilder.setRecipe(Instr, Recipe);
8724 if (isa<VPHeaderPHIRecipe>(Recipe)) {
8735 "unexpected recipe needs moving");
8755 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
8756 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8757 "entry block must be set to a VPRegionBlock having a non-empty entry "
8759 RecipeBuilder.fixHeaderPhis();
8767 adjustRecipesForReductions(LatchVPBB, Plan, RecipeBuilder,
Range.Start);
8772 for (
const auto *IG : InterleaveGroups) {
8774 cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IG->getInsertPos()));
8776 for (
unsigned i = 0; i < IG->getFactor(); ++i)
8777 if (
auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i))) {
8778 auto *StoreR = cast<VPWidenStoreRecipe>(RecipeBuilder.getRecipe(SI));
8779 StoredValues.
push_back(StoreR->getStoredValue());
8782 bool NeedsMaskForGaps =
8785 Recipe->getMask(), NeedsMaskForGaps);
8786 VPIG->insertBefore(Recipe);
8788 for (
unsigned i = 0; i < IG->getFactor(); ++i)
8790 VPRecipeBase *MemberR = RecipeBuilder.getRecipe(Member);
8791 if (!
Member->getType()->isVoidTy()) {
8802 Plan->setName(
"Initial VPlan");
8807 auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
8808 auto *ScevStride = dyn_cast<SCEVConstant>(PSE.
getSCEV(StrideV));
8812 Constant *CI = ConstantInt::get(Stride->getType(), ScevStride->getAPInt());
8814 auto *ConstVPV = Plan->getOrAddLiveIn(CI);
8817 Plan->getOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV);
8833 bool WithoutRuntimeCheck =
8836 WithoutRuntimeCheck);
8856 HCFGBuilder.buildHierarchicalCFG();
8864 *PSE.
getSE(), *TLI);
8869 Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
8870 Term->eraseFromParent();
8890void LoopVectorizationPlanner::adjustRecipesForReductions(
8893 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
8900 if (
auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R))
8903 bool HasIntermediateStore =
false;
8908 auto *IS2 =
R2->getRecurrenceDescriptor().IntermediateStore;
8909 HasIntermediateStore |= IS1 || IS2;
8930 if (HasIntermediateStore && ReductionPHIList.
size() > 1)
8932 R->moveBefore(*Header, Header->getFirstNonPhi());
8935 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8936 if (!PhiR || !PhiR->isInLoop() || (MinVF.
isScalar() && !PhiR->isOrdered()))
8942 "AnyOf reductions are not allowed for in-loop reductions");
8947 for (
unsigned I = 0;
I != Worklist.
size(); ++
I) {
8950 auto *UserRecipe = dyn_cast<VPSingleDefRecipe>(U);
8952 assert(isa<VPLiveOut>(U) &&
8953 "U must either be a VPSingleDef or VPLiveOut");
8956 Worklist.
insert(UserRecipe);
8969 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
8972 unsigned IndexOfFirstOperand;
8980 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
8981 assert(((MinVF.
isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
8982 isa<VPWidenCallRecipe>(CurrentLink)) &&
8983 CurrentLink->getOperand(2) == PreviousLink &&
8984 "expected a call where the previous link is the added operand");
8992 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
8994 LinkVPBB->
insert(FMulRecipe, CurrentLink->getIterator());
8997 auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink);
8998 if (PhiR->isInLoop() && Blend) {
8999 assert(Blend->getNumIncomingValues() == 2 &&
9000 "Blend must have 2 incoming values");
9001 if (Blend->getIncomingValue(0) == PhiR)
9002 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
9004 assert(Blend->getIncomingValue(1) == PhiR &&
9005 "PhiR must be an operand of the blend");
9006 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
9012 if (isa<VPWidenRecipe>(CurrentLink)) {
9013 assert(isa<CmpInst>(CurrentLinkI) &&
9014 "need to have the compare of the select");
9017 assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
9018 "must be a select recipe");
9019 IndexOfFirstOperand = 1;
9022 "Expected to replace a VPWidenSC");
9023 IndexOfFirstOperand = 0;
9028 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
9029 ? IndexOfFirstOperand + 1
9030 : IndexOfFirstOperand;
9031 VecOp = CurrentLink->getOperand(VecOpId);
9032 assert(VecOp != PreviousLink &&
9033 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
9034 (VecOpId - IndexOfFirstOperand)) ==
9036 "PreviousLink must be the operand other than VecOp");
9052 CurrentLink->replaceAllUsesWith(RedRecipe);
9053 PreviousLink = RedRecipe;
9058 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
9071 assert(OrigExitingVPV->getDefiningRecipe()->getParent() != LatchVPBB &&
9072 "reduction recipe must be defined before latch");
9074 std::optional<FastMathFlags> FMFs =
9081 return isa<VPInstruction>(&U) &&
9082 cast<VPInstruction>(&U)->getOpcode() ==
9097 assert(!PhiR->
isInLoop() &&
"Unexpected truncated inloop reduction!");
9106 Trunc->
insertAfter(NewExitingVPV->getDefiningRecipe());
9107 Extnd->insertAfter(Trunc);
9109 PhiR->
setOperand(1, Extnd->getVPSingleValue());
9110 NewExitingVPV = Extnd;
9129 ->appendRecipe(FinalReductionResult);
9131 FinalReductionResult,
9138#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
9141 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
" at ";
9142 IG->getInsertPos()->printAsOperand(O,
false);
9152 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
9153 if (!IG->getMember(i))
9156 O <<
"\n" << Indent <<
" store ";
9158 O <<
" to index " << i;
9160 O <<
"\n" << Indent <<
" ";
9162 O <<
" = load from index " << i;
9171 "Not a pointer induction according to InductionDescriptor!");
9173 "Unexpected type.");
9175 "Recipe should have been replaced");
9178 PHINode *CanonicalIV = cast<PHINode>(State.
get(IVR, 0,
true));
9183 Type *ScStValueType = ScalarStartValue->
getType();
9188 NewPointerPhi->
addIncoming(ScalarStartValue, VectorPH);
9195 Value *NumUnrolledElems =
9206 NewPointerPhi->
addIncoming(InductionGEP, VectorPH);
9211 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
9213 Value *StartOffsetScalar =
9215 Value *StartOffset =
9222 "scalar step must be the same across all parts");
9229 State.
set(
this,
GEP, Part);
9234 assert(!State.
Instance &&
"VPDerivedIVRecipe being replicated.");
9245 Kind, cast_if_present<BinaryOperator>(FPBinOp));
9246 DerivedIV->
setName(
"offset.idx");
9247 assert(DerivedIV != CanonicalIV &&
"IV didn't need transforming?");
9267 if (State.
Instance->Lane.isFirstLane()) {
9281 if ((isa<LoadInst>(UI) || isa<StoreInst>(UI)) &&
9283 return Op->isDefinedOutsideVectorRegions();
9287 for (
unsigned Part = 1; Part < State.
UF; ++Part)
9296 for (
unsigned Part = 0; Part < State.
UF; ++Part)
9303 if (isa<StoreInst>(UI) &&
9314 for (
unsigned Part = 0; Part < State.
UF; ++Part)
9315 for (
unsigned Lane = 0; Lane < EndLane; ++Lane)
9323 Value *StoredVal,
bool IsScatter,
Value *Mask,
9328 Intrinsic::vp_scatter,
9329 {StoredVal, Addr, Mask, EVL});
9335 {StoredVal, Addr}));
9348 const Align &Alignment) {
9353 nullptr,
"wide.masked.gather");
9358 Instruction::Load, DataTy,
Addr,
"vp.op.load"));
9373 auto &Builder = State.
Builder;
9375 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
9377 Value *Mask =
nullptr;
9378 if (
auto *VPMask =
getMask()) {
9381 Mask = State.
get(VPMask, Part);
9383 Mask = Builder.CreateVectorReverse(Mask,
"reverse");
9388 assert(State.
UF == 1 &&
"Expected only UF == 1 when vectorizing with "
9389 "explicit vector length.");
9390 assert(cast<VPInstruction>(State.
EVL)->getOpcode() ==
9392 "EVL must be VPInstruction::ExplicitVectorLength.");
9401 Builder, DataTy, State.
get(
getAddr(), Part, !CreateGather),
9402 CreateGather, Mask, EVL, Alignment);
9403 }
else if (CreateGather) {
9405 NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, Mask,
9406 nullptr,
"wide.masked.gather");
9409 auto *VecPtr = State.
get(
getAddr(), Part,
true);
9411 NewLI = Builder.CreateMaskedLoad(DataTy, VecPtr, Alignment, Mask,
9413 "wide.masked.load");
9416 Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment,
"wide.load");
9421 NewLI = Builder.CreateVectorReverse(NewLI,
"reverse");
9424 State.
set(
this, NewLI, Part);
9435 auto &Builder = State.
Builder;
9438 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
9440 Value *Mask =
nullptr;
9441 if (
auto *VPMask =
getMask()) {
9444 Mask = State.
get(VPMask, Part);
9446 Mask = Builder.CreateVectorReverse(Mask,
"reverse");
9449 Value *StoredVal = State.
get(StoredVPValue, Part);
9451 assert(!State.
EVL &&
"reversing not yet implemented with EVL");
9454 StoredVal = Builder.CreateVectorReverse(StoredVal,
"reverse");
9460 assert(State.
UF == 1 &&
"Expected only UF == 1 when vectorizing with "
9461 "explicit vector length.");
9462 assert(cast<VPInstruction>(State.
EVL)->getOpcode() ==
9464 "EVL must be VPInstruction::ExplicitVectorLength.");
9473 Builder, State.
get(
getAddr(), Part, !CreateScatter), StoredVal,
9474 CreateScatter, Mask, EVL, Alignment);
9475 }
else if (CreateScatter) {
9478 Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment, Mask);
9480 auto *VecPtr = State.
get(
getAddr(), Part,
true);
9482 NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment, Mask);
9484 NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
9552 LLVM_DEBUG(
dbgs() <<
"LV: cannot compute the outer-loop trip count\n");
9556 Function *
F = L->getHeader()->getParent();
9562 LoopVectorizationCostModel CM(
SEL, L, PSE, LI, LVL, *
TTI, TLI, DB, AC, ORE,
F,
9567 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *
TTI, LVL, CM, IAI, PSE, Hints,
9587 bool AddBranchWeights =
9589 GeneratedRTChecks Checks(*PSE.
getSE(), DT, LI,
TTI,
9590 F->getParent()->getDataLayout(), AddBranchWeights);
9592 VF.
Width, 1, LVL, &CM, BFI, PSI, Checks);
9594 << L->getHeader()->getParent()->getName() <<
"\"\n");
9614 if (
auto *S = dyn_cast<StoreInst>(&Inst)) {
9615 if (S->getValueOperand()->getType()->isFloatTy())
9625 while (!Worklist.
empty()) {
9627 if (!L->contains(
I))
9629 if (!Visited.
insert(
I).second)
9636 if (isa<FPExtInst>(
I) && EmittedRemark.
insert(
I).second)
9639 I->getDebugLoc(), L->getHeader())
9640 <<
"floating point conversion changes vector width. "
9641 <<
"Mixed floating point precision requires an up/down "
9642 <<
"cast that will negatively impact performance.";
9645 for (
Use &
Op :
I->operands())
9646 if (
auto *OpI = dyn_cast<Instruction>(
Op))
9653 std::optional<unsigned> VScale,
Loop *L,
9666 <<
"LV: Interleaving only is not profitable due to runtime checks\n");
9707 unsigned AssumedMinimumVscale = 1;
9709 AssumedMinimumVscale = *VScale;
9710 IntVF *= AssumedMinimumVscale;
9728 uint64_t MinTC = std::max(MinTC1, MinTC2);
9730 MinTC =
alignTo(MinTC, IntVF);
9734 dbgs() <<
"LV: Minimum required TC for runtime checks to be profitable:"
9742 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is not beneficial: expected "
9743 "trip count < minimum profitable VF ("
9754 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9756 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9761 "VPlan-native path is not enabled. Only process inner loops.");
9768 << L->getHeader()->getParent()->getName() <<
"' from "
9769 << DebugLocStr <<
"\n");
9774 dbgs() <<
"LV: Loop hints:"
9785 Function *
F = L->getHeader()->getParent();
9807 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Cannot prove legality.\n");
9817 if (!L->isInnermost())
9821 assert(L->isInnermost() &&
"Inner loop expected.");
9843 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop with a very small trip count. "
9844 <<
"This loop is worth vectorizing only if no scalar "
9845 <<
"iteration overheads are incurred.");
9847 LLVM_DEBUG(
dbgs() <<
" But vectorizing was explicitly forced.\n");
9860 LLVM_DEBUG(
dbgs() <<
" But the target considers the trip count too "
9861 "small to consider vectorizing.\n");
9863 "The trip count is below the minial threshold value.",
9864 "loop trip count is too low, avoiding vectorization",
9865 "LowTripCount",
ORE, L);
9874 if (
F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9876 "Can't vectorize when the NoImplicitFloat attribute is used",
9877 "loop not vectorized due to NoImplicitFloat attribute",
9878 "NoImplicitFloat",
ORE, L);
9890 "Potentially unsafe FP op prevents vectorization",
9891 "loop not vectorized due to unsafe FP support.",
9892 "UnsafeFP",
ORE, L);
9897 bool AllowOrderedReductions;
9907 ExactFPMathInst->getDebugLoc(),
9908 ExactFPMathInst->getParent())
9909 <<
"loop not vectorized: cannot prove it is safe to reorder "
9910 "floating-point operations";
9912 LLVM_DEBUG(
dbgs() <<
"LV: loop not vectorized: cannot prove it is safe to "
9913 "reorder floating-point operations\n");
9919 LoopVectorizationCostModel CM(
SEL, L, PSE,
LI, &LVL, *
TTI,
TLI,
DB,
AC,
ORE,
9922 LoopVectorizationPlanner LVP(L,
LI,
DT,
TLI, *
TTI, &LVL, CM, IAI, PSE, Hints,
9930 std::optional<VectorizationFactor> MaybeVF = LVP.
plan(UserVF, UserIC);
9935 bool AddBranchWeights =
9938 F->getParent()->getDataLayout(), AddBranchWeights);
9944 unsigned SelectedIC = std::max(IC, UserIC);
9951 bool ForceVectorization =
9953 if (!ForceVectorization &&
9955 *PSE.
getSE(),
SEL)) {
9958 DEBUG_TYPE,
"CantReorderMemOps", L->getStartLoc(),
9960 <<
"loop not vectorized: cannot prove it is safe to reorder "
9961 "memory operations";
9970 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
9971 bool VectorizeLoop =
true, InterleaveLoop =
true;
9973 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is possible but not beneficial.\n");
9974 VecDiagMsg = std::make_pair(
9975 "VectorizationNotBeneficial",
9976 "the cost-model indicates that vectorization is not beneficial");
9977 VectorizeLoop =
false;
9980 if (!MaybeVF && UserIC > 1) {
9983 LLVM_DEBUG(
dbgs() <<
"LV: Ignoring UserIC, because vectorization and "
9984 "interleaving should be avoided up front\n");
9985 IntDiagMsg = std::make_pair(
9986 "InterleavingAvoided",
9987 "Ignoring UserIC, because interleaving was avoided up front");
9988 InterleaveLoop =
false;
9989 }
else if (IC == 1 && UserIC <= 1) {
9992 IntDiagMsg = std::make_pair(
9993 "InterleavingNotBeneficial",
9994 "the cost-model indicates that interleaving is not beneficial");
9995 InterleaveLoop =
false;
9997 IntDiagMsg.first =
"InterleavingNotBeneficialAndDisabled";
9998 IntDiagMsg.second +=
9999 " and is explicitly disabled or interleave count is set to 1";
10001 }
else if (IC > 1 && UserIC == 1) {
10004 dbgs() <<
"LV: Interleaving is beneficial but is explicitly disabled.");
10005 IntDiagMsg = std::make_pair(
10006 "InterleavingBeneficialButDisabled",
10007 "the cost-model indicates that interleaving is beneficial "
10008 "but is explicitly disabled or interleave count is set to 1");
10009 InterleaveLoop =
false;
10013 IC = UserIC > 0 ? UserIC : IC;
10017 if (!VectorizeLoop && !InterleaveLoop) {
10021 L->getStartLoc(), L->getHeader())
10022 << VecDiagMsg.second;
10026 L->getStartLoc(), L->getHeader())
10027 << IntDiagMsg.second;
10030 }
else if (!VectorizeLoop && InterleaveLoop) {
10034 L->getStartLoc(), L->getHeader())
10035 << VecDiagMsg.second;
10037 }
else if (VectorizeLoop && !InterleaveLoop) {
10039 <<
") in " << DebugLocStr <<
'\n');
10042 L->getStartLoc(), L->getHeader())
10043 << IntDiagMsg.second;
10045 }
else if (VectorizeLoop && InterleaveLoop) {
10047 <<
") in " << DebugLocStr <<
'\n');
10051 bool DisableRuntimeUnroll =
false;
10052 MDNode *OrigLoopID = L->getLoopID();
10054 using namespace ore;
10055 if (!VectorizeLoop) {
10056 assert(IC > 1 &&
"interleave count should not be 1 or 0");
10059 InnerLoopUnroller Unroller(L, PSE,
LI,
DT,
TLI,
TTI,
AC,
ORE, IC, &LVL,
10068 <<
"interleaved loop (interleaved count: "
10069 << NV(
"InterleaveCount", IC) <<
")";
10084 EPI, &LVL, &CM,
BFI,
PSI, Checks);
10086 std::unique_ptr<VPlan> BestMainPlan(
10088 const auto &[ExpandedSCEVs, ReductionResumeValues] = LVP.
executePlan(
10103 Header->setName(
"vec.epilog.vector.body");
10113 auto *ExpandR = cast<VPExpandSCEVRecipe>(&R);
10115 ExpandedSCEVs.find(ExpandR->getSCEV())->second);
10119 ExpandR->eraseFromParent();
10126 if (isa<VPCanonicalIVPHIRecipe>(&R))
10129 Value *ResumeV =
nullptr;
10131 if (
auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
10132 ResumeV = ReductionResumeValues
10133 .find(&ReductionPhi->getRecurrenceDescriptor())
10141 if (
auto *Ind = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
10142 IndPhi = cast<PHINode>(Ind->getUnderlyingValue());
10143 ID = &Ind->getInductionDescriptor();
10145 auto *WidenInd = cast<VPWidenIntOrFpInductionRecipe>(&R);
10146 IndPhi = WidenInd->getPHINode();
10147 ID = &WidenInd->getInductionDescriptor();
10154 assert(ResumeV &&
"Must have a resume value");
10156 cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
10160 DT,
true, &ExpandedSCEVs);
10161 ++LoopsEpilogueVectorized;
10164 DisableRuntimeUnroll =
true;
10178 DisableRuntimeUnroll =
true;
10188 std::optional<MDNode *> RemainderLoopID =
10191 if (RemainderLoopID) {
10192 L->setLoopID(*RemainderLoopID);
10194 if (DisableRuntimeUnroll)
10233 bool Changed =
false, CFGChanged =
false;
10240 for (
const auto &L : *
LI)
10241 Changed |= CFGChanged |=
10252 LoopsAnalyzed += Worklist.
size();
10255 while (!Worklist.
empty()) {
10301 runImpl(
F,
SE,
LI,
TTI,
DT,
BFI, &
TLI,
DB,
AC,
LAIs,
ORE,
PSI);
10302 if (!Result.MadeAnyChange)
10321 if (Result.MadeCFGChange) {
10337 OS, MapClassName2PassName);
10340 OS << (InterleaveOnlyWhenForced ?
"" :
"no-") <<
"interleave-forced-only;";
10341 OS << (VectorizeOnlyWhenForced ?
"" :
"no-") <<
"vectorize-forced-only;";
static unsigned getIntrinsicID(const SDNode *N)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMDGPU Lower Kernel Arguments
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefAnalysis InstSet & ToRemove
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This is the interface for a simple mod/ref and alias analysis over globals.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
loop Loop Strength Reduction
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static std::optional< unsigned > getSmallBestKnownTC(ScalarEvolution &SE, Loop *L)
Returns "best known" trip count for the specified loop L as defined by the following procedure: 1) Re...
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void createAndCollectMergePhiForReduction(VPInstruction *RedResult, DenseMap< const RecurrenceDescriptor *, Value * > &ReductionResumeValues, VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock)
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop, VFRange &Range)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)
Return a vector containing interleaved elements from multiple smaller input vectors.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or it's operands.
static void emitInvalidCostRemarks(SmallVector< InstructionVFPair > InvalidCosts, OptimizationRemarkEmitter *ORE, Loop *TheLoop)
static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan)
const char LLVMLoopVectorizeFollowupAll[]
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, DebugLoc DL)
static std::optional< unsigned > getVScaleForTuning(const Loop *L, const TargetTransformInfo &TTI)
Convenience function that returns the value of vscale_range iff vscale_range.min == vscale_range....
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static constexpr uint32_t MemCheckBypassWeights[]
static Type * MaybeVectorizeType(Type *Elt, ElementCount VF)
static Instruction * lowerLoadUsingVectorIntrinsics(IRBuilderBase &Builder, VectorType *DataTy, Value *Addr, bool IsGather, Value *Mask, Value *EVL, const Align &Alignment)
Creates either vp_load or vp_gather intrinsics calls to represent predicated load/gather.
static Instruction * lowerStoreUsingVectorIntrinsics(IRBuilderBase &Builder, Value *Addr, Value *StoredVal, bool IsScatter, Value *Mask, Value *EVL, const Align &Alignment)
Creates either vp_store or vp_scatter intrinsics calls to represent predicated store/scatter.
static cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static Type * smallestIntegerVectorType(Type *T1, Type *T2)
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, std::optional< unsigned > VScale, Loop *L, ScalarEvolution &SE, ScalarEpilogueLowering SEL)
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
const char VerboseDebug[]
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I)
Create an analysis remark that explains why vectorization failed.
static constexpr uint32_t SCEVCheckBypassWeights[]
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
const char LLVMLoopVectorizeFollowupVectorized[]
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static std::string getDebugLocString(const Loop *L)
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
const char LLVMLoopVectorizeFollowupEpilogue[]
static bool useActiveLaneMask(TailFoldingStyle Style)
static Type * largestIntegerVectorType(Type *T1, Type *T2)
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static void cse(BasicBlock *BB)
Perform cse of induction variable instructions.
static unsigned getReciprocalPredBlockProb()
A helper function that returns the reciprocal of the block probability of predicated blocks.
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static void AddRuntimeUnrollDisableMetaData(Loop *L)
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static cl::opt< bool > PrintVPlansInDotFormat("vplan-print-in-dot-format", cl::Hidden, cl::desc("Use dot format instead of plain text when dumping VPlans"))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
separate const offset from gep
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
This defines the Use class.
This file defines the VPlanHCFGBuilder class which contains the public interface (buildHierarchicalCF...
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
static const uint32_t IV[8]
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
int64_t getSExtValue() const
Get sign extended value.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BinaryOps getOpcode() const
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
void setCondition(Value *V)
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
Value * getCondition() const
Represents analyses that only rely on functions' control flow.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getFalse(LLVMContext &Context)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
const ValueT & at(const_arg_type_t< KeyT > Val) const
at - Return the entry for the specified key, or abort if no such entry exists.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
DomTreeNodeBase * getIDom() const
Analysis pass which computes a DominatorTree.
bool verify(VerificationLevel VL=VerificationLevel::Full) const
verify - checks if the tree is correct.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
BasicBlock * emitMinimumVectorEpilogueIterCountCheck(BasicBlock *Bypass, BasicBlock *Insert)
Emits an iteration count bypass check after the main vector loop has finished to see if there are any...
void printDebugTracesAtEnd() override
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
std::pair< BasicBlock *, Value * > createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (ie the ...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check)
void printDebugTracesAtEnd() override
std::pair< BasicBlock *, Value * > createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final
Implements the interface for creating a vectorized skeleton using the main loop strategy (ie the firs...
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * emitIterationCountCheck(BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Convenience struct for specifying and reasoning about fast-math flags.
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Module * getParent()
Get the module that this global value is contained inside of...
Common base class shared among various IRBuilders.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
BinaryOperator * getInductionBinOp() const
InductionKind getKind() const
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
An extension of the inner loop vectorizer that creates a skeleton for a vectorized loop that has its ...
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks)
virtual std::pair< BasicBlock *, Value * > createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs)=0
The interface for creating a vectorized skeleton using one of two different strategies,...
std::pair< BasicBlock *, Value * > createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final
Create a new empty loop that will contain vectorized instructions later on, while the old loop will b...
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, unsigned UnrollFactor, LoopVectorizationLegality *LVL, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check)
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
PHINode * createInductionResumeValue(PHINode *OrigPhi, const InductionDescriptor &ID, Value *Step, ArrayRef< BasicBlock * > BypassBlocks, std::pair< BasicBlock *, Value * > AdditionalBypass={nullptr, nullptr})
Create a new phi node for the induction variable OrigPhi to resume iteration count in the scalar epil...
void scalarizeInstruction(const Instruction *Instr, VPReplicateRecipe *RepRecipe, const VPIteration &Instance, VPTransformState &State)
A helper function to scalarize a single Instruction in the innermost loop.
BasicBlock * LoopScalarBody
The scalar loop body.
Value * TripCount
Trip count of the original loop.
void sinkScalarOperands(Instruction *PredInst)
Iteratively sink the scalarized operands of a predicated instruction into the block that was created ...
const TargetLibraryInfo * TLI
Target Library Info.
DenseMap< PHINode *, Value * > IVEndValues
ElementCount MinProfitableTripCount
Value * createBitOrPointerCast(Value *V, VectorType *DstVTy, const DataLayout &DL)
Returns a bitcasted value to the requested vector type.
const TargetTransformInfo * TTI
Target Transform Info.
Value * VectorTripCount
Trip count of the widened loop (TripCount - TripCount % (VF*UF))
bool areSafetyChecksAdded()
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor, LoopVectorizationLegality *LVL, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks)
BasicBlock * emitSCEVChecks(BasicBlock *Bypass)
Emit a bypass check to see if all of the SCEV assumptions we've had to make are correct.
LoopVectorizationCostModel * Cost
The profitablity analysis.
SmallMapVector< const RecurrenceDescriptor *, PHINode *, 4 > ReductionResumeValues
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
BasicBlock * LoopMiddleBlock
Middle Block between the vector and the scalar.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
SmallVector< Instruction *, 4 > PredicatedInstructions
Store instructions that were predicated.
void createVectorLoopSkeleton(StringRef Prefix)
Emit basic blocks (prefixed with Prefix) for the iteration check, vector loop preheader,...
BasicBlock * completeLoopSkeleton()
Complete the loop skeleton by adding debug MDs, creating appropriate conditional branches in the midd...
BasicBlock * emitMemRuntimeChecks(BasicBlock *Bypass)
Emit bypass checks to check any memory assumptions we may have made.
BasicBlock * LoopScalarPreHeader
The scalar-loop preheader.
LoopVectorizationLegality * Legal
The legality analysis.
void emitIterationCountCheck(BasicBlock *Bypass)
Emit a bypass check to see if the vector trip count is zero, including if it overflows.
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II, Value *VectorTripCount, Value *EndValue, BasicBlock *MiddleBlock, BasicBlock *VectorHeader, VPlan &Plan, VPTransformState &State)
Set up the values of the IVs correctly when exiting the vector loop.
void createInductionResumeValues(const SCEV2ValueTy &ExpandedSCEVs, std::pair< BasicBlock *, Value * > AdditionalBypass={nullptr, nullptr})
Create new phi nodes for the induction variables to resume iteration count in the scalar epilogue,...
void fixNonInductionPHIs(VPlan &Plan, VPTransformState &State)
Fix the non-induction PHIs in Plan.
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
bool OptForSizeBasedOnProfile
BasicBlock * LoopVectorPreHeader
The vector-loop preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
Value * getOrCreateVectorTripCount(BasicBlock *InsertBlock)
Returns (and creates if needed) the trip count of the widened loop.
IRBuilder Builder
The builder that we use.
void vectorizeInterleaveGroup(const InterleaveGroup< Instruction > *Group, ArrayRef< VPValue * > VPDefs, VPTransformState &State, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *BlockInMask, bool NeedsMaskForGaps)
Try to vectorize interleaved access group Group with the base address given in Addr,...
void fixFixedOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State)
Create the exit value of first order recurrences in the middle block and update their users.
virtual std::pair< BasicBlock *, Value * > createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs)
Create a new empty loop that will contain vectorized instructions later on, while the old loop will b...
unsigned UF
The vectorization unroll factor to use.
void fixVectorizedLoop(VPTransformState &State, VPlan &Plan)
Fix the vectorized code, taking care of header phi's, live-outs, and more.
BasicBlock * LoopExitBlock
The unique ExitBlock of the scalar loop if one exists.
SmallVector< BasicBlock *, 4 > LoopBypassBlocks
A list of all bypass blocks. The first block is the entry of the loop.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
std::optional< CostType > getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void replaceSuccessorWith(BasicBlock *OldBB, BasicBlock *NewBB)
Replace specified successor OldBB to point at the provided block.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
uint32_t getIndex(const InstTy *Instr) const
Get the index for the given member.
InstTy * getInsertPos() const
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
Drive the analysis of interleaved memory accesses in the loop.
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool isInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleave group.
bool invalidateGroups()
Invalidate groups, e.g., in case all blocks in loop will be predicated contrary to original assumptio...
iterator_range< SmallPtrSetIterator< llvm::InterleaveGroup< Instruction > * > > getInterleaveGroups()
void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
const DenseMap< Value *, const SCEV * > & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
BlockT * getHeader() const
unsigned getLoopDepth() const
Return the nesting level of this loop.
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop.
BlockT * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool requiresScalarEpilogue(VFRange Range) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
bool hasPredStores() const
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
bool isAccessInterleaved(Instruction *Instr)
Check if Instr belongs to any interleaved access group.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
const TargetTransformInfo & TTI
Vector target information.
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI)
const Function * TheFunction
LoopVectorizationLegality * Legal
Vectorization legality.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
std::pair< InstructionCost, bool > VectorizationCostTy
The vectorization cost is a combination of the cost itself and a boolean indicating whether any of th...
DemandedBits * DB
Demanded bits analysis.
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
Loop * TheLoop
The loop that we evaluate.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallVector< RegisterUsage, 8 > calculateRegisterUsage(ArrayRef< ElementCount > VFs)
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
VectorizationCostTy expectedCost(ElementCount VF, SmallVectorImpl< InstructionVFPair > *Invalid=nullptr)
Returns the expected execution cost.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool runtimeChecksRequired()
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool isEpilogueVectorizationProfitable(const ElementCount VF) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
void collectUniformsAndScalars(ElementCount VF)
Collect Uniform and Scalar values for the given VF.
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr)
Get the interleaved access group that Instr belongs to.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
unsigned selectInterleaveCount(ElementCount VF, InstructionCost LoopCost)
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
unsigned getNumStores() const
bool hasVectorCallVariants() const
Returns true if there is at least one function call in the loop which has a vectorized variant availa...
uint64_t getMaxSafeVectorWidthInBits() const
bool isInvariantAddressOfReduction(Value *V)
Returns True if given address is invariant and is used to store recurrent expression.
bool blockNeedsPredication(BasicBlock *BB) const
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
int isConsecutivePtr(Type *AccessTy, Value *Ptr) const
Check if this pointer is consecutive when vectorizing.
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
bool isReductionVariable(PHINode *PN) const
Returns True if PN is a reduction variable in this loop.
bool isFixedOrderRecurrence(const PHINode *Phi) const
Returns True if Phi is a fixed-order recurrence in this loop.
const InductionDescriptor * getPointerInductionDescriptor(PHINode *Phi) const
Returns a pointer to the induction descriptor, if Phi is pointer induction.
const InductionDescriptor * getIntOrFpInductionDescriptor(PHINode *Phi) const
Returns a pointer to the induction descriptor, if Phi is an integer or floating point induction.
bool isInductionPhi(const Value *V) const
Returns True if V is a Phi node of an induction variable in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool isInvariant(Value *V) const
Returns true if value V is uniform across VF lanes, when VF is provided, and otherwise if V is invari...
const ReductionList & getReductionVars() const
Returns the reduction variables found in the loop.
bool isSafeForAnyVectorWidth() const
unsigned getNumLoads() const
Type * getWidestInductionType()
Returns the widest induction type.
const LoopAccessInfo * getLAI() const
bool prepareToFoldTailByMasking()
Return true if we can vectorize this loop while folding its tail by masking, and mark all respective ...
bool isUniformMemOp(Instruction &I, ElementCount VF) const
A uniform memory op is a load or store which accesses the same memory location on all VF lanes,...
bool isMaskRequired(const Instruction *I) const
Returns true if vector representation of the instruction I requires mask.
const RuntimePointerChecking * getRuntimePointerChecking() const
Returns the information that we collected about runtime memory check.
Planner drives the vectorization process after having passed Legality checks.
std::optional< VectorizationFactor > plan(ElementCount UserVF, unsigned UserIC)
Plan how to best vectorize, return the best VF and its cost, or std::nullopt if vectorization and int...
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
std::pair< DenseMap< const SCEV *, Value * >, DenseMap< const RecurrenceDescriptor *, Value * > > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool IsEpilogueVectorization, const DenseMap< const SCEV *, Value * > *ExpandedSCEVs=nullptr)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
VPlan & getBestPlanFor(ElementCount VF) const
Return the best VPlan for VF.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
void printPlans(raw_ostream &O)
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Instruction * getExactFPInst()
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
bool isScalableVectorizationDisabled() const
enum ForceKind getForce() const
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
bool allowReordering() const
When enabling loop hints are provided we allow the vectorizer to change the order of operations that ...
void emitRemarkWithHints() const
Dumps all the hint information.
bool isPotentiallyUnsafe() const
ElementCount getWidth() const
@ FK_Enabled
Forcing enabled.
@ FK_Undefined
Not selected.
@ FK_Disabled
Forcing disabled.
unsigned getPredicate() const
void setAlreadyVectorized()
Mark the loop L as already vectorized by setting the width to 1.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
unsigned getInterleave() const
void prepareNoAliasMetadata()
Set up the aliasing scopes based on the memchecks.
Represents a single loop in the control flow graph.
bool hasLoopInvariantOperands(const Instruction *I) const
Return true if all the operands of the specified instruction are loop invariant.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
void setIncomingValueForBlock(const BasicBlock *BB, Value *V)
Set every incoming value(s) for block BB to V.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Value * getIncomingValueForBlock(const BasicBlock *BB) const
static unsigned getIncomingValueNumForOperand(unsigned i)
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
const SCEVPredicate & getPredicate() const
const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
bool Need
This flag indicates if we need to add the runtime check.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class represents a constant integer value.
const APInt & getAPInt() const
Helper to remove instructions inserted during SCEV expansion, unless they are marked as used.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
This class represents an assumption made using SCEV expressions which can be checked at run-time.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
bool isOne() const
Return true if the expression is a constant one.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
unsigned getSmallConstantMaxTripCount(const Loop *L)
Returns the upper bound of the loop trip count as a normal unsigned value.
const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
iterator end()
Get an iterator to the end of the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
iterator begin()
Get an iterator to the beginning of the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
value_type pop_back_val()
This class provides computation of slot numbers for LLVM Assembly writing.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Value handle that tracks a Value across RAUW.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt1Ty(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isTokenTy() const
Return true if this is 'token'.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
iterator begin()
Recipe iterator methods.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
void setName(const Twine &newName)
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
RAII object that stores the current insertion point and restores it when the object is destroyed.
VPlan-based builder utility analogous to IRBuilder.
VPValue * createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL={}, const Twine &Name="")
VPBasicBlock * getInsertBlock() const
VPValue * createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, DebugLoc DL={}, const Twine &Name="")
Create a new ICmp VPInstruction with predicate Pred and operands A and B.
VPInstruction * createOverflowingOp(unsigned Opcode, std::initializer_list< VPValue * > Operands, VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPValue * createNot(VPValue *Operand, DebugLoc DL={}, const Twine &Name="")
VPValue * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL={}, const Twine &Name="", std::optional< FastMathFlags > FMFs=std::nullopt)
void setInsertPoint(VPBasicBlock *TheBB)
This specifies that created VPInstructions should be appended to the end of the specified block.
Canonical scalar induction phi of the vector loop.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
VPValue * getStartValue() const
This is a concrete Recipe that models a single VPlan-level instruction.
@ FirstOrderRecurrenceSplice
unsigned getOpcode() const
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
VPValue * getAddr() const
Return the address accessed by this recipe.
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
A value that is used outside the VPlan.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPValue * getVPValueOrAddLiveIn(Value *V, VPlan &Plan)
VPValue * createEdgeMask(BasicBlock *Src, BasicBlock *Dst)
A helper function that computes the predicate of the edge between SRC and DST.
VPReplicateRecipe * handleReplication(Instruction *I, VFRange &Range)
Build a VPReplicationRecipe for I.
VPValue * getBlockInMask(BasicBlock *BB) const
Returns the entry mask for the block BB.
VPValue * getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const
A helper that returns the previously computed predicate of the edge between SRC and DST.
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values of the range Operands to their corresponding VPValues.
void fixHeaderPhis()
Add the incoming values from the backedge to reduction & first-order recurrence cross-iteration phis.
VPRecipeBase * tryToCreateWidenRecipe(Instruction *Instr, ArrayRef< VPValue * > Operands, VFRange &Range, VPBasicBlock *VPBB)
Create and return a widened recipe for I if one can be created within the given VF Range.
void createHeaderMask()
Create the mask for the vector loop header block.
void createBlockInMask(BasicBlock *BB)
A helper function that computes the predicate of the block BB, assuming that the header block of the ...
VPRecipeBase * getRecipe(Instruction *I)
Return the recipe created for given ingredient.
void setFlags(Instruction *I) const
Set the IR flags for I.
A recipe for handling reduction phis.
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
const RecurrenceDescriptor & getRecurrenceDescriptor() const
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
This class can be used to assign names to VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
unsigned getNumOperands() const
VPValue * getOperand(unsigned N) const
void addOperand(VPValue *Operand)
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
void replaceAllUsesWith(VPValue *New)
user_iterator user_begin()
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A recipe to compute the pointers for widened memory accesses of IndexTy for all parts.
A recipe for widening Call instructions.
A Recipe for widening the canonical induction variable of the vector loop.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A common base class for widening memory operations.
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
VPValue * getMask() const
Return the mask used by this recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
A recipe for handling phis that are widened in the vector loop.
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Main class to build the VPlan H-CFG for an incoming IR.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State)
Prepare the plan for execution, setting up the required live-in values.
VPBasicBlock * getEntry()
VPValue & getVectorTripCount()
The vector trip count.
void setName(const Twine &newName)
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
void removeLiveOut(PHINode *PN)
void addLiveOut(PHINode *PN, VPValue *V)
VPBasicBlock * getPreheader()
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
static VPlanPtr createInitialVPlan(const SCEV *TripCount, ScalarEvolution &PSE)
Create initial VPlan skeleton, having an "entry" VPBasicBlock (wrapping original scalar pre-header) w...
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
VPValue * getSCEVExpansion(const SCEV *S) const
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUser() const
Return true if there is exactly one user of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
VectorBuilder & setEVL(Value *NewExplicitVectorLength)
VectorBuilder & setMask(Value *NewMask)
Value * createVectorInstruction(unsigned Opcode, Type *ReturnTy, ArrayRef< Value * > VecOpArray, const Twine &Name=Twine())
Base class of all SIMD vector types.
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isNonZero() const
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ PredicateElseScalarEpilogue
@ PredicateOrDontVectorize
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ C
The default llvm calling convention, compatible with C.
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
OneUse_match< T > m_OneUse(const T &SubPattern)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
NodeAddr< PhiNode * > Phi
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
const_iterator end(StringRef path)
Get end iterator over path.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
This is an optimization pass for GlobalISel generic memory operations.
bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
void stable_sort(R &&Range)
bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Returns a loop's estimated trip count based on branch weight metadata.
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
unsigned getLoadStoreAddressSpace(Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
std::pair< Instruction *, ElementCount > InstructionVFPair
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
std::optional< MDNode * > makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef< StringRef > FollowupAttrs, const char *InheritOptionsAttrsPrefix="", bool AlwaysNew=false)
Create a new loop identifier for a loop created from a loop transformation.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
auto map_range(ContainerTy &&C, FuncTy F)
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
void sort(IteratorTy Start, IteratorTy End)
llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
std::unique_ptr< VPlan > VPlanPtr
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableLoopVectorization
Align getLoadStoreAlignment(Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Type * ToVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
RecurKind
These are the kinds of recurrences that we support.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop, Loop *RemainderLoop, uint64_t UF)
Set weights for UnrolledLoop and RemainderLoop based on weights for OrigLoop and the following distri...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
@ Invalid
Denotes invalid value.
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
void reportVectorizationInfo(const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Type * getLoadStoreType(Value *I)
A helper function that returns the type of a load or store instruction.
bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
cl::opt< bool > EnableLoopInterleaving
Implement std::hash so that hash_code can be used in STL containers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
A special type used by analysis passes to provide an address that identifies that particular analysis...
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
ElementCountComparator creates a total ordering for ElementCount for the purposes of using it in a se...
Encapsulate information regarding vectorization of a loop and its epilogue.
BasicBlock * SCEVSafetyCheck
BasicBlock * MemSafetyCheck
BasicBlock * MainLoopIterationCountCheck
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF)
BasicBlock * EpilogueIterationCountCheck
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
std::optional< unsigned > MaskPos
A struct that represents some properties of the register usage of a loop.
SmallMapVector< unsigned, unsigned, 4 > MaxLocalUsers
Holds the maximum number of concurrent live intervals in the loop.
SmallMapVector< unsigned, unsigned, 4 > LoopInvariantRegs
Holds the number of loop invariant values that are used in the loop.
bool processLoop(Loop *L)
LoopAccessInfoManager * LAIs
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LoopVectorizePass(LoopVectorizeOptions Opts={})
LoopVectorizeResult runImpl(Function &F, ScalarEvolution &SE_, LoopInfo &LI_, TargetTransformInfo &TTI_, DominatorTree &DT_, BlockFrequencyInfo *BFI_, TargetLibraryInfo *TLI_, DemandedBits &DB_, AssumptionCache &AC_, LoopAccessInfoManager &LAIs_, OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
Storage for information about made changes.
A CRTP mix-in to automatically provide informational APIs needed for passes.
A MapVector that performs no allocations if smaller than a certain size.
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
A recipe for handling first-order recurrence phis.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
bool isFirstIteration() const
A recipe for widening load operations, using the address to load from and an optional mask.
void execute(VPTransformState &State) override
Generate a wide load or gather.
A recipe for widening select instructions.
A recipe for widening store operations, using the stored value, the address to store to and an option...
void execute(VPTransformState &State) override
Generate a wide store or scatter.
VPValue * getStoredValue() const
Return the value stored by this recipe.
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static bool HoistRuntimeChecks