158#define LV_NAME "loop-vectorize"
159#define DEBUG_TYPE LV_NAME
169 "llvm.loop.vectorize.followup_vectorized";
171 "llvm.loop.vectorize.followup_epilogue";
174STATISTIC(LoopsVectorized,
"Number of loops vectorized");
175STATISTIC(LoopsAnalyzed,
"Number of loops analyzed for vectorization");
176STATISTIC(LoopsEpilogueVectorized,
"Number of epilogues vectorized");
180 cl::desc(
"Enable vectorization of epilogue loops."));
184 cl::desc(
"When epilogue vectorization is enabled, and a value greater than "
185 "1 is specified, forces the given VF for all applicable epilogue "
189 "epilogue-vectorization-minimum-VF",
cl::Hidden,
190 cl::desc(
"Only loops with vectorization factor equal to or larger than "
191 "the specified value are considered for epilogue vectorization."));
197 cl::desc(
"Loops with a constant trip count that is smaller than this "
198 "value are vectorized only if no scalar iteration overheads "
203 cl::desc(
"The maximum allowed number of runtime memory checks"));
219 "prefer-predicate-over-epilogue",
222 cl::desc(
"Tail-folding and predication preferences over creating a scalar "
226 "Don't tail-predicate loops, create scalar epilogue"),
228 "predicate-else-scalar-epilogue",
229 "prefer tail-folding, create scalar epilogue if tail "
232 "predicate-dont-vectorize",
233 "prefers tail-folding, don't attempt vectorization if "
234 "tail-folding fails.")));
237 "force-tail-folding-style",
cl::desc(
"Force the tail folding style"),
240 clEnumValN(TailFoldingStyle::None,
"none",
"Disable tail folding"),
242 TailFoldingStyle::Data,
"data",
243 "Create lane mask for data only, using active.lane.mask intrinsic"),
244 clEnumValN(TailFoldingStyle::DataWithoutLaneMask,
245 "data-without-lane-mask",
246 "Create lane mask with compare/stepvector"),
247 clEnumValN(TailFoldingStyle::DataAndControlFlow,
"data-and-control",
248 "Create lane mask using active.lane.mask intrinsic, and use "
249 "it for both data and control flow"),
250 clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck,
251 "data-and-control-without-rt-check",
252 "Similar to data-and-control, but remove the runtime check"),
253 clEnumValN(TailFoldingStyle::DataWithEVL,
"data-with-evl",
254 "Use predicated EVL instructions for tail folding. If EVL "
255 "is unsupported, fallback to data-without-lane-mask.")));
259 cl::desc(
"Maximize bandwidth when selecting vectorization factor which "
260 "will be determined by the smallest type in loop."));
264 cl::desc(
"Enable vectorization on interleaved memory accesses in a loop"));
270 cl::desc(
"Enable vectorization on masked interleaved memory accesses in a loop"));
274 cl::desc(
"A flag that overrides the target's number of scalar registers."));
278 cl::desc(
"A flag that overrides the target's number of vector registers."));
282 cl::desc(
"A flag that overrides the target's max interleave factor for "
287 cl::desc(
"A flag that overrides the target's max interleave factor for "
288 "vectorized loops."));
292 cl::desc(
"A flag that overrides the target's expected cost for "
293 "an instruction to a single constant value. Mostly "
294 "useful for getting consistent testing."));
299 "Pretend that scalable vectors are supported, even if the target does "
300 "not support them. This flag should only be used for testing."));
305 "The cost of a loop that is considered 'small' by the interleaver."));
309 cl::desc(
"Enable the use of the block frequency analysis to access PGO "
310 "heuristics minimizing code growth in cold regions and being more "
311 "aggressive in hot regions."));
317 "Enable runtime interleaving until load/store ports are saturated"));
322 cl::desc(
"Max number of stores to be predicated behind an if."));
326 cl::desc(
"Count the induction variable only once when interleaving"));
330 cl::desc(
"Enable if predication of stores during vectorization."));
334 cl::desc(
"The maximum interleave count to use when interleaving a scalar "
335 "reduction in a nested loop."));
340 cl::desc(
"Prefer in-loop vector reductions, "
341 "overriding the targets preference."));
345 cl::desc(
"Enable the vectorisation of loops with in-order (strict) "
351 "Prefer predicating a reduction operation over an after loop select."));
356 cl::desc(
"Enable VPlan-native vectorization path with "
357 "support for outer loop vectorization."));
367 "Build VPlan for every supported loop nest in the function and bail "
368 "out right after the build (stress test the VPlan H-CFG construction "
369 "in the VPlan-native vectorization path)."));
373 cl::desc(
"Enable loop interleaving in Loop vectorization passes"));
376 cl::desc(
"Run the Loop vectorization passes"));
379 "force-widen-divrem-via-safe-divisor",
cl::Hidden,
381 "Override cost based safe divisor widening for div/rem instructions"));
384 "vectorizer-maximize-bandwidth-for-vector-calls",
cl::init(
true),
386 cl::desc(
"Try wider VFs if they enable the use of vector variants"));
391 "Enable vectorization of early exit loops with uncountable exits."));
410 return DL.getTypeAllocSizeInBits(Ty) !=
DL.getTypeSizeInBits(Ty);
419static std::optional<unsigned>
421 bool CanUseConstantMax =
true) {
431 if (!CanUseConstantMax)
443class GeneratedRTChecks;
539 "Trying to access AdditionalBypassBlock but it has not been set");
574 Value *MainVectorTripCount);
708 "A high UF for the epilogue loop is likely not beneficial.");
730 EPI.MainLoopVF,
EPI.MainLoopVF,
EPI.MainLoopUF, LVL,
825 if (
I->getDebugLoc() != Empty)
826 return I->getDebugLoc();
828 for (
Use &
Op :
I->operands()) {
830 if (OpInst->getDebugLoc() != Empty)
831 return OpInst->getDebugLoc();
834 return I->getDebugLoc();
843 dbgs() <<
"LV: " << Prefix << DebugMsg;
865 if (
I &&
I->getDebugLoc())
866 DL =
I->getDebugLoc();
884 return B.CreateElementCount(Ty, VF);
895 <<
"loop not vectorized: " << OREMsg);
918 "Vectorizing: ", TheLoop->
isInnermost() ?
"innermost loop" :
"outer loop",
924 <<
"vectorized " << LoopType <<
"loop (vectorization width: "
926 <<
", interleaved count: " <<
ore::NV(
"InterleaveCount", IC) <<
")";
1072 "Profitable to scalarize relevant only for VF > 1.");
1075 "cost-model should not be used for outer loops (in VPlan-native path)");
1077 auto Scalars = InstsToScalarize.find(VF);
1078 assert(Scalars != InstsToScalarize.end() &&
1079 "VF not yet analyzed for scalarization profitability");
1080 return Scalars->second.contains(
I);
1087 "cost-model should not be used for outer loops (in VPlan-native path)");
1091 if (isa<PseudoProbeInst>(
I))
1097 auto UniformsPerVF = Uniforms.find(VF);
1098 assert(UniformsPerVF != Uniforms.end() &&
1099 "VF not yet analyzed for uniformity");
1100 return UniformsPerVF->second.count(
I);
1107 "cost-model should not be used for outer loops (in VPlan-native path)");
1111 auto ScalarsPerVF = Scalars.find(VF);
1112 assert(ScalarsPerVF != Scalars.end() &&
1113 "Scalar values are not calculated for VF");
1114 return ScalarsPerVF->second.count(
I);
1120 return VF.
isVector() && MinBWs.contains(
I) &&
1142 WideningDecisions[std::make_pair(
I, VF)] = std::make_pair(W,
Cost);
1164 WideningDecisions[std::make_pair(
I, VF)] =
1165 std::make_pair(W, InsertPosCost);
1167 WideningDecisions[std::make_pair(
I, VF)] =
1168 std::make_pair(W, OtherMemberCost);
1180 "cost-model should not be used for outer loops (in VPlan-native path)");
1182 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(
I, VF);
1183 auto Itr = WideningDecisions.
find(InstOnVF);
1184 if (Itr == WideningDecisions.
end())
1186 return Itr->second.first;
1193 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(
I, VF);
1195 "The cost is not calculated");
1196 return WideningDecisions[InstOnVF].second;
1209 std::optional<unsigned> MaskPos,
1212 CallWideningDecisions[std::make_pair(CI, VF)] = {Kind, Variant, IID,
1219 return CallWideningDecisions.
at(std::make_pair(CI, VF));
1227 auto *Trunc = dyn_cast<TruncInst>(
I);
1240 Value *
Op = Trunc->getOperand(0);
1260 if (VF.
isScalar() || Uniforms.contains(VF))
1263 collectLoopUniforms(VF);
1265 collectLoopScalars(VF);
1285 bool LI = isa<LoadInst>(V);
1286 bool SI = isa<StoreInst>(V);
1301 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1302 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1313 return ScalarCost < SafeDivisorCost;
1337 std::pair<InstructionCost, InstructionCost>
1365 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
1372 LLVM_DEBUG(
dbgs() <<
"LV: Loop requires scalar epilogue: not exiting "
1373 "from latch block\n");
1378 "interleaved group requires scalar epilogue\n");
1381 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
1390 auto RequiresScalarEpilogue = [
this](
ElementCount VF) {
1393 bool IsRequired =
all_of(
Range, RequiresScalarEpilogue);
1395 (IsRequired ||
none_of(
Range, RequiresScalarEpilogue)) &&
1396 "all VFs in range must agree on whether a scalar epilogue is required");
1408 if (!ChosenTailFoldingStyle)
1410 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1411 : ChosenTailFoldingStyle->second;
1419 assert(!ChosenTailFoldingStyle &&
"Tail folding must not be selected yet.");
1421 ChosenTailFoldingStyle =
1427 ChosenTailFoldingStyle = std::make_pair(
1450 ChosenTailFoldingStyle =
1455 <<
"LV: Preference for VP intrinsics indicated. Will "
1456 "not try to generate VP Intrinsics "
1458 ?
"since interleave count specified is greater than 1.\n"
1459 :
"due to non-interleaving reasons.\n"));
1494 return InLoopReductions.contains(Phi);
1521 WideningDecisions.
clear();
1522 CallWideningDecisions.
clear();
1541 const unsigned IC)
const;
1551 Type *VectorTy)
const;
1558 unsigned NumPredStores = 0;
1567 bool FoldTailByMasking);
1572 ElementCount getMaximizedVFForTarget(
unsigned MaxTripCount,
1573 unsigned SmallestType,
1574 unsigned WidestType,
1576 bool FoldTailByMasking);
1580 bool isScalableVectorizationAllowed();
1584 ElementCount getMaxLegalScalableVF(
unsigned MaxSafeElements);
1630 PredicatedBBsAfterVectorization;
1643 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1644 ChosenTailFoldingStyle;
1647 std::optional<bool> IsScalableVectorizationAllowed;
1653 std::optional<unsigned> MaxSafeElements;
1687 ScalarCostsTy &ScalarCosts,
1713 std::pair<InstWidening, InstructionCost>>;
1715 DecisionList WideningDecisions;
1717 using CallDecisionList =
1720 CallDecisionList CallWideningDecisions;
1744 Ops, [
this, VF](
Value *V) {
return this->needsExtract(V, VF); }));
1805class GeneratedRTChecks {
1811 Value *SCEVCheckCond =
nullptr;
1819 Value *MemRuntimeCheckCond =
nullptr;
1828 bool CostTooHigh =
false;
1829 const bool AddBranchWeights;
1831 Loop *OuterLoop =
nullptr;
1843 : DT(DT), LI(LI),
TTI(
TTI), SCEVExp(*PSE.
getSE(),
DL,
"scev.check"),
1844 MemCheckExp(*PSE.
getSE(),
DL,
"scev.check"),
1845 AddBranchWeights(AddBranchWeights), PSE(PSE), CostKind(CostKind) {}
1873 nullptr,
"vector.scevcheck");
1880 if (RtPtrChecking.Need) {
1881 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1882 MemCheckBlock =
SplitBlock(Pred, Pred->getTerminator(), DT, LI,
nullptr,
1885 auto DiffChecks = RtPtrChecking.getDiffChecks();
1887 Value *RuntimeVF =
nullptr;
1892 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1898 MemCheckBlock->
getTerminator(), L, RtPtrChecking.getChecks(),
1901 assert(MemRuntimeCheckCond &&
1902 "no RT checks generated although RtPtrChecking "
1903 "claimed checks are required");
1906 if (!MemCheckBlock && !SCEVCheckBlock)
1916 if (SCEVCheckBlock) {
1922 if (MemCheckBlock) {
1930 if (MemCheckBlock) {
1934 if (SCEVCheckBlock) {
1940 OuterLoop =
L->getParentLoop();
1944 if (SCEVCheckBlock || MemCheckBlock)
1957 if (SCEVCheckBlock->getTerminator() == &
I)
1963 if (MemCheckBlock) {
1966 if (MemCheckBlock->getTerminator() == &
I)
1988 unsigned BestTripCount = 2;
1992 PSE, OuterLoop,
false))
1993 BestTripCount = *EstimatedTC;
1995 BestTripCount = std::max(BestTripCount, 1U);
1999 NewMemCheckCost = std::max(*NewMemCheckCost.
getValue(),
2002 if (BestTripCount > 1)
2004 <<
"We expect runtime memory checks to be hoisted "
2005 <<
"out of the outer loop. Cost reduced from "
2006 << MemCheckCost <<
" to " << NewMemCheckCost <<
'\n');
2008 MemCheckCost = NewMemCheckCost;
2012 RTCheckCost += MemCheckCost;
2015 if (SCEVCheckBlock || MemCheckBlock)
2016 LLVM_DEBUG(
dbgs() <<
"Total cost of runtime checks: " << RTCheckCost
2024 ~GeneratedRTChecks() {
2028 SCEVCleaner.markResultUsed();
2030 if (!MemRuntimeCheckCond)
2031 MemCheckCleaner.markResultUsed();
2033 if (MemRuntimeCheckCond) {
2034 auto &SE = *MemCheckExp.
getSE();
2041 I.eraseFromParent();
2044 MemCheckCleaner.cleanup();
2045 SCEVCleaner.cleanup();
2048 SCEVCheckBlock->eraseFromParent();
2049 if (MemRuntimeCheckCond)
2050 MemCheckBlock->eraseFromParent();
2063 SCEVCheckCond =
nullptr;
2064 if (
auto *
C = dyn_cast<ConstantInt>(
Cond))
2075 SCEVCheckBlock->getTerminator()->eraseFromParent();
2076 SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
2077 Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
2084 if (AddBranchWeights)
2087 return SCEVCheckBlock;
2096 if (!MemRuntimeCheckCond)
2105 MemCheckBlock->moveBefore(LoopVectorPreHeader);
2112 if (AddBranchWeights) {
2116 MemCheckBlock->getTerminator()->setDebugLoc(
2117 Pred->getTerminator()->getDebugLoc());
2120 MemRuntimeCheckCond =
nullptr;
2121 return MemCheckBlock;
2127 return Style == TailFoldingStyle::Data ||
2128 Style == TailFoldingStyle::DataAndControlFlow ||
2129 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
2133 return Style == TailFoldingStyle::DataAndControlFlow ||
2134 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
2164 LLVM_DEBUG(
dbgs() <<
"LV: Loop hints prevent outer loop vectorization.\n");
2170 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Interleave is not supported for "
2190 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
2200 for (
Loop *InnerL : L)
2222 ?
B.CreateSExtOrTrunc(Index, StepTy)
2223 :
B.CreateCast(Instruction::SIToFP, Index, StepTy);
2224 if (CastedIndex != Index) {
2226 Index = CastedIndex;
2236 assert(
X->getType() ==
Y->getType() &&
"Types don't match!");
2237 if (
auto *CX = dyn_cast<ConstantInt>(
X))
2240 if (
auto *CY = dyn_cast<ConstantInt>(
Y))
2243 return B.CreateAdd(
X,
Y);
2249 assert(
X->getType()->getScalarType() ==
Y->getType() &&
2250 "Types don't match!");
2251 if (
auto *CX = dyn_cast<ConstantInt>(
X))
2254 if (
auto *CY = dyn_cast<ConstantInt>(
Y))
2257 VectorType *XVTy = dyn_cast<VectorType>(
X->getType());
2258 if (XVTy && !isa<VectorType>(
Y->getType()))
2259 Y =
B.CreateVectorSplat(XVTy->getElementCount(),
Y);
2260 return B.CreateMul(
X,
Y);
2263 switch (InductionKind) {
2265 assert(!isa<VectorType>(Index->getType()) &&
2266 "Vector indices not supported for integer inductions yet");
2268 "Index type does not match StartValue type");
2269 if (isa<ConstantInt>(Step) && cast<ConstantInt>(Step)->isMinusOne())
2270 return B.CreateSub(StartValue, Index);
2275 return B.CreatePtrAdd(StartValue,
CreateMul(Index, Step));
2277 assert(!isa<VectorType>(Index->getType()) &&
2278 "Vector indices not supported for FP inductions yet");
2281 (InductionBinOp->
getOpcode() == Instruction::FAdd ||
2282 InductionBinOp->
getOpcode() == Instruction::FSub) &&
2283 "Original bin op should be defined for FP induction");
2285 Value *MulExp =
B.CreateFMul(Step, Index);
2286 return B.CreateBinOp(InductionBinOp->
getOpcode(), StartValue, MulExp,
2300 if (
F.hasFnAttribute(Attribute::VScaleRange))
2301 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2303 return std::nullopt;
2312 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2314 unsigned MaxUF = UF ? *UF :
Cost->TTI.getMaxInterleaveFactor(VF);
2316 Type *IdxTy =
Cost->Legal->getWidestInductionType();
2317 APInt MaxUIntTripCount = cast<IntegerType>(IdxTy)->getMask();
2322 if (
unsigned TC =
Cost->PSE.getSmallConstantMaxTripCount()) {
2325 std::optional<unsigned> MaxVScale =
2329 MaxVF *= *MaxVScale;
2332 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2353 assert(!Instr->getType()->isAggregateType() &&
"Can't handle vectors");
2356 bool IsVoidRetTy = Instr->getType()->isVoidTy();
2360 Cloned->
setName(Instr->getName() +
".cloned");
2365 "inferred type and type from generated instructions do not match");
2371 if (
auto DL = Instr->getDebugLoc())
2377 auto InputLane = Lane;
2388 State.
set(RepRecipe, Cloned, Lane);
2391 if (
auto *
II = dyn_cast<AssumeInst>(Cloned))
2396 bool IfPredicateInstr = Parent ? Parent->
isReplicator() :
false;
2400 [](
VPValue *
Op) { return Op->isDefinedOutsideLoopRegions(); })) &&
2401 "Expected a recipe is either within a region or all of its operands "
2402 "are defined outside the vectorized region.");
2403 if (IfPredicateInstr)
2427 if (
Cost->foldTailByMasking()) {
2429 "VF*UF must be a power of 2 when folding tail by masking");
2463 "Unexpected successor");
2466 PreVectorPH = CheckVPIRBB;
2490 auto CreateStep = [&]() ->
Value * {
2505 Value *Step = CreateStep();
2516 TripCountSCEV, SE.
getSCEV(Step))) {
2529 Value *MaxUIntTripCount =
2530 ConstantInt::get(CountTy, cast<IntegerType>(CountTy)->getMask());
2544 "TC check is expected to dominate Bypass");
2560 if (!SCEVCheckBlock)
2566 "Cannot SCEV check stride or overflow when optimizing for size");
2568 "Should already be a bypass block due to iteration count check");
2573 return SCEVCheckBlock;
2592 "Cannot emit memory checks when optimizing for size, unless forced "
2598 <<
"Code-size may be reduced by not forcing "
2599 "vectorization, or by source-code modifications "
2600 "eliminating the need for runtime checks "
2601 "(e.g., adding 'restrict').";
2610 return MemCheckBlock;
2620 assert(!R.isPhi() &&
"Tried to move phi recipe to end of block");
2621 R.moveBefore(*IRVPBB, IRVPBB->
end());
2633 "loops not exiting via the latch without required epilogue?");
2637 LI,
nullptr,
Twine(Prefix) +
"middle.block");
2641 nullptr,
Twine(Prefix) +
"scalar.ph");
2648 const SCEV2ValueTy &ExpandedSCEVs) {
2649 const SCEV *Step =
ID.getStep();
2650 if (
auto *
C = dyn_cast<SCEVConstant>(Step))
2651 return C->getValue();
2652 if (
auto *U = dyn_cast<SCEVUnknown>(Step))
2653 return U->getValue();
2654 auto I = ExpandedSCEVs.find(Step);
2655 assert(
I != ExpandedSCEVs.end() &&
"SCEV must be expanded at this point");
2665 auto *Cmp = L->getLatchCmpInst();
2667 InstsToIgnore.
insert(Cmp);
2668 for (
const auto &KV : IL) {
2675 cast<Instruction>(
IV->getIncomingValueForBlock(L->getLoopLatch()));
2677 [&](
const User *U) { return U == IV || U == Cmp; }))
2678 InstsToIgnore.
insert(IVInst);
2683 const SCEV2ValueTy &ExpandedSCEVs,
Value *MainVectorTripCount) {
2684 assert(MainVectorTripCount &&
"Must have bypass information");
2690 PHINode *OrigPhi = InductionEntry.first;
2695 Value *EndValueFromAdditionalBypass = MainVectorTripCount;
2696 if (OrigPhi != OldInduction) {
2697 auto *BinOp =
II.getInductionBinOp();
2699 if (isa_and_nonnull<FPMathOperator>(BinOp))
2703 EndValueFromAdditionalBypass =
2705 II.getStartValue(), Step,
II.getKind(), BinOp);
2706 EndValueFromAdditionalBypass->
setName(
"ind.end");
2713 "entry for OrigPhi already exits");
2719 const SCEV2ValueTy &ExpandedSCEVs) {
2780struct CSEDenseMapInfo {
2782 return isa<InsertElementInst>(
I) || isa<ExtractElementInst>(
I) ||
2783 isa<ShuffleVectorInst>(
I) || isa<GetElementPtrInst>(
I);
2795 assert(canHandle(
I) &&
"Unknown instruction!");
2797 I->value_op_end()));
2801 if (
LHS == getEmptyKey() ||
RHS == getEmptyKey() ||
2802 LHS == getTombstoneKey() ||
RHS == getTombstoneKey())
2804 return LHS->isIdenticalTo(
RHS);
2815 if (!CSEDenseMapInfo::canHandle(&In))
2821 In.replaceAllUsesWith(V);
2822 In.eraseFromParent();
2836 return CallWideningDecisions.at(std::make_pair(CI, VF)).Cost;
2844 for (
auto &ArgOp : CI->
args())
2845 Tys.push_back(ArgOp->getType());
2853 return std::min(ScalarCallCost, IntrinsicCost);
2855 return ScalarCallCost;
2868 assert(
ID &&
"Expected intrinsic call!");
2871 if (
auto *FPMO = dyn_cast<FPMathOperator>(CI))
2872 FMF = FPMO->getFastMathFlags();
2878 std::back_inserter(ParamTys),
2879 [&](
Type *Ty) { return maybeVectorizeType(Ty, VF); });
2882 dyn_cast<IntrinsicInst>(CI));
2901 for (
PHINode &PN : Exit->phis())
2951 auto IsBlockOfUsePredicated = [&](
Use &U) ->
bool {
2952 auto *
I = cast<Instruction>(U.getUser());
2954 if (
auto *Phi = dyn_cast<PHINode>(
I))
2955 BB = Phi->getIncomingBlock(
2957 return BB == PredBB;
2968 Worklist.
insert(InstsToReanalyze.
begin(), InstsToReanalyze.
end());
2969 InstsToReanalyze.
clear();
2972 while (!Worklist.
empty()) {
2979 if (!
I || isa<PHINode>(
I) || !VectorLoop->contains(
I) ||
2980 I->mayHaveSideEffects() ||
I->mayReadFromMemory())
2988 if (
I->getParent() == PredBB) {
2989 Worklist.
insert(
I->op_begin(),
I->op_end());
3003 I->moveBefore(PredBB->getFirstInsertionPt());
3004 Worklist.
insert(
I->op_begin(),
I->op_end());
3015 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
3020 PHINode *NewPhi = cast<PHINode>(State.
get(VPPhi));
3032void LoopVectorizationCostModel::collectLoopScalars(
ElementCount VF) {
3037 "This function should not be visited twice for the same VF");
3043 Scalars[VF].
insert(Uniforms[VF].begin(), Uniforms[VF].end());
3062 "Widening decision should be ready at this moment");
3063 if (
auto *Store = dyn_cast<StoreInst>(MemAccess))
3064 if (
Ptr == Store->getValueOperand())
3067 "Ptr is neither a value or pointer operand");
3073 auto IsLoopVaryingGEP = [&](
Value *
V) {
3084 if (!IsLoopVaryingGEP(
Ptr))
3089 auto *
I = cast<Instruction>(
Ptr);
3096 if (IsScalarUse(MemAccess,
Ptr) &&
3097 all_of(
I->users(), IsaPred<LoadInst, StoreInst>))
3100 PossibleNonScalarPtrs.
insert(
I);
3117 for (
auto &
I : *BB) {
3118 if (
auto *Load = dyn_cast<LoadInst>(&
I)) {
3119 EvaluatePtrUse(Load,
Load->getPointerOperand());
3120 }
else if (
auto *Store = dyn_cast<StoreInst>(&
I)) {
3121 EvaluatePtrUse(Store,
Store->getPointerOperand());
3122 EvaluatePtrUse(Store,
Store->getValueOperand());
3125 for (
auto *
I : ScalarPtrs)
3126 if (!PossibleNonScalarPtrs.
count(
I)) {
3134 auto ForcedScalar = ForcedScalars.
find(VF);
3135 if (ForcedScalar != ForcedScalars.
end())
3136 for (
auto *
I : ForcedScalar->second) {
3137 LLVM_DEBUG(
dbgs() <<
"LV: Found (forced) scalar instruction: " << *
I <<
"\n");
3146 while (
Idx != Worklist.
size()) {
3148 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
3150 auto *Src = cast<Instruction>(Dst->getOperand(0));
3152 auto *J = cast<Instruction>(U);
3153 return !TheLoop->contains(J) || Worklist.count(J) ||
3154 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
3155 IsScalarUse(J, Src));
3158 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Src <<
"\n");
3165 auto *Ind = Induction.first;
3166 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
3175 auto IsDirectLoadStoreFromPtrIndvar = [&](
Instruction *Indvar,
3177 return Induction.second.getKind() ==
3179 (isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
3185 bool ScalarInd =
all_of(Ind->users(), [&](
User *U) ->
bool {
3186 auto *I = cast<Instruction>(U);
3187 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3188 IsDirectLoadStoreFromPtrIndvar(Ind, I);
3196 auto *IndUpdatePhi = dyn_cast<PHINode>(IndUpdate);
3202 bool ScalarIndUpdate =
all_of(IndUpdate->users(), [&](
User *U) ->
bool {
3203 auto *I = cast<Instruction>(U);
3204 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
3205 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
3207 if (!ScalarIndUpdate)
3212 Worklist.
insert(IndUpdate);
3213 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Ind <<
"\n");
3214 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *IndUpdate
3228 switch(
I->getOpcode()) {
3231 case Instruction::Call:
3234 return CallWideningDecisions.at(std::make_pair(cast<CallInst>(
I), VF))
3236 case Instruction::Load:
3237 case Instruction::Store: {
3249 case Instruction::UDiv:
3250 case Instruction::SDiv:
3251 case Instruction::SRem:
3252 case Instruction::URem: {
3270 isa<BranchInst, SwitchInst, PHINode, AllocaInst>(
I))
3283 switch(
I->getOpcode()) {
3286 "instruction should have been considered by earlier checks");
3287 case Instruction::Call:
3291 "should have returned earlier for calls not needing a mask");
3293 case Instruction::Load:
3296 case Instruction::Store: {
3304 case Instruction::UDiv:
3305 case Instruction::SDiv:
3306 case Instruction::SRem:
3307 case Instruction::URem:
3313std::pair<InstructionCost, InstructionCost>
3316 assert(
I->getOpcode() == Instruction::UDiv ||
3317 I->getOpcode() == Instruction::SDiv ||
3318 I->getOpcode() == Instruction::SRem ||
3319 I->getOpcode() == Instruction::URem);
3328 ScalarizationCost = 0;
3343 ScalarizationCost += getScalarizationOverhead(
I, VF);
3363 Value *Op2 =
I->getOperand(1);
3372 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
3374 return {ScalarizationCost, SafeDivisorCost};
3381 "Decision should not be set yet.");
3383 assert(Group &&
"Must have a group.");
3384 unsigned InterleaveFactor = Group->getFactor();
3388 auto &
DL =
I->getDataLayout();
3396 if (VF.
isScalable() && InterleaveFactor != 2)
3401 bool ScalarNI =
DL.isNonIntegralPointerType(ScalarTy);
3402 for (
unsigned Idx = 0;
Idx < InterleaveFactor;
Idx++) {
3407 bool MemberNI =
DL.isNonIntegralPointerType(
MemberTy);
3409 if (MemberNI != ScalarNI)
3412 if (MemberNI && ScalarNI &&
3413 ScalarTy->getPointerAddressSpace() !=
3414 MemberTy->getPointerAddressSpace())
3423 bool PredicatedAccessRequiresMasking =
3426 bool LoadAccessWithGapsRequiresEpilogMasking =
3427 isa<LoadInst>(
I) && Group->requiresScalarEpilogue() &&
3429 bool StoreAccessWithGapsRequiresMasking =
3430 isa<StoreInst>(
I) && (Group->getNumMembers() < Group->getFactor());
3431 if (!PredicatedAccessRequiresMasking &&
3432 !LoadAccessWithGapsRequiresEpilogMasking &&
3433 !StoreAccessWithGapsRequiresMasking)
3440 "Masked interleave-groups for predicated accesses are not enabled.");
3442 if (Group->isReverse())
3454 assert((isa<LoadInst, StoreInst>(
I)) &&
"Invalid memory instruction");
3470 auto &
DL =
I->getDataLayout();
3477void LoopVectorizationCostModel::collectLoopUniforms(
ElementCount VF) {
3484 "This function should not be visited twice for the same VF");
3488 Uniforms[VF].
clear();
3496 auto IsOutOfScope = [&](
Value *V) ->
bool {
3508 auto AddToWorklistIfAllowed = [&](
Instruction *
I) ->
void {
3509 if (IsOutOfScope(
I)) {
3516 dbgs() <<
"LV: Found not uniform due to requiring predication: " << *
I
3520 LLVM_DEBUG(
dbgs() <<
"LV: Found uniform instruction: " << *
I <<
"\n");
3533 auto *
Cmp = dyn_cast<Instruction>(E->getTerminator()->getOperand(0));
3535 AddToWorklistIfAllowed(Cmp);
3544 if (PrevVF.isVector()) {
3545 auto Iter = Uniforms.
find(PrevVF);
3546 if (Iter != Uniforms.
end() && !Iter->second.contains(
I))
3551 if (isa<LoadInst>(
I))
3562 "Widening decision should be ready at this moment");
3564 if (IsUniformMemOpUse(
I))
3567 return (WideningDecision ==
CM_Widen ||
3576 if (isa<StoreInst>(
I) &&
I->getOperand(0) ==
Ptr)
3592 for (
auto &
I : *BB) {
3594 switch (
II->getIntrinsicID()) {
3595 case Intrinsic::sideeffect:
3596 case Intrinsic::experimental_noalias_scope_decl:
3597 case Intrinsic::assume:
3598 case Intrinsic::lifetime_start:
3599 case Intrinsic::lifetime_end:
3601 AddToWorklistIfAllowed(&
I);
3610 if (
auto *EVI = dyn_cast<ExtractValueInst>(&
I)) {
3611 assert(IsOutOfScope(EVI->getAggregateOperand()) &&
3612 "Expected aggregate value to be loop invariant");
3613 AddToWorklistIfAllowed(EVI);
3622 if (IsUniformMemOpUse(&
I))
3623 AddToWorklistIfAllowed(&
I);
3625 if (IsVectorizedMemAccessUse(&
I,
Ptr))
3632 for (
auto *V : HasUniformUse) {
3633 if (IsOutOfScope(V))
3635 auto *
I = cast<Instruction>(V);
3636 bool UsersAreMemAccesses =
all_of(
I->users(), [&](
User *U) ->
bool {
3637 auto *UI = cast<Instruction>(U);
3638 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
3640 if (UsersAreMemAccesses)
3641 AddToWorklistIfAllowed(
I);
3648 while (
Idx != Worklist.
size()) {
3651 for (
auto *OV :
I->operand_values()) {
3653 if (IsOutOfScope(OV))
3657 auto *
OP = dyn_cast<PHINode>(OV);
3662 auto *OI = cast<Instruction>(OV);
3664 auto *J = cast<Instruction>(U);
3665 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
3667 AddToWorklistIfAllowed(OI);
3679 auto *Ind = Induction.first;
3680 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
3684 bool UniformInd =
all_of(Ind->users(), [&](
User *U) ->
bool {
3685 auto *I = cast<Instruction>(U);
3686 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3687 IsVectorizedMemAccessUse(I, Ind);
3694 bool UniformIndUpdate =
all_of(IndUpdate->users(), [&](
User *U) ->
bool {
3695 auto *I = cast<Instruction>(U);
3696 return I == Ind || Worklist.count(I) ||
3697 IsVectorizedMemAccessUse(I, IndUpdate);
3699 if (!UniformIndUpdate)
3703 AddToWorklistIfAllowed(Ind);
3704 AddToWorklistIfAllowed(IndUpdate);
3715 "runtime pointer checks needed. Enable vectorization of this "
3716 "loop with '#pragma clang loop vectorize(enable)' when "
3717 "compiling with -Os/-Oz",
3718 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3724 "runtime SCEV checks needed. Enable vectorization of this "
3725 "loop with '#pragma clang loop vectorize(enable)' when "
3726 "compiling with -Os/-Oz",
3727 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3734 "runtime stride == 1 checks needed. Enable vectorization of "
3735 "this loop without such check by compiling with -Os/-Oz",
3736 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3743bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
3744 if (IsScalableVectorizationAllowed)
3745 return *IsScalableVectorizationAllowed;
3747 IsScalableVectorizationAllowed =
false;
3753 "ScalableVectorizationDisabled",
ORE,
TheLoop);
3757 LLVM_DEBUG(
dbgs() <<
"LV: Scalable vectorization is available\n");
3760 std::numeric_limits<ElementCount::ScalarTy>::max());
3771 "Scalable vectorization not supported for the reduction "
3772 "operations found in this loop.",
3784 "for all element types found in this loop.",
3791 "for safe distance analysis.",
3796 IsScalableVectorizationAllowed =
true;
3801LoopVectorizationCostModel::getMaxLegalScalableVF(
unsigned MaxSafeElements) {
3802 if (!isScalableVectorizationAllowed())
3806 std::numeric_limits<ElementCount::ScalarTy>::max());
3808 return MaxScalableVF;
3816 "Max legal vector width too small, scalable vectorization "
3820 return MaxScalableVF;
3824 unsigned MaxTripCount,
ElementCount UserVF,
bool FoldTailByMasking) {
3826 unsigned SmallestType, WidestType;
3833 unsigned MaxSafeElements =
3837 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElements);
3839 this->MaxSafeElements = MaxSafeElements;
3841 LLVM_DEBUG(
dbgs() <<
"LV: The max safe fixed VF is: " << MaxSafeFixedVF
3843 LLVM_DEBUG(
dbgs() <<
"LV: The max safe scalable VF is: " << MaxSafeScalableVF
3848 auto MaxSafeUserVF =
3849 UserVF.
isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
3866 <<
" is unsafe, clamping to max safe VF="
3867 << MaxSafeFixedVF <<
".\n");
3872 <<
"User-specified vectorization factor "
3873 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3874 <<
" is unsafe, clamping to maximum safe vectorization factor "
3875 <<
ore::NV(
"VectorizationFactor", MaxSafeFixedVF);
3877 return MaxSafeFixedVF;
3882 <<
" is ignored because scalable vectors are not "
3888 <<
"User-specified vectorization factor "
3889 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3890 <<
" is ignored because the target does not support scalable "
3891 "vectors. The compiler will pick a more suitable value.";
3895 <<
" is unsafe. Ignoring scalable UserVF.\n");
3900 <<
"User-specified vectorization factor "
3901 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3902 <<
" is unsafe. Ignoring the hint to let the compiler pick a "
3903 "more suitable value.";
3908 LLVM_DEBUG(
dbgs() <<
"LV: The Smallest and Widest types: " << SmallestType
3909 <<
" / " << WidestType <<
" bits.\n");
3914 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3915 MaxSafeFixedVF, FoldTailByMasking))
3919 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3920 MaxSafeScalableVF, FoldTailByMasking))
3921 if (MaxVF.isScalable()) {
3922 Result.ScalableVF = MaxVF;
3923 LLVM_DEBUG(
dbgs() <<
"LV: Found feasible scalable VF = " << MaxVF
3936 "Not inserting runtime ptr check for divergent target",
3937 "runtime pointer checks needed. Not enabled for divergent target",
3938 "CantVersionLoopWithDivergentTarget",
ORE,
TheLoop);
3947 LLVM_DEBUG(
dbgs() <<
"LV: Found maximum trip count: " << MaxTC <<
'\n');
3950 "loop trip count is one, irrelevant for vectorization",
3959 if (!isa<SCEVCouldNotCompute>(BTC) &&
3965 "Trip count computation wrapped",
3966 "backedge-taken count is -1, loop trip count wrapped to 0",
3971 switch (ScalarEpilogueStatus) {
3973 return computeFeasibleMaxVF(MaxTC, UserVF,
false);
3978 dbgs() <<
"LV: vector predicate hint/switch found.\n"
3979 <<
"LV: Not allowing scalar epilogue, creating predicated "
3980 <<
"vector loop.\n");
3987 dbgs() <<
"LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
3989 LLVM_DEBUG(
dbgs() <<
"LV: Not allowing scalar epilogue due to low trip "
4008 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with a "
4009 "scalar epilogue instead.\n");
4011 return computeFeasibleMaxVF(MaxTC, UserVF,
false);
4022 "No decisions should have been taken at this point");
4032 std::optional<unsigned> MaxPowerOf2RuntimeVF =
4037 MaxPowerOf2RuntimeVF = std::max<unsigned>(
4038 *MaxPowerOf2RuntimeVF,
4041 MaxPowerOf2RuntimeVF = std::nullopt;
4044 if (MaxPowerOf2RuntimeVF && *MaxPowerOf2RuntimeVF > 0) {
4046 "MaxFixedVF must be a power of 2");
4047 unsigned MaxVFtimesIC =
4048 UserIC ? *MaxPowerOf2RuntimeVF * UserIC : *MaxPowerOf2RuntimeVF;
4056 "Invalid loop count");
4058 BackedgeTakenCount, SE->
getOne(BackedgeTakenCount->
getType()));
4064 LLVM_DEBUG(
dbgs() <<
"LV: No tail will remain for any chosen VF.\n");
4078 <<
"LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
4079 "try to generate VP Intrinsics with scalable vector "
4085 "Expected scalable vector factor.");
4095 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with a "
4096 "scalar epilogue instead.\n");
4102 LLVM_DEBUG(
dbgs() <<
"LV: Can't fold tail by masking: don't vectorize\n");
4108 "unable to calculate the loop count due to complex control flow",
4114 "Cannot optimize for size and vectorize at the same time.",
4115 "cannot optimize for size and vectorize at the same time. "
4116 "Enable vectorization of this loop with '#pragma clang loop "
4117 "vectorize(enable)' when compiling with -Os/-Oz",
4122ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
4123 unsigned MaxTripCount,
unsigned SmallestType,
unsigned WidestType,
4125 bool ComputeScalableMaxVF = MaxSafeVF.
isScalable();
4133 "Scalable flags must match");
4141 ComputeScalableMaxVF);
4142 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
4144 << (MaxVectorElementCount * WidestType) <<
" bits.\n");
4146 if (!MaxVectorElementCount) {
4148 << (ComputeScalableMaxVF ?
"scalable" :
"fixed")
4149 <<
" vector registers.\n");
4153 unsigned WidestRegisterMinEC = MaxVectorElementCount.getKnownMinValue();
4154 if (MaxVectorElementCount.isScalable() &&
4158 WidestRegisterMinEC *= Min;
4167 if (MaxTripCount && MaxTripCount <= WidestRegisterMinEC &&
4175 LLVM_DEBUG(
dbgs() <<
"LV: Clamping the MaxVF to maximum power of two not "
4176 "exceeding the constant trip count: "
4177 << ClampedUpperTripCount <<
"\n");
4179 ClampedUpperTripCount,
4180 FoldTailByMasking ? MaxVectorElementCount.isScalable() :
false);
4193 ComputeScalableMaxVF);
4194 MaxVectorElementCountMaxBW = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
4208 for (
int I = RUs.size() - 1;
I >= 0; --
I) {
4209 const auto &MLU = RUs[
I].MaxLocalUsers;
4210 if (
all_of(MLU, [&](
decltype(MLU.front()) &LU) {
4211 return LU.second <= TTI.getNumberOfRegisters(LU.first);
4221 <<
") with target's minimum: " << MinVF <<
'\n');
4237static std::optional<unsigned>
4239 const Function *Fn = L->getHeader()->getParent();
4243 auto Max = Attr.getVScaleRangeMax();
4244 if (Max && Min == Max)
4261 EstimatedVF *= *VScale;
4262 assert(EstimatedVF >= 1 &&
"Estimated VF shouldn't be less than 1");
4266bool LoopVectorizationPlanner::isMoreProfitable(
4268 const unsigned MaxTripCount)
const {
4273 unsigned EstimatedWidthA =
A.Width.getKnownMinValue();
4274 unsigned EstimatedWidthB =
B.Width.getKnownMinValue();
4276 if (
A.Width.isScalable())
4277 EstimatedWidthA *= *VScale;
4278 if (
B.Width.isScalable())
4279 EstimatedWidthB *= *VScale;
4286 A.Width.isScalable() && !
B.Width.isScalable();
4297 return CmpFn(CostA * EstimatedWidthB, CostB * EstimatedWidthA);
4299 auto GetCostForTC = [MaxTripCount,
this](
unsigned VF,
4311 return VectorCost *
divideCeil(MaxTripCount, VF);
4312 return VectorCost * (MaxTripCount / VF) + ScalarCost * (MaxTripCount % VF);
4315 auto RTCostA = GetCostForTC(EstimatedWidthA, CostA,
A.ScalarCost);
4316 auto RTCostB = GetCostForTC(EstimatedWidthB, CostB,
B.ScalarCost);
4317 return CmpFn(RTCostA, RTCostB);
4320bool LoopVectorizationPlanner::isMoreProfitable(
4323 return LoopVectorizationPlanner::isMoreProfitable(
A,
B, MaxTripCount);
4328 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
4330 for (
const auto &Plan : VPlans) {
4334 precomputeCosts(*Plan, VF, CostCtx);
4336 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
4337 for (
auto &R : *VPBB) {
4338 if (!R.cost(VF, CostCtx).isValid())
4344 if (InvalidCosts.
empty())
4352 for (
auto &Pair : InvalidCosts)
4353 if (!Numbering.
count(Pair.first))
4354 Numbering[Pair.first] =
I++;
4357 sort(InvalidCosts, [&Numbering](RecipeVFPair &
A, RecipeVFPair &
B) {
4358 if (Numbering[
A.first] != Numbering[
B.first])
4359 return Numbering[
A.first] < Numbering[
B.first];
4360 const auto &
LHS =
A.second;
4361 const auto &
RHS =
B.second;
4362 return std::make_tuple(
LHS.isScalable(),
LHS.getKnownMinValue()) <
4363 std::make_tuple(
RHS.isScalable(),
RHS.getKnownMinValue());
4375 Subset =
Tail.take_front(1);
4382 [](
const auto *R) {
return Instruction::PHI; })
4383 .Case<VPWidenSelectRecipe>(
4384 [](
const auto *R) {
return Instruction::Select; })
4385 .Case<VPWidenStoreRecipe>(
4386 [](
const auto *R) {
return Instruction::Store; })
4387 .Case<VPWidenLoadRecipe>(
4388 [](
const auto *R) {
return Instruction::Load; })
4389 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
4390 [](
const auto *R) {
return Instruction::Call; })
4393 [](
const auto *R) {
return R->getOpcode(); })
4395 return R->getStoredValues().empty() ? Instruction::Load
4396 : Instruction::Store;
4404 if (Subset ==
Tail ||
Tail[Subset.size()].first != R) {
4405 std::string OutString;
4407 assert(!Subset.empty() &&
"Unexpected empty range");
4408 OS <<
"Recipe with invalid costs prevented vectorization at VF=(";
4409 for (
const auto &Pair : Subset)
4410 OS << (Pair.second == Subset.front().second ?
"" :
", ") << Pair.second;
4412 if (Opcode == Instruction::Call) {
4414 if (
auto *
Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) {
4415 Name =
Int->getIntrinsicName();
4417 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
4419 WidenCall ? WidenCall->getCalledScalarFunction()
4420 : cast<Function>(R->getOperand(R->getNumOperands() - 1)
4421 ->getLiveInIRValue());
4424 OS <<
" call to " <<
Name;
4429 Tail =
Tail.drop_front(Subset.size());
4433 Subset =
Tail.take_front(Subset.size() + 1);
4434 }
while (!
Tail.empty());
4447 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4456 switch (R.getVPDefID()) {
4457 case VPDef::VPDerivedIVSC:
4458 case VPDef::VPScalarIVStepsSC:
4459 case VPDef::VPScalarCastSC:
4460 case VPDef::VPReplicateSC:
4461 case VPDef::VPInstructionSC:
4462 case VPDef::VPCanonicalIVPHISC:
4463 case VPDef::VPVectorPointerSC:
4464 case VPDef::VPReverseVectorPointerSC:
4465 case VPDef::VPExpandSCEVSC:
4466 case VPDef::VPEVLBasedIVPHISC:
4467 case VPDef::VPPredInstPHISC:
4468 case VPDef::VPBranchOnMaskSC:
4470 case VPDef::VPReductionSC:
4471 case VPDef::VPActiveLaneMaskPHISC:
4472 case VPDef::VPWidenCallSC:
4473 case VPDef::VPWidenCanonicalIVSC:
4474 case VPDef::VPWidenCastSC:
4475 case VPDef::VPWidenGEPSC:
4476 case VPDef::VPWidenIntrinsicSC:
4477 case VPDef::VPWidenSC:
4478 case VPDef::VPWidenSelectSC:
4479 case VPDef::VPBlendSC:
4480 case VPDef::VPFirstOrderRecurrencePHISC:
4481 case VPDef::VPWidenPHISC:
4482 case VPDef::VPWidenIntOrFpInductionSC:
4483 case VPDef::VPWidenPointerInductionSC:
4484 case VPDef::VPReductionPHISC:
4485 case VPDef::VPInterleaveSC:
4486 case VPDef::VPWidenLoadEVLSC:
4487 case VPDef::VPWidenLoadSC:
4488 case VPDef::VPWidenStoreEVLSC:
4489 case VPDef::VPWidenStoreSC:
4495 auto WillWiden = [&
TTI, VF](
Type *ScalarTy) {
4513 if (R.getNumDefinedValues() == 0 &&
4514 !isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe, VPInterleaveRecipe>(
4523 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
4525 if (!Visited.
insert({ScalarTy}).second)
4527 if (WillWiden(ScalarTy))
4538 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ExpectedCost <<
".\n");
4539 assert(ExpectedCost.
isValid() &&
"Unexpected invalid cost for scalar loop");
4541 [](std::unique_ptr<VPlan> &
P) {
4544 "Expected Scalar VF to be a candidate");
4551 if (ForceVectorization &&
4552 (VPlans.
size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
4559 for (
auto &
P : VPlans) {
4570 <<
" costs: " << (Candidate.Cost / Width));
4571 if (VF.isScalable())
4580 <<
"LV: Not considering vector loop of width " << VF
4581 <<
" because it will not generate any vector instructions.\n");
4585 if (isMoreProfitable(Candidate, ChosenFactor))
4586 ChosenFactor = Candidate;
4592 "There are conditional stores.",
4593 "store that is conditionally executed prevents vectorization",
4594 "ConditionalStore", ORE, OrigLoop);
4595 ChosenFactor = ScalarCost;
4599 !isMoreProfitable(ChosenFactor, ScalarCost))
dbgs()
4600 <<
"LV: Vectorization seems to be not beneficial, "
4601 <<
"but was forced by a user.\n");
4602 return ChosenFactor;
4606bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
4611 [&](
PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
4621 if (!OrigLoop->
contains(cast<Instruction>(U)))
4625 if (!OrigLoop->
contains(cast<Instruction>(U)))
4659 unsigned Multiplier = VF.
isFixed() ? IC : 1;
4670 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is disabled.\n");
4675 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because no "
4676 "epilogue is allowed.\n");
4682 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
4683 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because the loop "
4684 "is not a supported candidate.\n");
4689 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization factor is forced.\n");
4692 return {ForcedEC, 0, 0};
4694 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization forced factor is not "
4702 dbgs() <<
"LEV: Epilogue vectorization skipped due to opt for size.\n");
4707 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is not profitable for "
4720 const SCEV *RemainingIterations =
nullptr;
4721 unsigned MaxTripCount = 0;
4722 for (
auto &NextVF : ProfitableVFs) {
4729 if ((!NextVF.Width.isScalable() && MainLoopVF.
isScalable() &&
4731 (NextVF.Width.isScalable() &&
4733 (!NextVF.Width.isScalable() && !MainLoopVF.
isScalable() &&
4739 if (!MainLoopVF.
isScalable() && !NextVF.Width.isScalable()) {
4741 if (!RemainingIterations) {
4744 assert(!isa<SCEVCouldNotCompute>(TC) &&
4745 "Trip count SCEV must be computable");
4755 << MaxTripCount <<
"\n");
4759 SE.
getConstant(TCType, NextVF.Width.getKnownMinValue()),
4760 RemainingIterations))
4764 if (Result.Width.isScalar() ||
4765 isMoreProfitable(NextVF, Result, MaxTripCount))
4771 << Result.Width <<
"\n");
4775std::pair<unsigned, unsigned>
4777 unsigned MinWidth = -1U;
4778 unsigned MaxWidth = 8;
4791 MaxWidth = std::min<unsigned>(
4792 MaxWidth, std::min<unsigned>(
4798 MinWidth = std::min<unsigned>(
4799 MinWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
4800 MaxWidth = std::max<unsigned>(
4801 MaxWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
4804 return {MinWidth, MaxWidth};
4812 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
4820 if (!isa<LoadInst>(
I) && !isa<StoreInst>(
I) && !isa<PHINode>(
I))
4825 if (
auto *PN = dyn_cast<PHINode>(&
I)) {
4839 if (
auto *ST = dyn_cast<StoreInst>(&
I))
4840 T = ST->getValueOperand()->getType();
4843 "Expected the load/store/recurrence type to be sized");
4872 LLVM_DEBUG(
dbgs() <<
"LV: Preference for VP intrinsics indicated. "
4873 "Unroll factor forced to be 1.\n");
4892 if (LoopCost == 0) {
4894 assert(LoopCost.
isValid() &&
"Expected to have chosen a VF with valid cost");
4904 for (
auto &Pair : R.MaxLocalUsers) {
4905 Pair.second = std::max(Pair.second, 1U);
4919 unsigned IC = UINT_MAX;
4921 for (
const auto &Pair : R.MaxLocalUsers) {
4926 <<
" register class\n");
4934 unsigned MaxLocalUsers = Pair.second;
4935 unsigned LoopInvariantRegs = 0;
4936 if (R.LoopInvariantRegs.find(Pair.first) != R.LoopInvariantRegs.end())
4937 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
4939 unsigned TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
4943 TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
4944 std::max(1U, (MaxLocalUsers - 1)));
4947 IC = std::min(IC, TmpIC);
4967 unsigned AvailableTC =
4979 std::max(1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
4980 unsigned InterleaveCountLB =
bit_floor(std::max(
4981 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
4982 MaxInterleaveCount = InterleaveCountLB;
4984 if (InterleaveCountUB != InterleaveCountLB) {
4985 unsigned TailTripCountUB =
4986 (AvailableTC % (EstimatedVF * InterleaveCountUB));
4987 unsigned TailTripCountLB =
4988 (AvailableTC % (EstimatedVF * InterleaveCountLB));
4991 if (TailTripCountUB == TailTripCountLB)
4992 MaxInterleaveCount = InterleaveCountUB;
4994 }
else if (BestKnownTC && *BestKnownTC > 0) {
4998 ? (*BestKnownTC) - 1
5006 MaxInterleaveCount =
bit_floor(std::max(
5007 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
5010 assert(MaxInterleaveCount > 0 &&
5011 "Maximum interleave count must be greater than 0");
5015 if (IC > MaxInterleaveCount)
5016 IC = MaxInterleaveCount;
5019 IC = std::max(1u, IC);
5021 assert(IC > 0 &&
"Interleave count must be greater than 0.");
5025 if (VF.
isVector() && HasReductions) {
5026 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving because of reductions.\n");
5034 bool ScalarInterleavingRequiresPredication =
5036 return Legal->blockNeedsPredication(BB);
5038 bool ScalarInterleavingRequiresRuntimePointerCheck =
5044 <<
"LV: IC is " << IC <<
'\n'
5045 <<
"LV: VF is " << VF <<
'\n');
5046 const bool AggressivelyInterleaveReductions =
5048 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
5049 !ScalarInterleavingRequiresPredication && LoopCost <
SmallLoopCost) {
5053 unsigned SmallIC = std::min(IC, (
unsigned)llvm::bit_floor<uint64_t>(
5060 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
5061 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
5067 bool HasSelectCmpReductions =
5070 const RecurrenceDescriptor &RdxDesc = Reduction.second;
5071 RecurKind RK = RdxDesc.getRecurrenceKind();
5072 return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
5073 RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK);
5075 if (HasSelectCmpReductions) {
5076 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving select-cmp reductions.\n");
5086 bool HasOrderedReductions =
5088 const RecurrenceDescriptor &RdxDesc = Reduction.second;
5089 return RdxDesc.isOrdered();
5091 if (HasOrderedReductions) {
5093 dbgs() <<
"LV: Not interleaving scalar ordered reductions.\n");
5098 SmallIC = std::min(SmallIC,
F);
5099 StoresIC = std::min(StoresIC,
F);
5100 LoadsIC = std::min(LoadsIC,
F);
5104 std::max(StoresIC, LoadsIC) > SmallIC) {
5106 dbgs() <<
"LV: Interleaving to saturate store or load ports.\n");
5107 return std::max(StoresIC, LoadsIC);
5112 if (VF.
isScalar() && AggressivelyInterleaveReductions) {
5116 return std::max(IC / 2, SmallIC);
5119 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving to reduce branch cost.\n");
5125 if (AggressivelyInterleaveReductions) {
5175 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5179 for (
Value *U :
I.operands()) {
5180 auto *Instr = dyn_cast<Instruction>(U);
5191 LoopInvariants.
insert(Instr);
5196 EndPoint[Instr] = IdxToInstr.
size();
5214 LLVM_DEBUG(
dbgs() <<
"LV(REG): Calculating max register usage:\n");
5216 const auto &TTICapture =
TTI;
5220 !TTICapture.isElementTypeLegalForScalableVector(Ty)))
5225 for (
unsigned int Idx = 0, Sz = IdxToInstr.
size();
Idx < Sz; ++
Idx) {
5229 InstrList &
List = TransposeEnds[
Idx];
5244 for (
unsigned J = 0, E = VFs.
size(); J < E; ++J) {
5252 if (VFs[J].isScalar()) {
5253 for (
auto *Inst : OpenIntervals) {
5262 for (
auto *Inst : OpenIntervals) {
5275 RegUsage[ClassID] += GetRegUsage(Inst->getType(), VFs[J]);
5280 for (
const auto &Pair :
RegUsage) {
5281 auto &Entry = MaxUsages[J][Pair.first];
5282 Entry = std::max(Entry, Pair.second);
5287 << OpenIntervals.
size() <<
'\n');
5299 for (
auto *Inst : LoopInvariants) {
5302 bool IsScalar =
all_of(Inst->users(), [&](
User *U) {
5303 auto *I = cast<Instruction>(U);
5304 return TheLoop != LI->getLoopFor(I->getParent()) ||
5305 isScalarAfterVectorization(I, VFs[Idx]);
5311 Invariant[ClassID] += GetRegUsage(Inst->getType(), VF);
5315 dbgs() <<
"LV(REG): VF = " << VFs[
Idx] <<
'\n';
5316 dbgs() <<
"LV(REG): Found max usage: " << MaxUsages[
Idx].
size()
5318 for (
const auto &pair : MaxUsages[
Idx]) {
5319 dbgs() <<
"LV(REG): RegisterClass: "
5323 dbgs() <<
"LV(REG): Found invariant usage: " << Invariant.
size()
5325 for (
const auto &pair : Invariant) {
5326 dbgs() <<
"LV(REG): RegisterClass: "
5340bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
Instruction *
I,
5351 "Expecting a scalar emulated instruction");
5352 return isa<LoadInst>(
I) ||
5353 (isa<StoreInst>(
I) &&
5370 PredicatedBBsAfterVectorization[VF].
clear();
5387 !useEmulatedMaskMemRefHack(&
I, VF) &&
5388 computePredInstDiscount(&
I, ScalarCosts, VF) >= 0) {
5392 for (
const auto &[
I,
_] : ScalarCosts) {
5393 auto *CI = dyn_cast<CallInst>(
I);
5394 if (!CI || !CallWideningDecisions.contains({CI, VF}))
5397 CallWideningDecisions[{CI, VF}].Cost = ScalarCosts[CI];
5401 PredicatedBBsAfterVectorization[VF].
insert(BB);
5403 if (Pred->getSingleSuccessor() == BB)
5404 PredicatedBBsAfterVectorization[VF].
insert(Pred);
5413 "Instruction marked uniform-after-vectorization will be predicated");
5431 if (!
I->hasOneUse() || PredInst->
getParent() !=
I->getParent() ||
5450 for (
Use &U :
I->operands())
5451 if (
auto *J = dyn_cast<Instruction>(U.get()))
5463 while (!Worklist.
empty()) {
5467 if (ScalarCosts.contains(
I))
5496 for (
Use &U :
I->operands())
5497 if (
auto *J = dyn_cast<Instruction>(
U.get())) {
5499 "Instruction has non-scalar type");
5500 if (CanBeScalarized(J))
5502 else if (needsExtract(J, VF)) {
5504 cast<VectorType>(
toVectorTy(J->getType(), VF)),
5515 Discount += VectorCost - ScalarCost;
5516 ScalarCosts[
I] = ScalarCost;
5532 ValuesToIgnoreForVF);
5539 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5552 LLVM_DEBUG(
dbgs() <<
"LV: Found an estimated cost of " <<
C <<
" for VF "
5553 << VF <<
" For instruction: " <<
I <<
'\n');
5581 const Loop *TheLoop) {
5583 auto *Gep = dyn_cast<GetElementPtrInst>(
Ptr);
5589 auto *SE = PSE.
getSE();
5590 unsigned NumOperands = Gep->getNumOperands();
5591 for (
unsigned Idx = 1;
Idx < NumOperands; ++
Idx) {
5594 !
Legal->isInductionVariable(Opd))
5603LoopVectorizationCostModel::getMemInstScalarizationCost(
Instruction *
I,
5606 "Scalarization cost of instruction implies vectorization.");
5636 Cost += getScalarizationOverhead(
I, VF);
5652 if (useEmulatedMaskMemRefHack(
I, VF))
5662LoopVectorizationCostModel::getConsecutiveMemOpCost(
Instruction *
I,
5665 auto *VectorTy = cast<VectorType>(
toVectorTy(ValTy, VF));
5670 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5671 "Stride should be 1 or -1 for consecutive memory access");
5683 bool Reverse = ConsecutiveStride < 0;
5691LoopVectorizationCostModel::getUniformMemOpCost(
Instruction *
I,
5696 auto *VectorTy = cast<VectorType>(
toVectorTy(ValTy, VF));
5699 if (isa<LoadInst>(
I)) {
5712 (IsLoopInvariantStoreValue
5719LoopVectorizationCostModel::getGatherScatterCost(
Instruction *
I,
5722 auto *VectorTy = cast<VectorType>(
toVectorTy(ValTy, VF));
5733LoopVectorizationCostModel::getInterleaveGroupCost(
Instruction *
I,
5736 assert(Group &&
"Fail to get an interleaved access group.");
5740 auto *VectorTy = cast<VectorType>(
toVectorTy(ValTy, VF));
5743 unsigned InterleaveFactor = Group->getFactor();
5748 for (
unsigned IF = 0;
IF < InterleaveFactor;
IF++)
5749 if (Group->getMember(IF))
5753 bool UseMaskForGaps =
5755 (isa<StoreInst>(
I) && (Group->getNumMembers() < Group->getFactor()));
5757 InsertPos->
getOpcode(), WideVecTy, Group->getFactor(), Indices,
5761 if (Group->isReverse()) {
5764 "Reverse masked interleaved access not supported.");
5765 Cost += Group->getNumMembers() *
5772std::optional<InstructionCost>
5778 if (InLoopReductions.
empty() || VF.
isScalar() || !isa<VectorType>(Ty))
5779 return std::nullopt;
5780 auto *VectorTy = cast<VectorType>(Ty);
5797 return std::nullopt;
5808 if (!InLoopReductionImmediateChains.
count(RetI))
5809 return std::nullopt;
5813 Instruction *LastChain = InLoopReductionImmediateChains.
at(RetI);
5815 while (!isa<PHINode>(ReductionPhi))
5816 ReductionPhi = InLoopReductionImmediateChains.
at(ReductionPhi);
5848 : dyn_cast<Instruction>(RetI->
getOperand(1));
5853 if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
5866 bool IsUnsigned = isa<ZExtInst>(Op0);
5883 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
5884 return I == RetI ? RedCost : 0;
5888 bool IsUnsigned = isa<ZExtInst>(RedOp);
5897 if (RedCost.
isValid() && RedCost < BaseCost + ExtCost)
5898 return I == RetI ? RedCost : 0;
5899 }
else if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
5904 bool IsUnsigned = isa<ZExtInst>(Op0);
5927 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
5928 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
5936 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
5937 return I == RetI ? RedCost : 0;
5946 if (RedCost.
isValid() && RedCost < MulCost + BaseCost)
5947 return I == RetI ? RedCost : 0;
5951 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
5955LoopVectorizationCostModel::getMemoryInstructionCost(
Instruction *
I,
5973LoopVectorizationCostModel::getScalarizationOverhead(
Instruction *
I,
5986 if (!
RetTy->isVoidTy() &&
6008 for (
auto *V : filterExtractingOperands(Ops, VF))
6011 filterExtractingOperands(Ops, VF), Tys,
CostKind);
6033 auto IsLegalToScalarize = [&]() {
6047 if (isa<LoadInst>(
I))
6052 auto &SI = cast<StoreInst>(
I);
6065 IsLegalToScalarize() ? getUniformMemOpCost(&
I, VF)
6071 if (GatherScatterCost < ScalarizationCost)
6083 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
6084 "Expected consecutive stride.");
6093 unsigned NumAccesses = 1;
6096 assert(Group &&
"Fail to get an interleaved access group.");
6102 NumAccesses = Group->getNumMembers();
6104 InterleaveCost = getInterleaveGroupCost(&
I, VF);
6109 ? getGatherScatterCost(&
I, VF) * NumAccesses
6113 getMemInstScalarizationCost(&
I, VF) * NumAccesses;
6119 if (InterleaveCost <= GatherScatterCost &&
6120 InterleaveCost < ScalarizationCost) {
6122 Cost = InterleaveCost;
6123 }
else if (GatherScatterCost < ScalarizationCost) {
6125 Cost = GatherScatterCost;
6128 Cost = ScalarizationCost;
6162 while (!Worklist.
empty()) {
6164 for (
auto &
Op :
I->operands())
6165 if (
auto *InstOp = dyn_cast<Instruction>(
Op))
6166 if ((InstOp->getParent() ==
I->getParent()) && !isa<PHINode>(InstOp) &&
6167 AddrDefs.
insert(InstOp).second)
6171 for (
auto *
I : AddrDefs) {
6172 if (isa<LoadInst>(
I)) {
6186 for (
unsigned I = 0;
I < Group->getFactor(); ++
I) {
6203 "Trying to set a vectorization decision for a scalar VF");
6205 auto ForcedScalar = ForcedScalars.
find(VF);
6220 for (
auto &ArgOp : CI->
args())
6238 if (VF.
isVector() && ((ForcedScalar != ForcedScalars.
end() &&
6239 ForcedScalar->second.contains(CI)) ||
6250 for (
Type *ScalarTy : ScalarTys)
6259 std::nullopt, *RedCost);
6265 bool UsesMask =
false;
6271 if (
Info.Shape.VF != VF)
6275 if (MaskRequired && !
Info.isMasked())
6279 bool ParamsOk =
true;
6281 switch (Param.ParamKind) {
6300 dyn_cast<SCEVAddRecExpr>(SE->
getSCEV(ScalarParam));
6302 if (!SAR || SAR->getLoop() !=
TheLoop) {
6308 dyn_cast<SCEVConstant>(SAR->getStepRecurrence(*SE));
6336 if (VecFunc && UsesMask && !MaskRequired)
6357 if (VectorCost <=
Cost) {
6362 if (IntrinsicCost <=
Cost) {
6363 Cost = IntrinsicCost;
6378 auto *OpI = dyn_cast<Instruction>(
Op);
6395 return InstsToScalarize[VF][
I];
6398 auto ForcedScalar = ForcedScalars.
find(VF);
6399 if (VF.
isVector() && ForcedScalar != ForcedScalars.
end()) {
6400 auto InstSet = ForcedScalar->second;
6401 if (InstSet.count(
I))
6411 auto HasSingleCopyAfterVectorization = [
this](
Instruction *
I,
6416 auto Scalarized = InstsToScalarize.
find(VF);
6417 assert(Scalarized != InstsToScalarize.
end() &&
6418 "VF not yet analyzed for scalarization profitability");
6419 return !Scalarized->second.count(
I) &&
6421 auto *UI = cast<Instruction>(U);
6422 return !Scalarized->second.count(UI);
6425 (void)HasSingleCopyAfterVectorization;
6434 assert(
I->getOpcode() == Instruction::GetElementPtr ||
6435 I->getOpcode() == Instruction::PHI ||
6436 (
I->getOpcode() == Instruction::BitCast &&
6437 I->getType()->isPointerTy()) ||
6438 HasSingleCopyAfterVectorization(
I, VF));
6448 switch (
I->getOpcode()) {
6449 case Instruction::GetElementPtr:
6455 case Instruction::Br: {
6462 bool ScalarPredicatedBB =
false;
6468 ScalarPredicatedBB =
true;
6470 if (ScalarPredicatedBB) {
6494 case Instruction::Switch: {
6497 auto *Switch = cast<SwitchInst>(
I);
6498 return Switch->getNumCases() *
6501 toVectorTy(Switch->getCondition()->getType(), VF),
6505 case Instruction::PHI: {
6506 auto *Phi = cast<PHINode>(
I);
6518 cast<VectorType>(VectorTy), Mask,
CostKind,
6526 Type *ResultTy = Phi->getType();
6530 auto *HeaderUser = cast_if_present<PHINode>(
6531 find_singleton<User>(Phi->users(), [
this](
User *U,
bool) ->
User * {
6532 auto *Phi = dyn_cast<PHINode>(U);
6533 if (Phi && Phi->getParent() == TheLoop->getHeader())
6539 auto Iter = ReductionVars.
find(HeaderUser);
6540 if (Iter != ReductionVars.end() &&
6542 Iter->second.getRecurrenceKind()))
6545 return (Phi->getNumIncomingValues() - 1) *
6547 Instruction::Select,
toVectorTy(ResultTy, VF),
6557 Intrinsic::vp_merge,
toVectorTy(Phi->getType(), VF),
6558 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
6564 case Instruction::UDiv:
6565 case Instruction::SDiv:
6566 case Instruction::URem:
6567 case Instruction::SRem:
6571 ScalarCost : SafeDivisorCost;
6575 case Instruction::Add:
6576 case Instruction::Sub: {
6584 if (!
RHS ||
RHS->getZExtValue() != 1)
6590 Type *ScalarTy =
I->getType();
6594 {PtrTy, ScalarTy, MaskTy});
6602 case Instruction::FAdd:
6603 case Instruction::FSub:
6604 case Instruction::Mul:
6605 case Instruction::FMul:
6606 case Instruction::FDiv:
6607 case Instruction::FRem:
6608 case Instruction::Shl:
6609 case Instruction::LShr:
6610 case Instruction::AShr:
6611 case Instruction::And:
6612 case Instruction::Or:
6613 case Instruction::Xor: {
6617 if (
I->getOpcode() == Instruction::Mul &&
6628 Value *Op2 =
I->getOperand(1);
6631 Op2 = cast<SCEVConstant>(
PSE.
getSCEV(Op2))->getValue();
6641 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6644 case Instruction::FNeg: {
6647 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6648 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6649 I->getOperand(0),
I);
6651 case Instruction::Select: {
6653 const SCEV *CondSCEV = SE->
getSCEV(SI->getCondition());
6656 const Value *Op0, *Op1;
6673 Type *CondTy = SI->getCondition()->getType();
6678 if (
auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
6679 Pred = Cmp->getPredicate();
6681 CostKind, {TTI::OK_AnyValue, TTI::OP_None},
6682 {TTI::OK_AnyValue, TTI::OP_None},
I);
6684 case Instruction::ICmp:
6685 case Instruction::FCmp: {
6686 Type *ValTy =
I->getOperand(0)->getType();
6689 Instruction *Op0AsInstruction = dyn_cast<Instruction>(
I->getOperand(0));
6690 (void)Op0AsInstruction;
6692 MinBWs[
I] == MinBWs[Op0AsInstruction]) &&
6693 "if both the operand and the compare are marked for "
6694 "truncation, they must have the same bitwidth");
6700 cast<CmpInst>(
I)->getPredicate(),
CostKind,
6701 {TTI::OK_AnyValue, TTI::OP_None},
6702 {TTI::OK_AnyValue, TTI::OP_None},
I);
6704 case Instruction::Store:
6705 case Instruction::Load: {
6710 "CM decision should be taken at this point");
6717 return getMemoryInstructionCost(
I, VF);
6719 case Instruction::BitCast:
6720 if (
I->getType()->isPointerTy())
6723 case Instruction::ZExt:
6724 case Instruction::SExt:
6725 case Instruction::FPToUI:
6726 case Instruction::FPToSI:
6727 case Instruction::FPExt:
6728 case Instruction::PtrToInt:
6729 case Instruction::IntToPtr:
6730 case Instruction::SIToFP:
6731 case Instruction::UIToFP:
6732 case Instruction::Trunc:
6733 case Instruction::FPTrunc: {
6736 assert((isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
6737 "Expected a load or a store!");
6763 unsigned Opcode =
I->getOpcode();
6766 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
6768 if (
StoreInst *Store = dyn_cast<StoreInst>(*
I->user_begin()))
6769 CCH = ComputeCCH(Store);
6772 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
6773 Opcode == Instruction::FPExt) {
6774 if (
LoadInst *Load = dyn_cast<LoadInst>(
I->getOperand(0)))
6775 CCH = ComputeCCH(Load);
6782 auto *Trunc = cast<TruncInst>(
I);
6784 Trunc->getSrcTy(), CCH,
CostKind, Trunc);
6791 Type *SrcScalarTy =
I->getOperand(0)->getType();
6792 Instruction *Op0AsInstruction = dyn_cast<Instruction>(
I->getOperand(0));
6803 (
I->getOpcode() == Instruction::ZExt ||
6804 I->getOpcode() == Instruction::SExt))
6810 case Instruction::Call:
6812 case Instruction::ExtractValue:
6814 case Instruction::Alloca:
6837 auto IsLiveOutDead = [
this, RequiresScalarEpilogue](
User *U) {
6838 return RequiresScalarEpilogue &&
6850 if ((SI = dyn_cast<StoreInst>(&
I)) &&
6853 DeadInvariantStoreOps[SI->getPointerOperand()].push_back(
6854 SI->getValueOperand());
6863 all_of(
I.users(), [
this, IsLiveOutDead](
User *U) {
6864 return VecValuesToIgnore.contains(U) ||
6865 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
6874 if (Group->getInsertPos() == &
I)
6877 DeadInterleavePointerOps.
push_back(PointerOp);
6882 if (
auto *Br = dyn_cast<BranchInst>(&
I)) {
6883 if (Br->isConditional())
6890 for (
unsigned I = 0;
I != DeadInterleavePointerOps.
size(); ++
I) {
6891 auto *
Op = dyn_cast<Instruction>(DeadInterleavePointerOps[
I]);
6893 Instruction *UI = cast<Instruction>(U);
6894 return !VecValuesToIgnore.contains(U) &&
6895 (!isAccessInterleaved(UI) ||
6896 getInterleavedAccessGroup(UI)->getInsertPos() == UI);
6900 DeadInterleavePointerOps.
append(
Op->op_begin(),
Op->op_end());
6903 for (
const auto &[
_, Ops] : DeadInvariantStoreOps) {
6917 (isa<BranchInst>(&
I) && !cast<BranchInst>(&
I)->isConditional());
6920 for (
unsigned I = 0;
I != DeadOps.
size(); ++
I) {
6921 auto *
Op = dyn_cast<Instruction>(DeadOps[
I]);
6924 if (
auto *Br = dyn_cast_or_null<BranchInst>(
Op)) {
6932 if ((ThenEmpty && ElseEmpty) ||
6934 ElseBB->
phis().empty()) ||
6936 ThenBB->
phis().empty())) {
6945 (isa<PHINode>(
Op) &&
Op->getParent() == Header) ||
6948 return !VecValuesToIgnore.contains(U) &&
6949 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
6960 [
this](
User *U) { return ValuesToIgnore.contains(U); }))
6964 DeadOps.
append(
Op->op_begin(),
Op->op_end());
7005 bool InLoop = !ReductionOperations.
empty();
7008 InLoopReductions.
insert(Phi);
7011 for (
auto *
I : ReductionOperations) {
7012 InLoopReductionImmediateChains[
I] = LastChain;
7016 LLVM_DEBUG(
dbgs() <<
"LV: Using " << (InLoop ?
"inloop" :
"out of loop")
7017 <<
" reduction for phi: " << *Phi <<
"\n");
7030 unsigned WidestType;
7039 unsigned N =
RegSize.getKnownMinValue() / WidestType;
7060 <<
"overriding computed VF.\n");
7065 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing. Scalable VF requested, but "
7066 <<
"not supported by the target.\n");
7068 "Scalable vectorization requested but not supported by the target",
7069 "the scalable user-specified vectorization width for outer-loop "
7070 "vectorization cannot be used because the target does not support "
7071 "scalable vectors.",
7072 "ScalableVFUnfeasible", ORE, OrigLoop);
7077 "VF needs to be a power of two");
7079 <<
"VF " << VF <<
" to build VPlans.\n");
7086 return {VF, 0 , 0 };
7090 dbgs() <<
"LV: Not vectorizing. Inner loops aren't supported in the "
7091 "VPlan-native path.\n");
7109 <<
"LV: Invalidate all interleaved groups due to fold-tail by masking "
7110 "which requires masked-interleaved support.\n");
7126 "UserVF ignored because it may be larger than the maximal safe VF",
7127 "InvalidUserVF", ORE, OrigLoop);
7130 "VF needs to be a power of two");
7136 buildVPlansWithVPRecipes(UserVF, UserVF);
7141 "InvalidCost", ORE, OrigLoop);
7155 for (
const auto &VF : VFCandidates) {
7204 for (
unsigned I = 0;
I != IVInsts.
size();
I++) {
7205 for (
Value *
Op : IVInsts[
I]->operands()) {
7206 auto *OpI = dyn_cast<Instruction>(
Op);
7207 if (
Op ==
IV || !OpI || !OrigLoop->
contains(OpI) || !
Op->hasOneUse())
7213 for (
User *U :
IV->users()) {
7214 auto *CI = cast<Instruction>(U);
7235 dbgs() <<
"Cost of " << InductionCost <<
" for VF " << VF
7236 <<
": induction instruction " << *IVInst <<
"\n";
7238 Cost += InductionCost;
7252 auto *
Term = dyn_cast<BranchInst>(EB->getTerminator());
7255 if (
auto *CondI = dyn_cast<Instruction>(
Term->getOperand(0))) {
7256 ExitInstrs.
insert(CondI);
7260 for (
unsigned I = 0;
I != ExitInstrs.
size(); ++
I) {
7267 dbgs() <<
"Cost of " << CondICost <<
" for VF " << VF
7268 <<
": exit condition instruction " << *CondI <<
"\n";
7272 auto *OpI = dyn_cast<Instruction>(
Op);
7273 if (!OpI ||
any_of(OpI->users(), [&ExitInstrs,
this](
User *U) {
7274 return OrigLoop->contains(cast<Instruction>(U)->getParent()) &&
7275 !ExitInstrs.contains(cast<Instruction>(U));
7293 const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop);
7296 auto IsZExtOrSExt = [](
const unsigned Opcode) ->
bool {
7297 return Opcode == Instruction::ZExt || Opcode == Instruction::SExt;
7306 for (
auto *ChainOp : ChainOps) {
7307 for (
Value *
Op : ChainOp->operands()) {
7308 if (
auto *
I = dyn_cast<Instruction>(
Op)) {
7309 ChainOpsAndOperands.insert(
I);
7310 if (
I->getOpcode() == Instruction::Mul) {
7311 auto *Ext0 = dyn_cast<Instruction>(
I->getOperand(0));
7312 auto *Ext1 = dyn_cast<Instruction>(
I->getOperand(1));
7313 if (Ext0 && IsZExtOrSExt(Ext0->getOpcode()) && Ext1 &&
7314 Ext0->getOpcode() == Ext1->getOpcode()) {
7315 ChainOpsAndOperands.insert(Ext0);
7316 ChainOpsAndOperands.insert(Ext1);
7325 auto ReductionCost =
7331 "reduction op visited multiple times");
7333 LLVM_DEBUG(
dbgs() <<
"Cost of " << ReductionCost <<
" for VF " << VF
7334 <<
":\n in-loop reduction " << *
I <<
"\n");
7335 Cost += *ReductionCost;
7350 auto BranchCost = CostCtx.
getLegacyCost(BB->getTerminator(), VF);
7357 for (
Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
7363 dbgs() <<
"Cost of " << ForcedCost <<
" for VF " << VF
7364 <<
": forced scalar " << *ForcedScalar <<
"\n";
7368 for (
const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
7373 dbgs() <<
"Cost of " << ScalarCost <<
" for VF " << VF
7374 <<
": profitable to scalarize " << *Scalarized <<
"\n";
7393 <<
" (Estimated cost per lane: ");
7395 double CostPerLane = double(*
Cost.
getValue()) / EstimatedWidth;
7414 if (
auto *S = dyn_cast<VPSingleDefRecipe>(R))
7415 return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
7416 if (
auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
7417 return &WidenMem->getIngredient();
7423 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
7425 if (
auto *
IR = dyn_cast<VPInterleaveRecipe>(&R)) {
7426 auto *IG =
IR->getInterleaveGroup();
7427 unsigned NumMembers = IG->getNumMembers();
7428 for (
unsigned I = 0;
I != NumMembers; ++
I) {
7436 if (isa<VPPartialReductionRecipe>(&R))
7446 return any_of(TheLoop->
blocks(), [&SeenInstrs, &CostCtx,
7448 return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
7449 if (isa<PHINode>(&I) && BB == TheLoop->getHeader())
7451 return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
7461 VPlan &FirstPlan = *VPlans[0];
7467 ?
"Reciprocal Throughput\n"
7469 ?
"Instruction Latency\n"
7472 ?
"Code Size and Latency\n"
7477 "More than a single plan/VF w/o any plan having scalar VF");
7481 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ScalarCost <<
".\n");
7486 if (ForceVectorization) {
7493 for (
auto &
P : VPlans) {
7500 <<
"LV: Not considering vector loop of width " << VF
7501 <<
" because it will not generate any vector instructions.\n");
7507 if (isMoreProfitable(CurrentFactor, BestFactor))
7508 BestFactor = CurrentFactor;
7511 if (isMoreProfitable(CurrentFactor, ScalarFactor))
7512 ProfitableVFs.push_back(CurrentFactor);
7530 precomputeCosts(BestPlan, BestFactor.
Width, CostCtx);
7533 CostCtx, OrigLoop) ||
7535 CostCtx, OrigLoop)) &&
7536 " VPlan cost model and legacy cost model disagreed");
7538 "when vectorizing, the scalar cost must be computed.");
7549 bool IsUnrollMetadata =
false;
7550 MDNode *LoopID = L->getLoopID();
7554 auto *MD = dyn_cast<MDNode>(LoopID->
getOperand(
I));
7556 const auto *S = dyn_cast<MDString>(MD->getOperand(0));
7558 S && S->getString().starts_with(
"llvm.loop.unroll.disable");
7564 if (!IsUnrollMetadata) {
7566 LLVMContext &Context = L->getHeader()->getContext();
7569 MDString::get(Context,
"llvm.loop.unroll.runtime.disable"));
7575 L->setLoopID(NewLoopID);
7585 auto *EpiRedResult = dyn_cast<VPInstruction>(R);
7586 if (!EpiRedResult ||
7590 auto *EpiRedHeaderPhi =
7591 cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
7593 EpiRedHeaderPhi->getRecurrenceDescriptor();
7594 Value *MainResumeValue =
7595 EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
7598 auto *Cmp = cast<ICmpInst>(MainResumeValue);
7600 "AnyOf expected to start with ICMP_NE");
7602 "AnyOf expected to start by comparing main resume value to original "
7604 MainResumeValue = Cmp->getOperand(0);
7608 Value *Cmp, *OrigResumeV;
7609 bool IsExpectedPattern =
7616 assert(IsExpectedPattern &&
"Unexpected reduction resume pattern");
7617 (void)IsExpectedPattern;
7618 MainResumeValue = OrigResumeV;
7620 PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
7625 using namespace VPlanPatternMatch;
7626 auto IsResumePhi = [](
VPUser *U) {
7628 U, m_VPInstruction<VPInstruction::ResumePhi>(m_VPValue(), m_VPValue()));
7631 "ResumePhi must have a single user");
7632 auto *EpiResumePhiVPI =
7633 cast<VPInstruction>(*
find_if(EpiRedResult->users(), IsResumePhi));
7634 auto *EpiResumePhi = cast<PHINode>(State.
get(EpiResumePhiVPI,
true));
7635 EpiResumePhi->setIncomingValueForBlock(
7644 "Trying to execute plan with unsupported VF");
7646 "Trying to execute plan with unsupported UF");
7648 ((VectorizingEpilogue && ExpandedSCEVs) ||
7649 (!VectorizingEpilogue && !ExpandedSCEVs)) &&
7650 "expanded SCEVs to reuse can only be used during epilogue vectorization");
7664#ifdef EXPENSIVE_CHECKS
7665 assert(DT->
verify(DominatorTree::VerificationLevel::Fast));
7676 assert(VectorizingEpilogue &&
"should only re-use the existing trip "
7677 "count during epilogue vectorization");
7685 if (VectorizingEpilogue)
7691 std::unique_ptr<LoopVersioning> LVer =
nullptr;
7699 LVer = std::make_unique<LoopVersioning>(
7702 State.
LVer = &*LVer;
7727 if (VectorizingEpilogue) {
7729 "Epilogue vectorisation not yet supported with early exits");
7733 &R, State, State.
CFG.
VPBB2IRBB[MiddleVPBB], BypassBlock);
7737 auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
7739 Inc->setIncomingValueForBlock(BypassBlock, V);
7749 std::optional<MDNode *> VectorizedLoopID =
7755 if (VectorizedLoopID) {
7756 L->setLoopID(*VectorizedLoopID);
7782 cast<BranchInst>(State.
CFG.
VPBB2IRBB[MiddleVPBB]->getTerminator());
7783 if (MiddleTerm->isConditional() &&
7787 assert(TripCount > 0 &&
"trip count should not be zero");
7788 const uint32_t Weights[] = {1, TripCount - 1};
7803 const SCEV2ValueTy &ExpandedSCEVs) {
7838 dbgs() <<
"Create Skeleton for epilogue vectorized loop (first pass)\n"
7848 dbgs() <<
"intermediate fn:\n"
7856 assert(Bypass &&
"Expected valid bypass basic block.");
7877 TCCheckBlock->
setName(
"vector.main.loop.iter.check");
7881 DT,
LI,
nullptr,
"vector.ph");
7886 "TC check is expected to dominate Bypass");
7903 return TCCheckBlock;
7914 const SCEV2ValueTy &ExpandedSCEVs) {
7922 nullptr,
"vec.epilog.iter.check",
true);
7924 VecEpilogueIterationCountCheck);
7930 "expected this to be saved from the previous pass.");
7958 for (
PHINode &Phi : VecEpilogueIterationCountCheck->
phis())
7961 for (
PHINode *Phi : PhisInBlock) {
7963 Phi->replaceIncomingBlockWith(
7965 VecEpilogueIterationCountCheck);
7972 return EPI.EpilogueIterationCountCheck == IncB;
7995 "Expected trip count to have been saved in the first pass.");
7999 "saved trip count does not dominate insertion point.");
8010 Value *CheckMinIters =
8014 "min.epilog.iters.check");
8020 unsigned EpilogueLoopStep =
8026 unsigned EstimatedSkipCount = std::min(MainLoopStep, EpilogueLoopStep);
8027 const uint32_t Weights[] = {EstimatedSkipCount,
8028 MainLoopStep - EstimatedSkipCount};
8048 dbgs() <<
"Create Skeleton for epilogue vectorized loop (second pass)\n"
8063 return getVPValueOrAddLiveIn(
Op);
8075 "unsupported switch either exiting loop or continuing to header");
8080 BasicBlock *DefaultDst = SI->getDefaultDest();
8082 for (
auto &
C : SI->cases()) {
8084 assert(!EdgeMaskCache.
contains({Src, Dst}) &&
"Edge masks already created");
8087 if (Dst == DefaultDst)
8089 auto &Compares = Dst2Compares[Dst];
8097 VPValue *DefaultMask =
nullptr;
8098 for (
const auto &[Dst, Conds] : Dst2Compares) {
8107 EdgeMaskCache[{Src, Dst}] = Mask;
8113 DefaultMask = DefaultMask ? Builder.
createOr(DefaultMask, Mask) : Mask;
8117 DefaultMask = Builder.
createNot(DefaultMask);
8121 EdgeMaskCache[{Src, DefaultDst}] = DefaultMask;
8128 std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
8130 if (ECEntryIt != EdgeMaskCache.
end())
8131 return ECEntryIt->second;
8133 if (
auto *SI = dyn_cast<SwitchInst>(Src->getTerminator())) {
8135 assert(EdgeMaskCache.
contains(Edge) &&
"Mask for Edge not created?");
8136 return EdgeMaskCache[Edge];
8142 BranchInst *BI = dyn_cast<BranchInst>(Src->getTerminator());
8143 assert(BI &&
"Unexpected terminator found");
8145 return EdgeMaskCache[Edge] = SrcMask;
8154 return EdgeMaskCache[Edge] = SrcMask;
8157 assert(EdgeMask &&
"No Edge Mask found for condition");
8169 return EdgeMaskCache[Edge] = EdgeMask;
8176 std::pair<BasicBlock *, BasicBlock *> Edge(Src, Dst);
8178 assert(ECEntryIt != EdgeMaskCache.
end() &&
8179 "looking up mask for edge which has not been created");
8180 return ECEntryIt->second;
8188 BlockMaskCache[Header] =
nullptr;
8200 HeaderVPBB->
insert(
IV, NewInsertionPoint);
8207 BlockMaskCache[Header] = BlockMask;
8213 assert(BCEntryIt != BlockMaskCache.
end() &&
8214 "Trying to access mask for block without one.");
8215 return BCEntryIt->second;
8219 assert(OrigLoop->
contains(BB) &&
"Block is not a part of a loop");
8220 assert(BlockMaskCache.
count(BB) == 0 &&
"Mask for block already computed");
8222 "Loop header must have cached block mask");
8228 for (
auto *Predecessor :
8232 BlockMaskCache[BB] = EdgeMask;
8237 BlockMask = EdgeMask;
8241 BlockMask = Builder.
createOr(BlockMask, EdgeMask, {});
8244 BlockMaskCache[BB] = BlockMask;
8250 assert((isa<LoadInst>(
I) || isa<StoreInst>(
I)) &&
8251 "Must be called with either a load or store");
8257 "CM decision should be taken at this point.");
8283 auto *
GEP = dyn_cast<GetElementPtrInst>(
8284 Ptr->getUnderlyingValue()->stripPointerCasts());
8298 GEP ?
GEP->getNoWrapFlags()
8305 if (
LoadInst *Load = dyn_cast<LoadInst>(
I))
8323 "step must be loop invariant");
8327 if (
auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
8330 TruncI->getDebugLoc());
8332 assert(isa<PHINode>(PhiOrTrunc) &&
"must be a phi node here");
8334 IndDesc, Phi->getDebugLoc());
8344 *PSE.
getSE(), *OrigLoop);
8357 Phi->getDebugLoc());
8371 auto IsOptimizableIVTruncate =
8379 IsOptimizableIVTruncate(
I),
Range)) {
8381 auto *
Phi = cast<PHINode>(
I->getOperand(0));
8392 unsigned NumIncoming =
Phi->getNumIncomingValues();
8401 for (
unsigned In = 0;
In < NumIncoming;
In++) {
8406 assert(In == 0 &&
"Both null and non-null edge masks found");
8408 "Distinct incoming values with one having a full mask");
8429 if (
ID && (
ID == Intrinsic::assume ||
ID == Intrinsic::lifetime_end ||
8430 ID == Intrinsic::lifetime_start ||
ID == Intrinsic::sideeffect ||
8431 ID == Intrinsic::pseudoprobe ||
8432 ID == Intrinsic::experimental_noalias_scope_decl))
8438 bool ShouldUseVectorIntrinsic =
8445 if (ShouldUseVectorIntrinsic)
8450 std::optional<unsigned> MaskPos;
8472 Variant = Decision.Variant;
8473 MaskPos = Decision.MaskPos;
8480 if (ShouldUseVectorCall) {
8481 if (MaskPos.has_value()) {
8496 Ops.insert(Ops.
begin() + *MaskPos, Mask);
8507 assert(!isa<BranchInst>(
I) && !isa<PHINode>(
I) && !isa<LoadInst>(
I) &&
8508 !isa<StoreInst>(
I) &&
"Instruction should have been handled earlier");
8523 switch (
I->getOpcode()) {
8526 case Instruction::SDiv:
8527 case Instruction::UDiv:
8528 case Instruction::SRem:
8529 case Instruction::URem: {
8537 auto *SafeRHS = Builder.
createSelect(Mask, Ops[1], One,
I->getDebugLoc());
8543 case Instruction::Add:
8544 case Instruction::And:
8545 case Instruction::AShr:
8546 case Instruction::FAdd:
8547 case Instruction::FCmp:
8548 case Instruction::FDiv:
8549 case Instruction::FMul:
8550 case Instruction::FNeg:
8551 case Instruction::FRem:
8552 case Instruction::FSub:
8553 case Instruction::ICmp:
8554 case Instruction::LShr:
8555 case Instruction::Mul:
8556 case Instruction::Or:
8557 case Instruction::Select:
8558 case Instruction::Shl:
8559 case Instruction::Sub:
8560 case Instruction::Xor:
8561 case Instruction::Freeze:
8568 auto GetConstantViaSCEV = [
this, &SE](
VPValue *
Op) {
8569 Value *
V =
Op->getUnderlyingValue();
8570 if (isa<Constant>(V) || !SE.
isSCEVable(
V->getType()))
8572 auto *
C = dyn_cast<SCEVConstant>(SE.
getSCEV(V));
8578 if (
I->getOpcode() == Instruction::Mul)
8579 NewOps[0] = GetConstantViaSCEV(NewOps[0]);
8581 NewOps[1] = GetConstantViaSCEV(NewOps[1]);
8588VPRecipeBuilder::tryToWidenHistogram(
const HistogramInfo *HI,
8591 unsigned Opcode =
HI->Update->getOpcode();
8592 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
8593 "Histogram update operation must be an Add or Sub");
8608 HI->Store->getDebugLoc());
8614 auto *PN = cast<PHINode>(R->getUnderlyingValue());
8616 getRecipe(cast<Instruction>(PN->getIncomingValueForBlock(OrigLatch)));
8633 if (!IsUniform &&
Range.Start.isScalable() && isa<IntrinsicInst>(
I)) {
8635 case Intrinsic::assume:
8636 case Intrinsic::lifetime_start:
8637 case Intrinsic::lifetime_end:
8659 VPValue *BlockInMask =
nullptr;
8660 if (!IsPredicated) {
8664 LLVM_DEBUG(
dbgs() <<
"LV: Scalarizing and predicating:" << *
I <<
"\n");
8675 assert((
Range.Start.isScalar() || !IsUniform || !IsPredicated ||
8676 (
Range.Start.isScalable() && isa<IntrinsicInst>(
I))) &&
8677 "Should not predicate a uniform recipe");
8679 IsUniform, BlockInMask);
8688 PartialReductionChains;
8690 if (std::optional<std::pair<PartialReductionChain, unsigned>> Pair =
8691 getScaledReduction(Phi, RdxDesc,
Range))
8692 PartialReductionChains.
push_back(*Pair);
8700 for (
const auto &[PartialRdx,
_] : PartialReductionChains)
8701 PartialReductionBinOps.
insert(PartialRdx.BinOp);
8703 auto ExtendIsOnlyUsedByPartialReductions =
8705 return all_of(Extend->users(), [&](
const User *U) {
8706 return PartialReductionBinOps.contains(U);
8712 for (
auto Pair : PartialReductionChains) {
8714 if (ExtendIsOnlyUsedByPartialReductions(Chain.
ExtendA) &&
8715 ExtendIsOnlyUsedByPartialReductions(Chain.
ExtendB))
8716 ScaledReductionMap.
insert(std::make_pair(Chain.
Reduction, Pair.second));
8720std::optional<std::pair<PartialReductionChain, unsigned>>
8721VPRecipeBuilder::getScaledReduction(
PHINode *
PHI,
8729 return std::nullopt;
8733 return std::nullopt;
8735 Value *
Op = Update->getOperand(0);
8736 Value *PhiOp = Update->getOperand(1);
8738 Op = Update->getOperand(1);
8739 PhiOp = Update->getOperand(0);
8742 return std::nullopt;
8744 auto *BinOp = dyn_cast<BinaryOperator>(
Op);
8745 if (!BinOp || !BinOp->hasOneUse())
8746 return std::nullopt;
8752 return std::nullopt;
8754 Instruction *ExtA = cast<Instruction>(BinOp->getOperand(0));
8755 Instruction *ExtB = cast<Instruction>(BinOp->getOperand(1));
8764 unsigned TargetScaleFactor =
8765 PHI->getType()->getPrimitiveSizeInBits().getKnownScalarFactor(
8766 A->getType()->getPrimitiveSizeInBits());
8771 Update->getOpcode(),
A->getType(),
B->getType(),
PHI->getType(),
8772 VF, OpAExtend, OpBExtend,
8773 std::make_optional(BinOp->getOpcode()));
8777 return std::make_pair(Chain, TargetScaleFactor);
8779 return std::nullopt;
8789 if (
auto *Phi = dyn_cast<PHINode>(Instr)) {
8790 if (Phi->getParent() != OrigLoop->
getHeader())
8793 if ((Recipe = tryToOptimizeInductionPHI(Phi,
Operands,
Range)))
8799 "can only widen reductions and fixed-order recurrences here");
8808 unsigned ScaleFactor =
8821 PhisToFix.push_back(PhiRecipe);
8825 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8834 if (
auto *CI = dyn_cast<CallInst>(Instr))
8837 if (
StoreInst *SI = dyn_cast<StoreInst>(Instr))
8839 return tryToWidenHistogram(*HistInfo,
Operands);
8841 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
8847 if (!shouldWiden(Instr,
Range))
8850 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(Instr))
8854 if (
auto *SI = dyn_cast<SelectInst>(Instr)) {
8859 if (
auto *CI = dyn_cast<CastInst>(Instr)) {
8864 return tryToWiden(Instr,
Operands, VPBB);
8871 "Unexpected number of operands for partial reduction");
8882void LoopVectorizationPlanner::buildVPlansWithVPRecipes(
ElementCount MinVF,
8886 auto MaxVFTimes2 = MaxVF * 2;
8888 VFRange SubRange = {VF, MaxVFTimes2};
8889 if (
auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
8901 VPlans.push_back(std::move(Plan));
8911 Value *StartIdx = ConstantInt::get(IdxTy, 0);
8918 Header->insert(CanonicalIVPHI, Header->begin());
8923 Instruction::Add, {CanonicalIVPHI, &Plan.
getVFxUF()}, {HasNUW,
false},
DL,
8925 CanonicalIVPHI->
addOperand(CanonicalIVIncrement);
8938 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
8941 if (WideIntOrFp && WideIntOrFp->getTruncInst())
8948 if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
8950 ID.getKind(), dyn_cast_or_null<FPMathOperator>(
ID.getInductionBinOp()),
8951 Start, VectorTC, Step);
8963 auto *ResumePhiRecipe =
8966 return ResumePhiRecipe;
8977 auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
8981 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8986 auto *ScalarPhiIRI = cast<VPIRInstruction>(&ScalarPhiR);
8987 auto *ScalarPhiI = dyn_cast<PHINode>(&ScalarPhiIRI->getInstruction());
8993 auto *VectorPhiR = cast<VPHeaderPHIRecipe>(Builder.
getRecipe(ScalarPhiI));
8994 if (
auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
8996 WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
8999 "Expected a ResumePhi");
9000 IVEndValues[WideIVR] = ResumePhi->getOperand(0);
9001 ScalarPhiIRI->addOperand(ResumePhi);
9007 assert(cast<VPWidenIntOrFpInductionRecipe>(VectorPhiR)->getTruncInst() &&
9008 "should only skip truncated wide inductions");
9015 bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR);
9016 auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
9018 "Cannot handle loops with uncountable early exits");
9022 "vector.recur.extract");
9023 StringRef Name = IsFOR ?
"scalar.recur.init" :
"bc.merge.rdx";
9026 {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {},
Name);
9039 auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
9042 auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
9046 assert(ExitIRI->getNumOperands() ==
9047 ExitVPBB->getPredecessors().size() &&
9048 "early-exit must update exit values on construction");
9052 Value *IncomingValue = ExitPhi->getIncomingValueForBlock(ExitingBB);
9054 ExitIRI->addOperand(V);
9057 assert(V->getDefiningRecipe()->getParent()->getEnclosingLoopRegion() &&
9058 "Only recipes defined inside a region should need fixing.");
9059 ExitUsersToFix.
insert(ExitIRI);
9062 return ExitUsersToFix;
9070 if (ExitUsersToFix.
empty())
9074 VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
9080 assert(ExitIRI->getNumOperands() == 1 &&
9081 ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB &&
9082 "exit values from early exits must be fixed when branch to "
9083 "early-exit is added");
9084 ExitIRI->extractLastLaneOfOperand(
B);
9097 VPBuilder ScalarPHBuilder(ScalarPHVPBB);
9098 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
9103 auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
9108 "Cannot handle loops with uncountable early exits");
9181 if (ExitIRI->getOperand(0) != FOR)
9185 "vector.recur.extract.for.phi");
9187 ExitUsersToFix.remove(ExitIRI);
9193LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
VFRange &
Range) {
9208 bool RequiresScalarEpilogueCheck =
9215 PSE, RequiresScalarEpilogueCheck,
9222 bool IVUpdateMayOverflow =
false;
9256 "Unsupported interleave factor for scalable vectors");
9261 InterleaveGroups.
insert(IG);
9279 bool NeedsBlends = BB != HeaderBB && !BB->phis().empty();
9280 return Legal->blockNeedsPredication(BB) || NeedsBlends;
9283 RecipeBuilder.collectScaledReductions(
Range);
9285 auto *MiddleVPBB = Plan->getMiddleBlock();
9290 if (VPBB != HeaderVPBB)
9294 if (VPBB == HeaderVPBB)
9295 RecipeBuilder.createHeaderMask();
9296 else if (NeedsMasks)
9297 RecipeBuilder.createBlockInMask(BB);
9304 auto *
Phi = dyn_cast<PHINode>(Instr);
9305 if (Phi &&
Phi->getParent() == HeaderBB) {
9306 Operands.push_back(Plan->getOrAddLiveIn(
9309 auto OpRange = RecipeBuilder.mapToVPValues(
Instr->operands());
9310 Operands = {OpRange.begin(), OpRange.end()};
9317 if ((SI = dyn_cast<StoreInst>(&
I)) &&
9323 SI, RecipeBuilder.mapToVPValues(
Instr->operands()),
9325 Recipe->insertBefore(*MiddleVPBB, MBIP);
9330 RecipeBuilder.tryToCreateWidenRecipe(Instr,
Operands,
Range, VPBB);
9332 Recipe = RecipeBuilder.handleReplication(Instr,
Range);
9334 RecipeBuilder.setRecipe(Instr, Recipe);
9335 if (isa<VPHeaderPHIRecipe>(Recipe)) {
9346 "unexpected recipe needs moving");
9359 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
9360 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
9361 "entry block must be set to a VPRegionBlock having a non-empty entry "
9363 RecipeBuilder.fixHeaderPhis();
9369 auto *IVInc = cast<Instruction>(
9374 cast<VPWidenInductionRecipe>(RecipeBuilder.getRecipe(Phi));
9379 if (
auto *UncountableExitingBlock =
9382 *Plan, *PSE.
getSE(), OrigLoop, UncountableExitingBlock,
9385 "Some exit values in loop with uncountable exit not supported yet",
9386 "UncountableEarlyExitLoopsUnsupportedExitValue", ORE, OrigLoop);
9403 adjustRecipesForReductions(Plan, RecipeBuilder,
Range.Start);
9413 Plan->setName(
"Initial VPlan");
9418 auto *
R = cast<VPRecipeBase>(&U);
9419 return R->getParent()->getParent() ||
9421 Plan->getVectorLoopRegion()->getSinglePredecessor();
9424 auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
9425 auto *ScevStride = dyn_cast<SCEVConstant>(PSE.
getSCEV(StrideV));
9430 auto *CI = Plan->getOrAddLiveIn(
9431 ConstantInt::get(Stride->getType(), ScevStride->getAPInt()));
9432 if (
VPValue *StrideVPV = Plan->getLiveIn(StrideV))
9438 if (!isa<SExtInst, ZExtInst>(U))
9440 VPValue *StrideVPV = Plan->getLiveIn(U);
9443 unsigned BW =
U->getType()->getScalarSizeInBits();
9444 APInt C = isa<SExtInst>(U) ? ScevStride->getAPInt().sext(BW)
9445 : ScevStride->getAPInt().zext(BW);
9446 VPValue *CI = Plan->getOrAddLiveIn(ConstantInt::get(
U->getType(),
C));
9464 bool WithoutRuntimeCheck =
9467 WithoutRuntimeCheck);
9485 true,
false, OrigLoop);
9489 HCFGBuilder.buildHierarchicalCFG();
9497 *PSE.
getSE(), *TLI);
9509 for (
auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
9510 if (isa<VPCanonicalIVPHIRecipe>(&R))
9512 auto *HeaderR = cast<VPHeaderPHIRecipe>(&R);
9513 RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
9537void LoopVectorizationPlanner::adjustRecipesForReductions(
9539 using namespace VPlanPatternMatch;
9540 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
9546 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
9547 if (!PhiR || !PhiR->isInLoop() || (MinVF.
isScalar() && !PhiR->isOrdered()))
9555 "AnyOf and FindLast reductions are not allowed for in-loop reductions");
9560 for (
unsigned I = 0;
I != Worklist.
size(); ++
I) {
9563 auto *UserRecipe = cast<VPSingleDefRecipe>(U);
9564 if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
9565 assert((UserRecipe->getParent() == MiddleVPBB ||
9566 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
9567 "U must be either in the loop region, the middle block or the "
9568 "scalar preheader.");
9571 Worklist.
insert(UserRecipe);
9584 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
9587 unsigned IndexOfFirstOperand;
9595 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
9596 assert(((MinVF.
isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
9597 isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
9598 CurrentLink->getOperand(2) == PreviousLink &&
9599 "expected a call where the previous link is the added operand");
9607 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
9609 LinkVPBB->
insert(FMulRecipe, CurrentLink->getIterator());
9612 auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink);
9613 if (PhiR->isInLoop() && Blend) {
9614 assert(Blend->getNumIncomingValues() == 2 &&
9615 "Blend must have 2 incoming values");
9616 if (Blend->getIncomingValue(0) == PhiR)
9617 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
9619 assert(Blend->getIncomingValue(1) == PhiR &&
9620 "PhiR must be an operand of the blend");
9621 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
9627 if (isa<VPWidenRecipe>(CurrentLink)) {
9628 assert(isa<CmpInst>(CurrentLinkI) &&
9629 "need to have the compare of the select");
9632 assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
9633 "must be a select recipe");
9634 IndexOfFirstOperand = 1;
9637 "Expected to replace a VPWidenSC");
9638 IndexOfFirstOperand = 0;
9643 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
9644 ? IndexOfFirstOperand + 1
9645 : IndexOfFirstOperand;
9646 VecOp = CurrentLink->getOperand(VecOpId);
9647 assert(VecOp != PreviousLink &&
9648 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
9649 (VecOpId - IndexOfFirstOperand)) ==
9651 "PreviousLink must be the operand other than VecOp");
9660 RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp,
9667 CurrentLink->replaceAllUsesWith(RedRecipe);
9669 PreviousLink = RedRecipe;
9676 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
9689 assert(OrigExitingVPV->getDefiningRecipe()->getParent() != LatchVPBB &&
9690 "reduction recipe must be defined before latch");
9692 std::optional<FastMathFlags> FMFs =
9699 return isa<VPInstruction>(&U) &&
9700 cast<VPInstruction>(&U)->getOpcode() ==
9715 assert(!PhiR->
isInLoop() &&
"Unexpected truncated inloop reduction!");
9724 Trunc->
insertAfter(NewExitingVPV->getDefiningRecipe());
9725 Extnd->insertAfter(Trunc);
9727 PhiR->
setOperand(1, Extnd->getVPSingleValue());
9728 NewExitingVPV = Extnd;
9748 FinalReductionResult, [](
VPUser &
User,
unsigned) {
9749 auto *Parent = cast<VPRecipeBase>(&
User)->getParent();
9750 return Parent && !Parent->getParent();
9752 FinalReductionResult->insertBefore(*MiddleVPBB, IP);
9761 return isa<VPWidenSelectRecipe>(U) ||
9762 (isa<VPReplicateRecipe>(U) &&
9763 cast<VPReplicateRecipe>(U)->getUnderlyingInstr()->getOpcode() ==
9764 Instruction::Select);
9770 for (
unsigned I = 0;
I != CmpR->getNumOperands(); ++
I)
9771 if (CmpR->getOperand(
I) == PhiR)
9779 if (
Select->getOperand(1) == PhiR)
9782 Select->getVPSingleValue()->replaceAllUsesWith(
Or);
9803 R->eraseFromParent();
9807 assert(!State.
Lane &&
"VPDerivedIVRecipe being replicated.");
9818 cast_if_present<BinaryOperator>(FPBinOp));
9824 assert((DerivedIV != Index ||
9826 "IV didn't need transforming?");
9834 "uniform recipe shouldn't be predicated");
9840 if (State.
Lane->isFirstLane()) {
9859 if (isa<StoreInst>(UI) &&
9869 for (
unsigned Lane = 0; Lane < EndLane; ++Lane)
9935 LLVM_DEBUG(
dbgs() <<
"LV: cannot compute the outer-loop trip count\n");
9939 Function *
F = L->getHeader()->getParent();
9945 LoopVectorizationCostModel CM(
SEL, L, PSE, LI, LVL, *
TTI, TLI, DB, AC, ORE,
F,
9950 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *
TTI, LVL, CM, IAI, PSE, Hints,
9970 bool AddBranchWeights =
9972 GeneratedRTChecks Checks(PSE, DT, LI,
TTI,
F->getDataLayout(),
9975 VF.
Width, 1, LVL, &CM, BFI, PSI, Checks, BestPlan);
9977 << L->getHeader()->getParent()->getName() <<
"\"\n");
9997 if (
auto *S = dyn_cast<StoreInst>(&Inst)) {
9998 if (S->getValueOperand()->getType()->isFloatTy())
10008 while (!Worklist.
empty()) {
10010 if (!L->contains(
I))
10012 if (!Visited.
insert(
I).second)
10019 if (isa<FPExtInst>(
I) && EmittedRemark.
insert(
I).second)
10022 I->getDebugLoc(), L->getHeader())
10023 <<
"floating point conversion changes vector width. "
10024 <<
"Mixed floating point precision requires an up/down "
10025 <<
"cast that will negatively impact performance.";
10028 for (
Use &
Op :
I->operands())
10029 if (
auto *OpI = dyn_cast<Instruction>(
Op))
10049 <<
"LV: Interleaving only is not profitable due to runtime checks\n");
10105 uint64_t MinTC = std::max(MinTC1, MinTC2);
10107 MinTC =
alignTo(MinTC, IntVF);
10111 dbgs() <<
"LV: Minimum required TC for runtime checks to be profitable:"
10119 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is not beneficial: expected "
10120 "trip count < minimum profitable VF ("
10131 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
10133 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
10146 if (isa<VPCanonicalIVPHIRecipe>(&R))
10149 cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
10153 auto *VPIRInst = cast<VPIRInstruction>(&R);
10154 auto *IRI = dyn_cast<PHINode>(&VPIRInst->getInstruction());
10169 using namespace VPlanPatternMatch;
10176 return match(&R, m_VPInstruction<VPInstruction::ResumePhi>(
10184 "vec.epilog.resume.val");
10191 const SCEV2ValueTy &ExpandedSCEVs,
10195 Header->setName(
"vec.epilog.vector.body");
10204 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
10207 auto *ExpandedVal =
10208 Plan.
getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
10212 ExpandR->eraseFromParent();
10218 if (
auto *
IV = dyn_cast<VPCanonicalIVPHIRecipe>(&R)) {
10225 BasicBlock *MainMiddle = find_singleton<BasicBlock>(
10228 if (BB != EPI.MainLoopIterationCountCheck &&
10229 BB != EPI.EpilogueIterationCountCheck &&
10230 BB != EPI.SCEVSafetyCheck && BB != EPI.MemSafetyCheck)
10235 Type *IdxTy =
IV->getScalarType();
10236 PHINode *EPResumeVal = find_singleton<PHINode>(
10237 L->getLoopPreheader()->phis(),
10239 if (P.getType() == IdxTy &&
10240 P.getIncomingValueForBlock(MainMiddle) == EPI.VectorTripCount &&
10242 P.getIncomingValueForBlock(EPI.MainLoopIterationCountCheck),
10247 assert(EPResumeVal &&
"must have a resume value for the canonical IV");
10251 return isa<VPScalarIVStepsRecipe>(U) ||
10252 isa<VPScalarCastRecipe>(U) ||
10253 isa<VPDerivedIVRecipe>(U) ||
10254 cast<VPInstruction>(U)->getOpcode() ==
10257 "the canonical IV should only be used by its increment or "
10258 "ScalarIVSteps when resetting the start value");
10259 IV->setOperand(0, VPV);
10263 Value *ResumeV =
nullptr;
10265 if (
auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
10266 ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
10267 ->getIncomingValueForBlock(L->getLoopPreheader());
10269 ReductionPhi->getRecurrenceDescriptor();
10287 cast<Instruction>(ResumeV)->
getParent()->getFirstNonPHI());
10296 PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
10301 assert(ResumeV &&
"Must have a resume value");
10303 cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
10309 "VPlan-native path is not enabled. Only process inner loops.");
10312 << L->getHeader()->getParent()->getName() <<
"' from "
10313 << L->getLocStr() <<
"\n");
10318 dbgs() <<
"LV: Loop hints:"
10329 Function *
F = L->getHeader()->getParent();
10340 LLVM_DEBUG(
dbgs() <<
"LV: Loop hints prevent vectorization.\n");
10351 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Cannot prove legality.\n");
10358 "early exit is not enabled",
10359 "UncountableEarlyExitLoopsDisabled",
ORE, L);
10365 "types is not yet supported",
10366 "StructCallVectorizationUnsupported",
ORE, L);
10375 if (!L->isInnermost())
10379 assert(L->isInnermost() &&
"Inner loop expected.");
10389 if (UseInterleaved)
10396 [LoopLatch](
BasicBlock *BB) { return BB != LoopLatch; })) {
10398 "requiring a scalar epilogue is unsupported",
10399 "UncountableEarlyExitUnsupported",
ORE, L);
10413 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop with a very small trip count. "
10414 <<
"This loop is worth vectorizing only if no scalar "
10415 <<
"iteration overheads are incurred.");
10417 LLVM_DEBUG(
dbgs() <<
" But vectorizing was explicitly forced.\n");
10430 LLVM_DEBUG(
dbgs() <<
" But the target considers the trip count too "
10431 "small to consider vectorizing.\n");
10433 "The trip count is below the minial threshold value.",
10434 "loop trip count is too low, avoiding vectorization",
10435 "LowTripCount",
ORE, L);
10444 if (
F->hasFnAttribute(Attribute::NoImplicitFloat)) {
10446 "Can't vectorize when the NoImplicitFloat attribute is used",
10447 "loop not vectorized due to NoImplicitFloat attribute",
10448 "NoImplicitFloat",
ORE, L);
10460 "Potentially unsafe FP op prevents vectorization",
10461 "loop not vectorized due to unsafe FP support.",
10462 "UnsafeFP",
ORE, L);
10467 bool AllowOrderedReductions;
10477 ExactFPMathInst->getDebugLoc(),
10478 ExactFPMathInst->getParent())
10479 <<
"loop not vectorized: cannot prove it is safe to reorder "
10480 "floating-point operations";
10482 LLVM_DEBUG(
dbgs() <<
"LV: loop not vectorized: cannot prove it is safe to "
10483 "reorder floating-point operations\n");
10489 LoopVectorizationCostModel CM(
SEL, L, PSE,
LI, &LVL, *
TTI,
TLI,
DB,
AC,
ORE,
10492 LoopVectorizationPlanner LVP(L,
LI,
DT,
TLI, *
TTI, &LVL, CM, IAI, PSE, Hints,
10500 LVP.
plan(UserVF, UserIC);
10507 bool AddBranchWeights =
10509 GeneratedRTChecks Checks(PSE,
DT,
LI,
TTI,
F->getDataLayout(),
10515 unsigned SelectedIC = std::max(IC, UserIC);
10522 bool ForceVectorization =
10524 if (!ForceVectorization &&
10528 DEBUG_TYPE,
"CantReorderMemOps", L->getStartLoc(),
10530 <<
"loop not vectorized: cannot prove it is safe to reorder "
10531 "memory operations";
10540 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
10541 bool VectorizeLoop =
true, InterleaveLoop =
true;
10543 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is possible but not beneficial.\n");
10544 VecDiagMsg = std::make_pair(
10545 "VectorizationNotBeneficial",
10546 "the cost-model indicates that vectorization is not beneficial");
10547 VectorizeLoop =
false;
10553 LLVM_DEBUG(
dbgs() <<
"LV: Ignoring UserIC, because vectorization and "
10554 "interleaving should be avoided up front\n");
10555 IntDiagMsg = std::make_pair(
10556 "InterleavingAvoided",
10557 "Ignoring UserIC, because interleaving was avoided up front");
10558 InterleaveLoop =
false;
10559 }
else if (IC == 1 && UserIC <= 1) {
10562 IntDiagMsg = std::make_pair(
10563 "InterleavingNotBeneficial",
10564 "the cost-model indicates that interleaving is not beneficial");
10565 InterleaveLoop =
false;
10567 IntDiagMsg.first =
"InterleavingNotBeneficialAndDisabled";
10568 IntDiagMsg.second +=
10569 " and is explicitly disabled or interleave count is set to 1";
10571 }
else if (IC > 1 && UserIC == 1) {
10574 dbgs() <<
"LV: Interleaving is beneficial but is explicitly disabled.");
10575 IntDiagMsg = std::make_pair(
10576 "InterleavingBeneficialButDisabled",
10577 "the cost-model indicates that interleaving is beneficial "
10578 "but is explicitly disabled or interleave count is set to 1");
10579 InterleaveLoop =
false;
10585 if (!VectorizeLoop && InterleaveLoop && LVL.
hasHistograms()) {
10586 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving without vectorization due "
10587 <<
"to histogram operations.\n");
10588 IntDiagMsg = std::make_pair(
10589 "HistogramPreventsScalarInterleaving",
10590 "Unable to interleave without vectorization due to constraints on "
10591 "the order of histogram operations");
10592 InterleaveLoop =
false;
10596 IC = UserIC > 0 ? UserIC : IC;
10600 if (!VectorizeLoop && !InterleaveLoop) {
10604 L->getStartLoc(), L->getHeader())
10605 << VecDiagMsg.second;
10609 L->getStartLoc(), L->getHeader())
10610 << IntDiagMsg.second;
10615 if (!VectorizeLoop && InterleaveLoop) {
10619 L->getStartLoc(), L->getHeader())
10620 << VecDiagMsg.second;
10622 }
else if (VectorizeLoop && !InterleaveLoop) {
10624 <<
") in " << L->getLocStr() <<
'\n');
10627 L->getStartLoc(), L->getHeader())
10628 << IntDiagMsg.second;
10630 }
else if (VectorizeLoop && InterleaveLoop) {
10632 <<
") in " << L->getLocStr() <<
'\n');
10636 bool DisableRuntimeUnroll =
false;
10637 MDNode *OrigLoopID = L->getLoopID();
10639 using namespace ore;
10640 if (!VectorizeLoop) {
10641 assert(IC > 1 &&
"interleave count should not be 1 or 0");
10654 <<
"interleaved loop (interleaved count: "
10655 << NV(
"InterleaveCount", IC) <<
")";
10665 std::unique_ptr<VPlan> BestMainPlan(BestPlan.
duplicate());
10675 EPI, &LVL, &CM,
BFI,
PSI, Checks,
10678 *BestMainPlan, MainILV,
DT,
false);
10687 Checks, BestEpiPlan);
10692 DT,
true, &ExpandedSCEVs);
10693 ++LoopsEpilogueVectorized;
10696 DisableRuntimeUnroll =
true;
10700 PSI, Checks, BestPlan);
10708 DisableRuntimeUnroll =
true;
10719 "DT not preserved correctly");
10721 std::optional<MDNode *> RemainderLoopID =
10724 if (RemainderLoopID) {
10725 L->setLoopID(*RemainderLoopID);
10727 if (DisableRuntimeUnroll)
10751 bool Changed =
false, CFGChanged =
false;
10758 for (
const auto &L : *
LI)
10759 Changed |= CFGChanged |=
10770 LoopsAnalyzed += Worklist.
size();
10773 while (!Worklist.
empty()) {
10818 if (!Result.MadeAnyChange)
10832 if (Result.MadeCFGChange) {
10848 OS, MapClassName2PassName);
10851 OS << (InterleaveOnlyWhenForced ?
"" :
"no-") <<
"interleave-forced-only;";
10852 OS << (VectorizeOnlyWhenForced ?
"" :
"no-") <<
"vectorize-forced-only;";
static unsigned getIntrinsicID(const SDNode *N)
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefAnalysis InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This is the interface for a simple mod/ref and alias analysis over globals.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
Legalize the Machine IR a function s Machine IR
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
loop Loop Strength Reduction
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)
Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions,...
static void addRuntimeUnrollDisableMetaData(Loop *L)
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static VPInstruction * addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC)
Create and return a ResumePhi for WideIV, unless it is truncated.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, const TargetTransformInfo &TTI, PredicatedScalarEvolution &PSE, ScalarEpilogueLowering SEL)
static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
const char LLVMLoopVectorizeFollowupAll[]
static SetVector< VPIRInstruction * > collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan)
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, SetVector< VPIRInstruction * > &ExitUsersToFix)
Handle users in the exit block for first order reductions in the original exit block.
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, DebugLoc DL)
static std::optional< unsigned > getVScaleForTuning(const Loop *L, const TargetTransformInfo &TTI)
Convenience function that returns the value of vscale_range iff vscale_range.min == vscale_range....
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static constexpr uint32_t MemCheckBypassWeights[]
cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static bool planContainsAdditionalSimplifications(VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop)
Return true if the original loop \ TheLoop contains any instructions that do not have corresponding r...
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static Type * maybeVectorizeType(Type *Elt, ElementCount VF)
static std::optional< unsigned > getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
Returns "best known" trip count for the specified loop L as defined by the following procedure: 1) Re...
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
const char VerboseDebug[]
static void fixReductionScalarResumeWhenVectorizingEpilog(VPRecipeBase *R, VPTransformState &State, BasicBlock *LoopMiddleBlock, BasicBlock *BypassBlock)
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static constexpr uint32_t SCEVCheckBypassWeights[]
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
const char LLVMLoopVectorizeFollowupVectorized[]
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
const char LLVMLoopVectorizeFollowupEpilogue[]
static void preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, const EpilogueLoopVectorizationInfo &EPI)
Prepare Plan for vectorizing the epilogue loop.
static bool useActiveLaneMask(TailFoldingStyle Style)
static unsigned getEstimatedRuntimeVF(const Loop *L, const TargetTransformInfo &TTI, ElementCount VF)
This function attempts to return a value that represents the vectorization factor at runtime.
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static void cse(BasicBlock *BB)
Perform cse of induction variable instructions.
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static void addUsersInExitBlocks(VPlan &Plan, const SetVector< VPIRInstruction * > &ExitUsersToFix)
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(false), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
Create an analysis remark that explains why vectorization failed.
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file defines the VPlanHCFGBuilder class which contains the public interface (buildHierarchicalCF...
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
static const uint32_t IV[8]
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
int64_t getSExtValue() const
Get sign extended value.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
void registerAssumption(AssumeInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BinaryOps getOpcode() const
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Value * getCondition() const
Represents analyses that only rely on functions' control flow.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getFalse(LLVMContext &Context)
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
const ValueT & at(const_arg_type_t< KeyT > Val) const
at - Return the entry for the specified key, or abort if no such entry exists.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
DomTreeNodeBase * getIDom() const
Analysis pass which computes a DominatorTree.
bool verify(VerificationLevel VL=VerificationLevel::Full) const
verify - checks if the tree is correct.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
BasicBlock * emitMinimumVectorEpilogueIterCountCheck(BasicBlock *Bypass, BasicBlock *Insert)
Emits an iteration count bypass check after the main vector loop has finished to see if there are any...
void printDebugTracesAtEnd() override
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (ie the ...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
void printDebugTracesAtEnd() override
BasicBlock * createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final
Implements the interface for creating a vectorized skeleton using the main loop strategy (ie the firs...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * emitIterationCountCheck(BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Convenience struct for specifying and reasoning about fast-math flags.
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags inBounds()
static GEPNoWrapFlags none()
Common base class shared among various IRBuilders.
ConstantInt * getTrue()
Get the constant value for i1 true.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
An extension of the inner loop vectorizer that creates a skeleton for a vectorized loop that has its ...
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
BasicBlock * createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs) final
Create a new empty loop that will contain vectorized instructions later on, while the old loop will b...
virtual BasicBlock * createEpilogueVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs)=0
The interface for creating a vectorized skeleton using one of two different strategies,...
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
Value * TripCount
Trip count of the original loop.
void sinkScalarOperands(Instruction *PredInst)
Iteratively sink the scalarized operands of a predicated instruction into the block that was created ...
const TargetLibraryInfo * TLI
Target Library Info.
ElementCount MinProfitableTripCount
const TargetTransformInfo * TTI
Target Transform Info.
Value * VectorTripCount
Trip count of the widened loop (TripCount - TripCount % (VF*UF))
bool areSafetyChecksAdded()
BasicBlock * emitSCEVChecks(BasicBlock *Bypass)
Emit a bypass check to see if all of the SCEV assumptions we've had to make are correct.
virtual BasicBlock * createVectorizedLoopSkeleton(const SCEV2ValueTy &ExpandedSCEVs)
Create a new empty loop that will contain vectorized instructions later on, while the old loop will b...
LoopVectorizationCostModel * Cost
The profitablity analysis.
BasicBlock * AdditionalBypassBlock
The additional bypass block which conditionally skips over the epilogue loop after executing the main...
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
BasicBlock * LoopMiddleBlock
Middle Block between the vector and the scalar.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
void scalarizeInstruction(const Instruction *Instr, VPReplicateRecipe *RepRecipe, const VPLane &Lane, VPTransformState &State)
A helper function to scalarize a single Instruction in the innermost loop.
SmallVector< Instruction *, 4 > PredicatedInstructions
Store instructions that were predicated.
DenseMap< PHINode *, Value * > Induction2AdditionalBypassValue
Mapping of induction phis to their additional bypass values.
void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB)
Introduces a new VPIRBasicBlock for CheckIRBB to Plan between the vector preheader and its predecesso...
void createVectorLoopSkeleton(StringRef Prefix)
Emit basic blocks (prefixed with Prefix) for the iteration check, vector loop preheader,...
BasicBlock * emitMemRuntimeChecks(BasicBlock *Bypass)
Emit bypass checks to check any memory assumptions we may have made.
BasicBlock * LoopScalarPreHeader
The scalar-loop preheader.
void createInductionAdditionalBypassValues(const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)
Create and record the values for induction variables to resume coming from the additional bypass bloc...
VPBlockBase * VectorPHVPB
The vector preheader block of Plan, used as target for check blocks introduced during skeleton creati...
LoopVectorizationLegality * Legal
The legality analysis.
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo *TTI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor, LoopVectorizationLegality *LVL, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks, VPlan &Plan)
void emitIterationCountCheck(BasicBlock *Bypass)
Emit a bypass check to see if the vector trip count is zero, including if it overflows.
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
Value * getInductionAdditionalBypassValue(PHINode *OrigPhi) const
induction header phi.
BasicBlock * getAdditionalBypassBlock() const
Return the additional bypass block which targets the scalar loop by skipping the epilogue loop after ...
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
bool OptForSizeBasedOnProfile
void fixVectorizedLoop(VPTransformState &State)
Fix the vectorized code, taking care of header phi's, and more.
BasicBlock * LoopVectorPreHeader
The vector-loop preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
Value * getOrCreateVectorTripCount(BasicBlock *InsertBlock)
Returns (and creates if needed) the trip count of the widened loop.
IRBuilder Builder
The builder that we use.
void fixNonInductionPHIs(VPTransformState &State)
Fix the non-induction PHIs in Plan.
unsigned UF
The vectorization unroll factor to use.
SmallVector< BasicBlock *, 4 > LoopBypassBlocks
A list of all bypass blocks. The first block is the entry of the loop.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
std::optional< CostType > getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void replaceSuccessorWith(BasicBlock *OldBB, BasicBlock *NewBB)
Replace specified successor OldBB to point at the provided block.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool isInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleave group.
bool invalidateGroups()
Invalidate groups, e.g., in case all blocks in loop will be predicated contrary to original assumptio...
iterator_range< SmallPtrSetIterator< llvm::InterleaveGroup< Instruction > * > > getInterleaveGroups()
void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
const DenseMap< Value *, const SCEV * > & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
BlockT * getUniqueLatchExitBlock() const
Return the unique exit block for the latch, or null if there are multiple different exit blocks or th...
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getHeader() const
unsigned getLoopDepth() const
Return the nesting level of this loop.
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool isEpilogueVectorizationProfitable(const ElementCount VF, const unsigned IC) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
bool requiresScalarEpilogue(VFRange Range) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
bool hasPredStores() const
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
bool usePredicatedReductionSelect(unsigned Opcode, Type *PhiTy) const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
std::optional< unsigned > getMaxSafeElements() const
Return maximum safe number of elements to be processed per vector iteration, which do not prevent sto...
const TargetTransformInfo & TTI
Vector target information.
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI)
const Function * TheFunction
LoopVectorizationLegality * Legal
Vectorization legality.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
std::optional< InstructionCost > getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy) const
Return the cost of instructions in an inloop reduction pattern, if I is part of that pattern.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
DemandedBits * DB
Demanded bits analysis.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
Loop * TheLoop
The loop that we evaluate.
TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallVector< RegisterUsage, 8 > calculateRegisterUsage(ArrayRef< ElementCount > VFs)
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool runtimeChecksRequired()
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
void collectUniformsAndScalars(ElementCount VF)
Collect Uniform and Scalar values for the given VF.
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
unsigned selectInterleaveCount(ElementCount VF, InstructionCost LoopCost)
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
unsigned getNumStores() const
bool isInvariantStoreOfReduction(StoreInst *SI)
Returns True if given store is a final invariant store of one of the reductions found in the loop.
bool hasVectorCallVariants() const
Returns true if there is at least one function call in the loop which has a vectorized variant availa...
uint64_t getMaxSafeVectorWidthInBits() const
RecurrenceSet & getFixedOrderRecurrences()
Return the fixed-order recurrences found in the loop.
bool isInvariantAddressOfReduction(Value *V)
Returns True if given address is invariant and is used to store recurrent expression.
bool blockNeedsPredication(BasicBlock *BB) const
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
int isConsecutivePtr(Type *AccessTy, Value *Ptr) const
Check if this pointer is consecutive when vectorizing.
std::optional< const HistogramInfo * > getHistogramInfo(Instruction *I) const
Returns a HistogramInfo* for the given instruction if it was determined to be part of a load -> updat...
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
bool isReductionVariable(PHINode *PN) const
Returns True if PN is a reduction variable in this loop.
bool isFixedOrderRecurrence(const PHINode *Phi) const
Returns True if Phi is a fixed-order recurrence in this loop.
const InductionDescriptor * getPointerInductionDescriptor(PHINode *Phi) const
Returns a pointer to the induction descriptor, if Phi is pointer induction.
const InductionDescriptor * getIntOrFpInductionDescriptor(PHINode *Phi) const
Returns a pointer to the induction descriptor, if Phi is an integer or floating point induction.
bool isInductionPhi(const Value *V) const
Returns True if V is a Phi node of an induction variable in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool hasStructVectorCall() const
Returns true if there is at least one function call in the loop which returns a struct type and needs...
bool isInvariant(Value *V) const
Returns true if V is invariant across all loop iterations according to SCEV.
const ReductionList & getReductionVars() const
Returns the reduction variables found in the loop.
bool isSafeForAnyVectorWidth() const
unsigned getNumLoads() const
bool canFoldTailByMasking() const
Return true if we can vectorize this loop while folding its tail by masking.
void prepareToFoldTailByMasking()
Mark all respective loads/stores for masking.
Type * getWidestInductionType()
Returns the widest induction type.
bool hasUncountableEarlyExit() const
Returns true if the loop has exactly one uncountable early exit, i.e.
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
bool isUniformMemOp(Instruction &I, ElementCount VF) const
A uniform memory op is a load or store which accesses the same memory location on all VF lanes,...
BasicBlock * getUncountableEarlyExitingBlock() const
Returns the uncountable early exiting block, if there is exactly one.
bool isMaskRequired(const Instruction *I) const
Returns true if vector representation of the instruction I requires mask.
const RuntimePointerChecking * getRuntimePointerChecking() const
Returns the information that we collected about runtime memory check.
Planner drives the vectorization process after having passed Legality checks.
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
VectorizationFactor computeBestVF()
Compute and return the most profitable vectorization factor.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
void printPlans(raw_ostream &O)
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool VectorizingEpilogue, const DenseMap< const SCEV *, Value * > *ExpandedSCEVs=nullptr)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Instruction * getExactFPInst()
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
bool isScalableVectorizationDisabled() const
enum ForceKind getForce() const
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
bool allowReordering() const
When enabling loop hints are provided we allow the vectorizer to change the order of operations that ...
void emitRemarkWithHints() const
Dumps all the hint information.
bool isPotentiallyUnsafe() const
ElementCount getWidth() const
@ FK_Enabled
Forcing enabled.
@ FK_Undefined
Not selected.
@ FK_Disabled
Forcing disabled.
unsigned getPredicate() const
void setAlreadyVectorized()
Mark the loop L as already vectorized by setting the width to 1.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
unsigned getInterleave() const
void prepareNoAliasMetadata()
Set up the aliasing scopes based on the memchecks.
Represents a single loop in the control flow graph.
bool hasLoopInvariantOperands(const Instruction *I) const
Return true if all the operands of the specified instruction are loop invariant.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
const MDOperand & getOperand(unsigned I) const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDString * get(LLVMContext &Context, StringRef Str)
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
bool contains(const KeyT &Key) const
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
static unsigned getIncomingValueNumForOperand(unsigned i)
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
const SCEVPredicate & getPredicate() const
unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
const SCEV * getSymbolicMaxBackedgeTakenCount()
Get the (predicated) symbolic max backedge count for the analyzed loop.
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
void preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
bool hasProfileSummary() const
Returns true if profile summary is available.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isFindLastIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
Value * getSentinelValue() const
Returns the sentinel value for FindLastIV recurrences to replace the start value.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
bool Need
This flag indicates if we need to add the runtime check.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class represents a constant integer value.
const APInt & getAPInt() const
Helper to remove instructions inserted during SCEV expansion, unless they are marked as used.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
This class represents an assumption made using SCEV expressions which can be checked at run-time.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
bool isOne() const
Return true if the expression is a constant one.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
iterator end()
Get an iterator to the end of the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
iterator begin()
Get an iterator to the beginning of the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
value_type pop_back_val()
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isTokenTy() const
Return true if this is 'token'.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::iterator iterator
Instruction iterators...
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
iterator begin()
Recipe iterator methods.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
void setName(const Twine &newName)
size_t getNumSuccessors() const
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
RAII object that stores the current insertion point and restores it when the object is destroyed.
VPlan-based builder utility analogous to IRBuilder.
VPValue * createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, DebugLoc DL={}, const Twine &Name="")
Create a new ICmp VPInstruction with predicate Pred and operands A and B.
VPValue * createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL={}, const Twine &Name="")
VPBasicBlock * getInsertBlock() const
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPScalarCastRecipe * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
VPInstruction * createOverflowingOp(unsigned Opcode, std::initializer_list< VPValue * > Operands, VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPValue * createNot(VPValue *Operand, DebugLoc DL={}, const Twine &Name="")
VPValue * createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL={}, const Twine &Name="")
VPValue * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL={}, const Twine &Name="", std::optional< FastMathFlags > FMFs=std::nullopt)
void setInsertPoint(VPBasicBlock *TheBB)
This specifies that created VPInstructions should be appended to the end of the specified block.
Canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
VPValue * getStartValue() const
A recipe representing a sequence of load -> update -> store as part of a histogram operation.
A special type of VPBasicBlock that wraps an existing IR basic block.
A recipe to wrap on original IR instruction not to be modified during execution, execept for PHIs.
This is a concrete Recipe that models a single VPlan-level instruction.
@ ResumePhi
Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
A recipe for forming partial reductions.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPRecipeBase * tryToCreatePartialReduction(Instruction *Reduction, ArrayRef< VPValue * > Operands)
Create and return a partial reduction recipe for a reduction instruction along with binary operation ...
VPValue * createEdgeMask(BasicBlock *Src, BasicBlock *Dst)
A helper function that computes the predicate of the edge between SRC and DST.
VPReplicateRecipe * handleReplication(Instruction *I, VFRange &Range)
Build a VPReplicationRecipe for I.
void createSwitchEdgeMasks(SwitchInst *SI)
Create an edge mask for every destination of cases and/or default.
VPValue * getBlockInMask(BasicBlock *BB) const
Returns the entry mask for the block BB.
VPValue * getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const
A helper that returns the previously computed predicate of the edge between SRC and DST.
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values of the range Operands to their corresponding VPValues.
void fixHeaderPhis()
Add the incoming values from the backedge to reduction & first-order recurrence cross-iteration phis.
VPRecipeBase * tryToCreateWidenRecipe(Instruction *Instr, ArrayRef< VPValue * > Operands, VFRange &Range, VPBasicBlock *VPBB)
Create and return a widened recipe for I if one can be created within the given VF Range.
VPValue * getVPValueOrAddLiveIn(Value *V)
void createHeaderMask()
Create the mask for the vector loop header block.
std::optional< unsigned > getScalingForReduction(const Instruction *ExitInst)
void createBlockInMask(BasicBlock *BB)
A helper function that computes the predicate of the block BB, assuming that the header block of the ...
void collectScaledReductions(VFRange &Range)
Find all possible partial reductions in the loop and track all of those that are valid so recipes can...
VPRecipeBase * getRecipe(Instruction *I)
Return the recipe created for given ingredient.
void setFlags(Instruction *I) const
Set the IR flags for I.
A recipe for handling reduction phis.
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
const RecurrenceDescriptor & getRecurrenceDescriptor() const
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
A recipe to compute the pointers for widened memory accesses of IndexTy in reverse order.
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
unsigned getNumOperands() const
VPValue * getOperand(unsigned N) const
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void replaceAllUsesWith(VPValue *New)
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A recipe to compute the pointers for widened memory accesses of IndexTy.
A recipe for widening Call instructions using library calls.
A Recipe for widening the canonical induction variable of the vector loop.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
VPValue * getStepValue()
Returns the step value of the induction.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A recipe for widening vector intrinsics.
A common base class for widening memory operations.
A recipe for handling phis that are widened in the vector loop.
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Main class to build the VPlan H-CFG for an incoming IR.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
void prepareToExecute(Value *TripCount, Value *VectorTripCount, VPTransformState &State)
Prepare the plan for execution, setting up the required live-in values.
VPBasicBlock * getEntry()
VPValue & getVectorTripCount()
The vector trip count.
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
VPValue & getVF()
Returns the VF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
static VPlanPtr createInitialVPlan(Type *InductionTy, PredicatedScalarEvolution &PSE, bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop)
Create initial VPlan, having an "entry" VPBasicBlock (wrapping original scalar pre-header) which cont...
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
auto getExitBlocks()
Return an iterator range over the VPIRBasicBlock wrapping the exit blocks of the VPlan,...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
const VPBasicBlock * getMiddleBlock() const
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
void setEntry(VPBasicBlock *VPBB)
VPIRBasicBlock * createVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock from IRBB containing VPIRInstructions for all instructions in IRBB,...
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUser() const
Return true if there is exactly one user of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
void replaceUsesWithIf(Value *New, llvm::function_ref< bool(Use &U)> ShouldReplace)
Go through the uses list for this definition and make each use point to "V" if the callback ShouldRep...
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
int getNumOccurrences() const
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isNonZero() const
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ PredicateElseScalarEpilogue
@ PredicateOrDontVectorize
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ C
The default llvm calling convention, compatible with C.
ID ArrayRef< Type * > Tys
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
OneUse_match< T > m_OneUse(const T &SubPattern)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
NodeAddr< PhiNode * > Phi
const_iterator begin(StringRef path LLVM_LIFETIME_BOUND, Style style=Style::native)
Get begin iterator over path.
const_iterator end(StringRef path LLVM_LIFETIME_BOUND)
Get end iterator over path.
bool isUniformAfterVectorization(const VPValue *VPV)
Returns true if VPV is uniform after vectorization.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
This is an optimization pass for GlobalISel generic memory operations.
bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Returns a loop's estimated trip count based on branch weight metadata.
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
auto pred_end(const MachineBasicBlock *BB)
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
auto successors(const MachineBasicBlock *BB)
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
std::pair< Instruction *, ElementCount > InstructionVFPair
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
std::optional< MDNode * > makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef< StringRef > FollowupAttrs, const char *InheritOptionsAttrsPrefix="", bool AlwaysNew=false)
Create a new loop identifier for a loop created from a loop transformation.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
auto map_range(ContainerTy &&C, FuncTy F)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
void sort(IteratorTy Start, IteratorTy End)
std::unique_ptr< VPlan > VPlanPtr
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableLoopVectorization
bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop, Loop *RemainderLoop, uint64_t UF)
Set weights for UnrolledLoop and RemainderLoop based on weights for OrigLoop and the following distri...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
DWARFExpression::Operation Op
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
auto pred_begin(const MachineBasicBlock *BB)
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
unsigned getReciprocalPredBlockProb()
A helper function that returns the reciprocal of the block probability of predicated blocks.
bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
cl::opt< bool > EnableLoopInterleaving
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
A special type used by analysis passes to provide an address that identifies that particular analysis...
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
Encapsulate information regarding vectorization of a loop and its epilogue.
BasicBlock * SCEVSafetyCheck
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF, VPlan &EpiloguePlan)
BasicBlock * MemSafetyCheck
BasicBlock * MainLoopIterationCountCheck
BasicBlock * EpilogueIterationCountCheck
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
std::optional< unsigned > MaskPos
A struct that represents some properties of the register usage of a loop.
SmallMapVector< unsigned, unsigned, 4 > MaxLocalUsers
Holds the maximum number of concurrent live intervals in the loop.
SmallMapVector< unsigned, unsigned, 4 > LoopInvariantRegs
Holds the number of loop invariant values that are used in the loop.
LoopVectorizeResult runImpl(Function &F)
bool processLoop(Loop *L)
LoopAccessInfoManager * LAIs
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LoopVectorizePass(LoopVectorizeOptions Opts={})
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
Storage for information about made changes.
A chain of instructions that form a partial reduction.
Instruction * Reduction
The top-level binary operation that forms the reduction to a scalar after the loop body.
Instruction * ExtendA
The extension of each of the inner binary operation's operands.
A CRTP mix-in to automatically provide informational APIs needed for passes.
A MapVector that performs no allocations if smaller than a certain size.
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
A recipe for widening load operations, using the address to load from and an optional mask.
A recipe for widening select instructions.
A recipe for widening store operations, using the stored value, the address to store to and an option...
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static bool HoistRuntimeChecks