165#define LV_NAME "loop-vectorize"
166#define DEBUG_TYPE LV_NAME
172STATISTIC(LoopsVectorized,
"Number of loops vectorized");
173STATISTIC(LoopsAnalyzed,
"Number of loops analyzed for vectorization");
174STATISTIC(LoopsEpilogueVectorized,
"Number of epilogues vectorized");
175STATISTIC(LoopsEarlyExitVectorized,
"Number of early exit loops vectorized");
177 "Number of partial aliasing loops vectorized");
182 cl::desc(
"When epilogue vectorization is enabled, and a value greater than "
183 "1 is specified, forces the given VF for all applicable epilogue "
184 "loops. Note: This allows all scalable VFs >= vscale x 1."));
253 bool CanUseConstantMax =
true,
bool CanExcludeZeroTrips =
false,
254 bool ComputeUpperBoundOnly =
false) {
264 if (Opts.loop_vectorize_with_block_frequency && !ComputeUpperBoundOnly)
268 if (!CanUseConstantMax)
278 if (CanUseConstantMax && CanExcludeZeroTrips)
301 if (
I->getDebugLoc() !=
Empty)
302 return I->getDebugLoc();
304 for (
Use &
Op :
I->operands()) {
306 if (OpInst->getDebugLoc() !=
Empty)
307 return OpInst->getDebugLoc();
310 return I->getDebugLoc();
317 return B.CreateElementCount(Ty, VF);
368 : Opts(Config.
getOpts()), Config(Config), EpilogueLoweringStatus(SEL),
393 "Profitable to scalarize relevant only for VF > 1.");
396 "cost-model should not be used for outer loops (in VPlan-native path)");
398 auto Scalars = InstsToScalarize.find(VF);
399 assert(Scalars != InstsToScalarize.end() &&
400 "VF not yet analyzed for scalarization profitability");
401 return Scalars->second.contains(
I);
408 "cost-model should not be used for outer loops (in VPlan-native path)");
419 auto UniformsPerVF = Uniforms.find(VF);
420 assert(UniformsPerVF != Uniforms.end() &&
421 "VF not yet analyzed for uniformity");
422 return UniformsPerVF->second.count(
I);
429 "cost-model should not be used for outer loops (in VPlan-native path)");
433 auto ScalarsPerVF = Scalars.find(VF);
434 assert(ScalarsPerVF != Scalars.end() &&
435 "Scalar values are not calculated for VF");
436 return ScalarsPerVF->second.count(
I);
442 const auto &MinBWs = Config.getMinimalBitwidths();
445 I->getType()->getScalarSizeInBits() < MinBWs.lookup(
I))
447 return VF.
isVector() && MinBWs.contains(
I) &&
469 "Unknown",
"Widen",
"Widen_Reverse",
"Interleave",
470 "GatherScatter",
"Scalarize",
"InvalidatedDecision"};
471 return WideningStr[W];
482 <<
" and instruction: " << *
I <<
'\n');
483 WideningDecisions[{
I, VF}] = {W,
Cost};
505 <<
" and instruction: " << *
I <<
'\n');
507 WideningDecisions[{
I, VF}] = {W, InsertPosCost};
509 WideningDecisions[{
I, VF}] = {W, OtherMemberCost};
520 "cost-model should not be used for outer loops (in VPlan-native path)");
522 std::pair<Instruction *, ElementCount> InstOnVF(
I, VF);
523 auto Itr = WideningDecisions.find(InstOnVF);
524 if (Itr == WideningDecisions.end())
526 return Itr->second.first;
533 std::pair<Instruction *, ElementCount> InstOnVF(
I, VF);
534 assert(WideningDecisions.contains(InstOnVF) &&
535 "The cost is not calculated");
536 return WideningDecisions[InstOnVF].second;
557 Value *
Op = Trunc->getOperand(0);
558 if (
Op !=
Legal->getPrimaryInduction() &&
TTI.isTruncateFree(SrcTy, DestTy))
562 return Legal->isInductionPhi(
Op);
578 if (VF.
isScalar() || Uniforms.contains(VF))
581 collectLoopUniforms(VF);
582 collectLoopScalars(VF);
591 return !
valueOr(Opts.force_widen_divrem_via_masked_intrinsic,
592 !(ScalarCost < MaskedCost));
633 std::pair<InstructionCost, InstructionCost>
671 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
677 !(Opts.enable_early_exit_vectorization &&
678 Legal->hasUncountableEarlyExit())) {
679 LLVM_DEBUG(
dbgs() <<
"LV: Loop requires scalar epilogue: not exiting "
680 "from latch block\n");
685 "interleaved group requires scalar epilogue\n");
688 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
700 return ChosenTailFoldingStyle;
708 "Tail folding must not be selected yet.");
709 if (!
Legal->canFoldTailByMasking()) {
715 ChosenTailFoldingStyle = Opts.force_tail_folding_style.value_or(
716 TTI.getPreferredTailFoldingStyle());
722 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
723 TTI.hasActiveVectorLength() &&
724 !Opts.enable_vplan_native_path;
736 dbgs() <<
"LV: Preference for VP intrinsics indicated. Will "
737 "not try to generate VP Intrinsics "
739 ?
"since interleave count specified is greater than 1.\n"
740 :
"due to non-interleaving reasons.\n"));
751 "Did not expect to enable alias masking with EVL!");
759 if (!Opts.force_partial_aliasing_vectorization ||
760 !
Legal->getFixedOrderRecurrences().empty())
768 if (!DiffChecks || DiffChecks->empty())
771 [[maybe_unused]]
auto HasPointerArgs = [](
CallBase *CB) {
773 return Arg->getType()->isPointerTy();
782 (!
I.mayReadOrWriteMemory() || (
Call && !HasPointerArgs(
Call))) &&
783 "Skipped unexpected memory access");
794 if (
Legal->isConsecutivePtr(ScalarTy, Ptr) == -1)
813 Legal->blockNeedsPredication(BB);
825 bool HasUsesOutsideReductionChain)
const {
841 HasUsesOutsideReductionChain)
844 return Opts.prefer_predicated_reduction_select ||
845 TTI.preferPredicatedReductionSelect();
860 WideningDecisions.clear();
881 auto FS = ForcedScalars.find(VF);
882 return FS != ForcedScalars.end() && FS->second.contains(
I);
886 const VectorizeOptions &Opts;
888 unsigned NumPredStores = 0;
901 "alias-mask status must be decided already");
913 "alias-mask status must be decided already");
924 InstructionCost getMemInstScalarizationCost(Instruction *
I, ElementCount VF);
927 InstructionCost getInterleaveGroupCost(Instruction *
I, ElementCount VF)
const;
930 InstructionCost getGatherScatterCost(Instruction *
I, ElementCount VF)
const;
946 ElementCount VF)
const;
951 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
955 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
956 PredicatedBBsAfterVectorization;
977 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
981 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
985 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
989 DenseMap<ElementCount, SmallSetVector<Instruction *, 4>> ForcedScalars;
997 ScalarCostsTy &ScalarCosts,
1009 void collectLoopUniforms(ElementCount VF);
1018 void collectLoopScalars(ElementCount VF);
1022 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1023 std::pair<InstWidening, InstructionCost>>;
1025 DecisionList WideningDecisions;
1029 bool needsExtract(
Value *
V, ElementCount VF)
const {
1105class GeneratedRTChecks {
1106 const VectorizeOptions &Opts;
1113 Value *SCEVCheckCond =
nullptr;
1120 Value *MemRuntimeCheckCond =
nullptr;
1124 bool HasChecks =
false;
1133 bool CostTooHigh =
false;
1135 Loop *OuterLoop =
nullptr;
1143 bool LoopUsesPartialAliasMasking =
false;
1146 GeneratedRTChecks(
const VectorizeOptions &Opts,
1150 bool LoopUsesPartialAliasMasking)
1151 : Opts(Opts), DT(DT), LI(LI),
TTI(
TTI),
1152 SCEVExp(*PSE.
getSE(),
"scev.check",
false),
1153 MemCheckExp(*PSE.
getSE(),
"scev.check",
false),
1154 PSE(PSE), CostKind(CostKind),
1155 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1163 void create(
Loop *L,
const LoopAccessInfo *LAI,
1164 const SCEVPredicate &UnionPred, ElementCount VF,
unsigned IC,
1165 OptimizationRemarkEmitter &ORE) {
1178 return OptimizationRemarkAnalysisAliasing(
1179 DEBUG_TYPE,
"TooManyMemoryRuntimeChecks",
L->getStartLoc(),
1181 <<
"loop not vectorized: too many memory checks needed";
1196 nullptr,
"vector.scevcheck");
1203 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1204 SCEVCleaner.cleanup();
1212 !LoopUsesPartialAliasMasking) {
1214 auto *
Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1218 auto DiffChecks = RtPtrChecking.getDiffChecks();
1221 MemCheckBlock->
getTerminator(), *DiffChecks, MemCheckExp, VF, IC);
1224 MemCheckBlock->
getTerminator(), L, RtPtrChecking.getChecks(),
1227 assert(MemRuntimeCheckCond &&
1228 "no RT checks generated although RtPtrChecking "
1229 "claimed checks are required");
1233 HasChecks = getSCEVChecks().first || getMemRuntimeChecks().first;
1235 if (!MemCheckBlock && !SCEVCheckBlock)
1245 if (SCEVCheckBlock) {
1248 auto *UI =
new UnreachableInst(Preheader->
getContext(), SCEVCheckBlock);
1252 if (MemCheckBlock) {
1255 auto *UI =
new UnreachableInst(Preheader->
getContext(), MemCheckBlock);
1261 if (MemCheckBlock) {
1265 if (SCEVCheckBlock) {
1272 if (MemCheckBlock && !getMemRuntimeChecks().first)
1273 eraseMemCheckBlock();
1276 OuterLoop =
L->getParentLoop();
1280 if (SCEVCheckBlock || MemCheckBlock)
1292 for (Instruction &
I : *SCEVCheckBlock) {
1293 if (SCEVCheckBlock->getTerminator() == &
I)
1299 if (MemCheckBlock) {
1301 for (Instruction &
I : *MemCheckBlock) {
1302 if (MemCheckBlock->getTerminator() == &
I)
1314 ScalarEvolution *SE = MemCheckExp.
getSE();
1319 const SCEV *
Cond = SE->
getSCEV(MemRuntimeCheckCond);
1324 unsigned BestTripCount = 2;
1328 Opts, PSE, OuterLoop,
false))
1329 if (EstimatedTC->isFixed())
1330 BestTripCount = EstimatedTC->getFixedValue();
1335 NewMemCheckCost = std::max(NewMemCheckCost.
getValue(),
1338 if (BestTripCount > 1)
1340 <<
"We expect runtime memory checks to be hoisted "
1341 <<
"out of the outer loop. Cost reduced from "
1342 << MemCheckCost <<
" to " << NewMemCheckCost <<
'\n');
1344 MemCheckCost = NewMemCheckCost;
1348 RTCheckCost += MemCheckCost;
1351 if (SCEVCheckBlock || MemCheckBlock)
1352 LLVM_DEBUG(
dbgs() <<
"Total cost of runtime checks: " << RTCheckCost
1360 ~GeneratedRTChecks() {
1361 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1362 bool SCEVChecksUsed = !SCEVCheckBlock || !
pred_empty(SCEVCheckBlock);
1364 SCEVCleaner.markResultUsed();
1366 if (MemCheckBlock &&
pred_empty(MemCheckBlock))
1367 eraseMemCheckBlock();
1369 SCEVCleaner.cleanup();
1371 if (!SCEVChecksUsed)
1372 SCEVCheckBlock->eraseFromParent();
1377 std::pair<Value *, BasicBlock *> getSCEVChecks()
const {
1378 using namespace llvm::PatternMatch;
1380 return {
nullptr,
nullptr};
1382 return {SCEVCheckCond, SCEVCheckBlock};
1387 std::pair<Value *, BasicBlock *> getMemRuntimeChecks()
const {
1388 using namespace llvm::PatternMatch;
1389 if (MemRuntimeCheckCond &&
match(MemRuntimeCheckCond,
m_ZeroInt()))
1390 return {
nullptr,
nullptr};
1391 return {MemRuntimeCheckCond, MemCheckBlock};
1395 bool hasChecks()
const {
return HasChecks; }
1398 void eraseMemCheckBlock() {
1399 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1400 auto &SE = *MemCheckExp.
getSE();
1407 I.eraseFromParent();
1409 MemCheckCleaner.cleanup();
1410 MemCheckBlock->eraseFromParent();
1411 MemCheckBlock =
nullptr;
1412 MemRuntimeCheckCond =
nullptr;
1451 LLVM_DEBUG(
dbgs() <<
"LV: Loop hints prevent outer loop vectorization.\n");
1457 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Interleave is not supported for "
1473 if (L.isInnermost() || Opts.vplan_build_outerloop_stress_test ||
1474 (Opts.enable_vplan_native_path &&
1488 for (
Loop *InnerL : L)
1502 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1505 : std::max(Cost->TTI.getMaxInterleaveFactor(VF,
false),
1506 Cost->TTI.getMaxInterleaveFactor(VF,
true));
1508 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1515 Opts, Cost->PSE, Cost->TheLoop,
1519 std::optional<uint64_t> MaxStep =
1521 std::optional<uint64_t> MaxTC =
1523 if (!MaxStep || !MaxTC)
1528 if (MaxUIntTripCount.
ult(*MaxTC))
1531 return (MaxUIntTripCount - *MaxTC).ugt(*MaxStep);
1543 return valueOr(Opts.enable_masked_interleaved_mem_accesses,
1544 TTI.enableMaskedInterleavedAccessVectorization());
1553 VPlan *Plan =
nullptr) {
1557 auto IP = IRVPBB->
begin();
1563 R.moveBefore(IRVPBB->
end());
1576 auto *Cmp = L->getLatchCmpInst();
1578 InstsToIgnore.
insert(Cmp);
1588 [&](
const User *U) { return U == IV || U == Cmp; }))
1589 InstsToIgnore.
insert(IVInst);
1595struct CSEDenseMapInfo {
1596 static bool canHandle(
const Instruction *
I) {
1601 static unsigned getHashValue(
const Instruction *
I) {
1602 assert(canHandle(
I) &&
"Unknown instruction!");
1607 static bool isEqual(
const Instruction *
LHS,
const Instruction *
RHS) {
1608 return LHS->isIdenticalTo(
RHS);
1620 if (!CSEDenseMapInfo::canHandle(&In))
1626 In.replaceAllUsesWith(
V);
1627 In.eraseFromParent();
1640 std::optional<unsigned> VScale) {
1644 EstimatedVF *= *VScale;
1645 assert(EstimatedVF >= 1 &&
"Estimated VF shouldn't be less than 1");
1659 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
1677 for (
auto &ArgOp : CI->
args())
1688 getScalarizationOverhead(CI, VF);
1698 TTI.getCallInstrCost(
1699 nullptr, Variant->getReturnType(),
1700 Variant->getFunctionType()->params(), Config.CostKind));
1715 assert(ID &&
"Expected intrinsic call!");
1719 FMF = FPMO->getFastMathFlags();
1725 std::back_inserter(ParamTys),
1726 [&](
Type *Ty) { return maybeVectorizeType(Ty, VF); });
1731 return TTI.getIntrinsicInstrCost(CostAttrs, Config.CostKind);
1734void LoopVectorizationCostModel::collectLoopScalars(
ElementCount VF) {
1739 "This function should not be visited twice for the same VF");
1764 "Widening decision should be ready at this moment");
1766 if (
Store && Ptr ==
Store->getValueOperand())
1769 "Ptr is neither a value or pointer operand");
1776 auto IsLoopVaryingGEP = [&](
Value *
V) {
1787 if (!IsLoopVaryingGEP(Ptr))
1799 if (IsScalarUse(MemAccess, Ptr) &&
1803 PossibleNonScalarPtrs.
insert(
I);
1819 for (
auto *BB :
TheLoop->blocks())
1820 for (
auto &
I : *BB) {
1822 EvaluatePtrUse(
Load,
Load->getPointerOperand());
1824 EvaluatePtrUse(
Store,
Store->getPointerOperand());
1825 EvaluatePtrUse(
Store,
Store->getValueOperand());
1828 for (
auto *
I : ScalarPtrs)
1829 if (!PossibleNonScalarPtrs.
count(
I)) {
1837 auto ForcedScalar = ForcedScalars.find(VF);
1838 if (ForcedScalar != ForcedScalars.end())
1839 for (
auto *
I : ForcedScalar->second) {
1840 LLVM_DEBUG(
dbgs() <<
"LV: Found (forced) scalar instruction: " << *
I <<
"\n");
1849 while (Idx != Worklist.
size()) {
1851 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
1855 auto *J = cast<Instruction>(U);
1856 return !TheLoop->contains(J) || Worklist.count(J) ||
1857 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
1858 IsScalarUse(J, Src));
1861 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Src <<
"\n");
1867 for (
const auto &Induction :
Legal->getInductionVars()) {
1868 auto *Ind = Induction.first;
1878 auto IsDirectLoadStoreFromPtrIndvar = [&](
Instruction *Indvar,
1880 return Induction.second.getKind() ==
1888 bool ScalarInd =
all_of(Ind->users(), [&](User *U) ->
bool {
1889 auto *I = cast<Instruction>(U);
1890 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
1891 IsDirectLoadStoreFromPtrIndvar(Ind, I);
1900 if (IndUpdatePhi &&
Legal->isFixedOrderRecurrence(IndUpdatePhi))
1905 bool ScalarIndUpdate =
all_of(IndUpdate->users(), [&](User *U) ->
bool {
1906 auto *I = cast<Instruction>(U);
1907 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
1908 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
1910 if (!ScalarIndUpdate)
1915 Worklist.
insert(IndUpdate);
1916 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Ind <<
"\n");
1917 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *IndUpdate
1921 Scalars[VF].insert_range(Worklist);
1946 switch(
I->getOpcode()) {
1949 case Instruction::Call: {
1957 case Instruction::Load:
1958 case Instruction::Store: {
1961 if (
Legal->isCompressedLoadOrStore(
I))
1963 bool IsConsecutive =
Legal->isConsecutivePtr(ScalarTy, Ptr);
1967 case Instruction::UDiv:
1968 case Instruction::SDiv:
1969 case Instruction::SRem:
1970 case Instruction::URem: {
1995 if (
Legal->blockNeedsPredication(
I->getParent()))
2008 switch(
I->getOpcode()) {
2011 "instruction should have been considered by earlier checks");
2012 case Instruction::Call:
2016 "should have returned earlier for calls not needing a mask");
2018 case Instruction::Load:
2021 case Instruction::Store: {
2029 case Instruction::UDiv:
2030 case Instruction::URem:
2032 return !
Legal->isInvariant(
I->getOperand(1));
2033 case Instruction::SDiv:
2034 case Instruction::SRem:
2047 if (!
Legal->blockNeedsPredication(BB))
2050 uint64_t HeaderFreq =
2054 "Header has smaller block freq than dominated BB?");
2055 return std::round((
double)HeaderFreq /
BBFreq);
2060 case Instruction::UDiv:
2061 return Intrinsic::masked_udiv;
2062 case Instruction::SDiv:
2063 return Intrinsic::masked_sdiv;
2064 case Instruction::URem:
2065 return Intrinsic::masked_urem;
2066 case Instruction::SRem:
2067 return Intrinsic::masked_srem;
2073std::pair<InstructionCost, InstructionCost>
2076 assert(
I->getOpcode() == Instruction::UDiv ||
2077 I->getOpcode() == Instruction::SDiv ||
2078 I->getOpcode() == Instruction::SRem ||
2079 I->getOpcode() == Instruction::URem);
2088 ScalarizationCost = 0;
2095 TTI.getCFInstrCost(Instruction::PHI, Config.CostKind);
2098 ScalarizationCost +=
2100 I->getOpcode(),
I->getType(), Config.CostKind);
2104 ScalarizationCost += getScalarizationOverhead(
I, VF);
2117 {VecTy, VecTy, MaskTy});
2119 return {ScalarizationCost, MaskedCost};
2126 "Decision should not be set yet.");
2128 assert(Group &&
"Must have a group.");
2129 unsigned InterleaveFactor = Group->getFactor();
2133 auto &
DL =
I->getDataLayout();
2145 bool ScalarNI =
DL.isNonIntegralPointerType(ScalarTy);
2148 bool MemberNI =
DL.isNonIntegralPointerType(MemberTy);
2150 if (MemberNI != ScalarNI)
2153 if (MemberNI && ScalarNI &&
2154 ScalarTy->getPointerAddressSpace() !=
2155 MemberTy->getPointerAddressSpace())
2164 bool PredicatedAccessRequiresMasking =
2166 bool LoadAccessWithGapsRequiresEpilogMasking =
2169 bool StoreAccessWithGapsRequiresMasking =
2171 if (!PredicatedAccessRequiresMasking &&
2172 !LoadAccessWithGapsRequiresEpilogMasking &&
2173 !StoreAccessWithGapsRequiresMasking)
2180 "Masked interleave-groups for predicated accesses are not enabled.");
2182 if (Group->isReverse())
2186 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2187 StoreAccessWithGapsRequiresMasking;
2194std::optional<LoopVectorizationCostModel::InstWidening>
2204 int Stride =
Legal->isConsecutivePtr(ScalarTy, Ptr);
2208 if (std::optional<CompressedPtrInfo> PtrInfo =
2209 Legal->getCompressedPtrInfo(Ptr)) {
2212 assert(Stride == 1 &&
"Compressed memory ops must be consecutive");
2216 return std::nullopt;
2221 return std::nullopt;
2225 auto &
DL =
I->getDataLayout();
2227 return std::nullopt;
2232void LoopVectorizationCostModel::collectLoopUniforms(
ElementCount VF) {
2239 "This function should not be visited twice for the same VF");
2243 Uniforms[VF].
clear();
2251 auto IsOutOfScope = [&](
Value *V) ->
bool {
2263 auto AddToWorklistIfAllowed = [&](
Instruction *
I) ->
void {
2264 if (IsOutOfScope(
I)) {
2271 dbgs() <<
"LV: Found not uniform due to requiring predication: " << *
I
2275 LLVM_DEBUG(
dbgs() <<
"LV: Found uniform instruction: " << *
I <<
"\n");
2284 TheLoop->getExitingBlocks(Exiting);
2285 for (BasicBlock *
E : Exiting) {
2286 if (
Legal->hasUncountableEarlyExit() &&
TheLoop->getLoopLatch() !=
E)
2289 if (!Cmp || !
TheLoop->contains(Cmp) || !
Cmp->hasOneUse())
2295 if (
Legal->hasUncountableExitWithSideEffects() &&
2297 if (Instruction *Countable =
2298 Legal->findCountableComparisonInCombinedCondition(Cmp)) {
2299 if (Countable->hasOneUse())
2300 AddToWorklistIfAllowed(Countable);
2306 AddToWorklistIfAllowed(Cmp);
2315 if (PrevVF.isVector()) {
2316 auto Iter = Uniforms.find(PrevVF);
2317 if (Iter != Uniforms.end() && !Iter->second.contains(
I))
2320 if (!isUniformMemOp(*
I, VF))
2330 auto IsUniformDecision = [&](
Instruction *
I, ElementCount VF) {
2333 "Widening decision should be ready at this moment");
2335 if (IsUniformMemOpUse(
I))
2338 return (WideningDecision ==
CM_Widen ||
2350 (IsUniformDecision(
I, VF) ||
Legal->isInvariant(Ptr));
2358 SetVector<Value *> HasUniformUse;
2362 for (
auto *BB :
TheLoop->blocks())
2363 for (
auto &
I : *BB) {
2365 switch (
II->getIntrinsicID()) {
2366 case Intrinsic::sideeffect:
2367 case Intrinsic::experimental_noalias_scope_decl:
2368 case Intrinsic::assume:
2369 case Intrinsic::lifetime_start:
2370 case Intrinsic::lifetime_end:
2371 if (
TheLoop->hasLoopInvariantOperands(&
I))
2372 AddToWorklistIfAllowed(&
I);
2380 if (IsOutOfScope(EVI->getAggregateOperand())) {
2381 AddToWorklistIfAllowed(EVI);
2387 "Expected aggregate value to be call return value");
2400 if (IsUniformMemOpUse(&
I))
2401 AddToWorklistIfAllowed(&
I);
2403 if (IsVectorizedMemAccessUse(&
I, Ptr))
2404 HasUniformUse.
insert(Ptr);
2410 for (
auto *
V : HasUniformUse) {
2411 if (IsOutOfScope(
V))
2414 bool UsersAreMemAccesses =
all_of(
I->users(), [&](User *U) ->
bool {
2415 auto *UI = cast<Instruction>(U);
2416 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
2418 if (UsersAreMemAccesses)
2419 AddToWorklistIfAllowed(
I);
2426 while (Idx != Worklist.
size()) {
2429 for (
auto *OV :
I->operand_values()) {
2431 if (IsOutOfScope(OV))
2436 if (
OP &&
Legal->isFixedOrderRecurrence(
OP))
2442 auto *J = cast<Instruction>(U);
2443 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
2445 AddToWorklistIfAllowed(OI);
2456 for (PHINode *Ind :
Legal->getInductionVars().keys()) {
2461 bool UniformInd =
all_of(Ind->users(), [&](User *U) ->
bool {
2462 auto *I = cast<Instruction>(U);
2463 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2464 IsVectorizedMemAccessUse(I, Ind);
2471 bool UniformIndUpdate =
all_of(IndUpdate->users(), [&](User *U) ->
bool {
2472 auto *I = cast<Instruction>(U);
2473 return I == Ind || Worklist.count(I) ||
2474 IsVectorizedMemAccessUse(I, IndUpdate);
2476 if (!UniformIndUpdate)
2480 AddToWorklistIfAllowed(Ind);
2481 AddToWorklistIfAllowed(IndUpdate);
2484 Uniforms[VF].insert_range(Worklist);
2490 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([
this] {
2497 if (!
TheLoop->isInnermost()) {
2498 return Config.computeVPlanOuterloopVF(UserVF);
2501 if (
Legal->getRuntimePointerChecking()->Need &&
TTI.hasBranchDivergence()) {
2505 "Not inserting runtime ptr check for divergent target",
2506 "runtime pointer checks needed. Not enabled for divergent target",
2507 "CantVersionLoopWithDivergentTarget",
ORE,
TheLoop);
2513 unsigned MaxTC =
PSE.getSmallConstantMaxTripCount();
2518 LLVM_DEBUG(
dbgs() <<
"LV: Found maximum trip count: " << MaxTC <<
'\n');
2521 "Single iteration (non) loop",
2522 "loop trip count is one, irrelevant for vectorization",
2533 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2537 "Trip count computation wrapped",
2538 "backedge-taken count is -1, loop trip count wrapped to 0",
2543 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2544 "No cost-modeling decisions should have been taken at this point");
2546 switch (EpilogueLoweringStatus) {
2548 return Config.computeFeasibleMaxVF(MaxTC, UserVF, UserIC,
false,
2554 <<
"LV: Not allowing epilogue, creating tail-folded "
2555 <<
"vector loop.\n");
2558 LLVM_DEBUG(
dbgs() <<
"LV: Not allowing epilogue due to low trip "
2562 LLVM_DEBUG(
dbgs() <<
"LV: Not allowing epilogue due to -Os/-Oz.\n");
2566 if (Config.runtimeChecksRequired())
2587 std::optional<uint64_t> MaxPowerOf2RuntimeVF =
2590 if (std::optional<uint64_t> MaxRuntimeScalableVF =
2592 MaxPowerOf2RuntimeVF =
2593 std::max(*MaxPowerOf2RuntimeVF, *MaxRuntimeScalableVF);
2595 MaxPowerOf2RuntimeVF = std::nullopt;
2601 auto NoScalarEpilogueNeeded = [
this, &UserIC](uint64_t MaxRuntimeVF) {
2605 !
Legal->hasUncountableEarlyExit())
2607 uint64_t MaxVFtimesIC = MaxRuntimeVF * std::max<uint64_t>(UserIC, 1);
2612 const SCEV *BackedgeTakenCount =
PSE.getSymbolicMaxBackedgeTakenCount();
2614 BackedgeTakenCount ==
PSE.getBackedgeTakenCount()) &&
2615 "Invalid loop count");
2617 BackedgeTakenCount, SE->
getOne(BackedgeTakenCount->
getType()));
2624 if (MaxPowerOf2RuntimeVF > 0u) {
2626 "MaxFixedVF must be a power of 2");
2627 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
2629 LLVM_DEBUG(
dbgs() <<
"LV: No tail will remain for any chosen VF.\n");
2635 if (ExpectedTC && ExpectedTC->isFixed() &&
2636 ExpectedTC->getFixedValue() <=
2637 TTI.getMinTripCountTailFoldingThreshold()) {
2644 LLVM_DEBUG(
dbgs() <<
"LV: Picking a fixed-width so that no tail will "
2645 "remain for any chosen VF.\n");
2661 unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
2663 if (TC.
getFixedValue() - MaxVFForTC == 1 && MaxVFForTC / EffectiveIC > 1 &&
2665 !Config.OptForSize) {
2670 if (NumOfInstructions > Opts.low_trip_count_loop_body_size_limit) {
2671 unsigned VF = MaxVFForTC / EffectiveIC;
2673 <<
" with 1 scalar iteration remaining.\n");
2681 "The trip count is below the minial threshold value.",
2682 "loop trip count is too low, avoiding vectorization",
"LowTripCount",
2697 <<
"LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
2698 "try to generate VP Intrinsics with scalable vector "
2703 assert(ContainsScalableVF &&
"Expected scalable vector factor.");
2715 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with an "
2716 "epilogue instead.\n");
2722 LLVM_DEBUG(
dbgs() <<
"LV: Can't fold tail by masking: don't vectorize\n");
2728 "unable to calculate the loop count due to complex control flow",
2734 "Cannot optimize for size and vectorize at the same time.",
2735 "cannot optimize for size and vectorize at the same time. "
2736 "Enable vectorization of this loop with '#pragma clang loop "
2737 "vectorize(enable)' when compiling with -Os/-Oz",
2744 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
2746 for (
const auto &Plan : VPlans) {
2749 if (Plan->hasTailFolded() != CM->foldTailByMasking())
2761 precomputeCosts(*Plan, VF, CostCtx);
2764 for (
auto &R : *VPBB) {
2765 if (!R.cost(VF, CostCtx).isValid())
2771 if (InvalidCosts.
empty())
2779 for (
auto &Pair : InvalidCosts)
2784 sort(InvalidCosts, [&Numbering](RecipeVFPair &
A, RecipeVFPair &
B) {
2785 unsigned NA = Numbering[
A.first];
2786 unsigned NB = Numbering[
B.first];
2801 Subset = Tail.take_front(1);
2812 return Instruction::Store;
2816 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
2817 [](
const auto *R) {
return Instruction::Call; })
2820 [](
const auto *R) {
return R->getOpcode(); })
2822 return R->getStoredValues().empty() ? Instruction::Load
2823 : Instruction::Store;
2829 return Instruction::CondBr;
2837 if (Subset == Tail || Tail[Subset.size()].first != R) {
2838 std::string OutString;
2840 assert(!Subset.empty() &&
"Unexpected empty range");
2841 OS <<
"Recipe with invalid costs prevented vectorization at VF=(";
2842 for (
const auto &Pair : Subset)
2843 OS << (Pair.second == Subset.front().second ?
"" :
", ") << Pair.second;
2845 if (Opcode == Instruction::Call) {
2848 Name =
Int->getIntrinsicName();
2853 ? WidenCall->getCalledScalarFunction()
2857 OS <<
" call to " << Name;
2862 Tail = Tail.drop_front(Subset.size());
2866 Subset = Tail.take_front(Subset.size() + 1);
2867 }
while (!Tail.empty());
2888 switch (R.getVPRecipeID()) {
2889 case VPRecipeBase::VPDerivedIVSC:
2890 case VPRecipeBase::VPScalarIVStepsSC:
2891 case VPRecipeBase::VPReplicateSC:
2892 case VPRecipeBase::VPInstructionSC:
2893 case VPRecipeBase::VPCurrentIterationPHISC:
2894 case VPRecipeBase::VPVectorPointerSC:
2895 case VPRecipeBase::VPVectorEndPointerSC:
2896 case VPRecipeBase::VPExpandSCEVSC:
2897 case VPRecipeBase::VPPredInstPHISC:
2898 case VPRecipeBase::VPBranchOnMaskSC:
2899 case VPRecipeBase::VPConditionalInductionPHISC:
2901 case VPRecipeBase::VPReductionSC:
2902 case VPRecipeBase::VPActiveLaneMaskPHISC:
2903 case VPRecipeBase::VPWidenCallSC:
2904 case VPRecipeBase::VPWidenCanonicalIVSC:
2905 case VPRecipeBase::VPWidenCastSC:
2906 case VPRecipeBase::VPWidenGEPSC:
2907 case VPRecipeBase::VPWidenIntrinsicSC:
2908 case VPRecipeBase::VPWidenMemIntrinsicSC:
2909 case VPRecipeBase::VPWidenSC:
2910 case VPRecipeBase::VPBlendSC:
2911 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
2912 case VPRecipeBase::VPHistogramSC:
2913 case VPRecipeBase::VPWidenPHISC:
2914 case VPRecipeBase::VPWidenIntOrFpInductionSC:
2915 case VPRecipeBase::VPWidenPointerInductionSC:
2916 case VPRecipeBase::VPReductionPHISC:
2917 case VPRecipeBase::VPInterleaveEVLSC:
2918 case VPRecipeBase::VPInterleaveSC:
2919 case VPRecipeBase::VPWidenLoadEVLSC:
2920 case VPRecipeBase::VPWidenLoadSC:
2921 case VPRecipeBase::VPWidenStoreEVLSC:
2922 case VPRecipeBase::VPWidenStoreSC:
2928 auto WillGenerateTargetVectors = [&
TTI, VF](
Type *VectorTy) {
2929 unsigned NumLegalParts =
TTI.getNumberOfParts(VectorTy);
2945 if (R.getNumDefinedValues() == 0 &&
2954 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
2956 if (!Visited.
insert({ScalarTy}).second)
2970 [](
auto *VPRB) { return VPRB->isReplicator(); });
2979 return RecurrenceDescriptor::isFindLastRecurrenceKind(
2980 RedPhi.getRecurrenceKind());
3003 if (!Opts.enable_epilogue_vectorization) {
3005 "Options conflict, epilogue vectorization is disallowed while "
3006 "epilogue tail-folding allowed!",
3007 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3013 "applied without forced main/epilogue loop VF",
3014 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3020 "when VF of the main loop <= VF of the epilogue",
3021 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3025 if (!L->isInnermost()) {
3027 "Epilogue tail-folding is not supported for outer loop",
3028 "InvalidTailFoldedEpilogue", ORE, L);
3035 "Epilogue tail-folding can't be applied because scalar epilogue is "
3036 "required. Fall back to a normal epilogue",
3037 "InvalidTailFoldedEpilogue", ORE, L);
3044 "no epilogue is allowed.",
3045 "InvalidTailFoldedEpilogue", ORE, L);
3049 if (L->getExitingBlock() != L->getLoopLatch() ||
3052 "Epilogue tail-folding is not supported yet for early-exit loops",
3053 "InvalidTailFoldedEpilogue", ORE, L);
3064 "Epilogue tail-folding is not supported with interleaved accesses "
3065 "when masking them isn't supported",
3066 "InvalidTailFoldedEpilogue", ORE, L);
3070 if (Opts.force_partial_aliasing_vectorization) {
3072 "Epilogue tail-folding is not supported with alias masking",
3073 "InvalidTailFoldedEpilogue", ORE, L);
3079 "Epilogue tail-folding is not supported with reductions",
3080 "InvalidTailFoldedEpilogue", ORE, L);
3086 "Epilogue tail-folding is not supported with fixed-order recurrence",
3087 "InvalidTailFoldedEpilogue", ORE, L);
3095 TTI->getPreferredTailFoldingStyle());
3099 "EVL-based tail-folding",
3100 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3116 if (!TTI.preferEpilogueVectorization(VF * IC))
3119 unsigned MinVFThreshold = Opts.epilogue_vectorization_minimum_VF.value_or(
3120 TTI.getEpilogueVectorizationMinVF());
3126 bool ScalarEpilogueAllowed) {
3127 if (!Opts.enable_epilogue_vectorization) {
3128 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is disabled.\n");
3132 if (!ScalarEpilogueAllowed) {
3133 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because no "
3134 "epilogue is allowed.\n");
3141 <<
"LEV: Epilogue vectorization not supported with alias masking.\n");
3147 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3148 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because the loop "
3149 "is not a supported candidate.\n");
3155 Config.getVScaleForTuning()) >=
3160 LLVM_DEBUG(
dbgs() <<
"LEV: Forced epilogue VF results in dead epilogue "
3161 "vector loop, skipping vectorizing epilogue.\n");
3165 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization factor is forced.\n");
3167 std::unique_ptr<VPlan> Clone(
3173 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization forced factor is not "
3178 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3180 dbgs() <<
"LEV: Epilogue vectorization skipped due to opt for size.\n");
3184 if (!Config.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
3185 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is not profitable for "
3196 if (
match(&Exiting->back(),
3206 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3214 Type *TCType = Legal->getWidestInductionType();
3215 const SCEV *RemainingIterations =
nullptr;
3216 unsigned MaxTripCount = 0;
3219 const SCEV *KnownMinTC;
3221 bool ScalableRemIter =
false;
3225 ScalableRemIter = ScalableTC;
3226 RemainingIterations =
3228 }
else if (ScalableTC) {
3231 SE.
getConstant(TCType, Config.getVScaleForTuning().value_or(1)));
3235 RemainingIterations =
3239 if (RemainingIterations->
isZero())
3249 << MaxTripCount <<
"\n");
3252 auto SkipVF = [&](
const SCEV *VF,
const SCEV *RemIter) ->
bool {
3256 VPlan *BestPlan =
nullptr;
3257 for (
auto &NextVF : ProfitableVFs) {
3263 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3278 if (!ScalableRemIter) {
3284 if (SkipVF(SE.
getElementCount(TCType, EffectiveVF), RemainingIterations))
3288 if (Result.Width.isScalar() ||
3289 isMoreProfitable(NextVF, Result, MaxTripCount,
3293 BestPlan = &CurrentPlan;
3301 << Result.Width <<
"\n");
3302 std::unique_ptr<VPlan> Clone(BestPlan->
duplicate());
3303 Clone->setVF(Result.Width);
3327 if (!CM->isEpilogueAllowed())
3333 "Unroll factor forced to be 1.\n");
3338 if (!Legal->isSafeForAnyVectorWidth())
3351 const bool HasReductions =
3363 if (LoopCost == 0) {
3365 LoopCost = CM->expectedCost(VF);
3367 LoopCost = cost(Plan, VF, &R, *CM);
3368 assert(LoopCost.
isValid() &&
"Expected to have chosen a VF with valid cost");
3377 for (
auto &Pair : R.MaxLocalUsers) {
3378 Pair.second = std::max(Pair.second, 1U);
3392 unsigned IC = UINT_MAX;
3394 for (
const auto &Pair : R.MaxLocalUsers) {
3395 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
3398 << TTI.getRegisterClassName(Pair.first)
3399 <<
" register class\n");
3400 TargetNumRegisters = (VF.
isScalar() ? Opts.force_target_num_scalar_regs
3401 : Opts.force_target_num_vector_regs)
3402 .value_or(TargetNumRegisters);
3403 unsigned MaxLocalUsers = Pair.second;
3404 unsigned LoopInvariantRegs = 0;
3405 if (R.LoopInvariantRegs.contains(Pair.first))
3406 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3408 unsigned TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
3411 if (Opts.enable_ind_var_reg_heur) {
3412 TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
3413 std::max(1U, (MaxLocalUsers - 1)));
3416 IC = std::min(IC, TmpIC);
3420 bool HasUnorderedReductions =
3425 unsigned MaxInterleaveCount =
3426 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3427 LLVM_DEBUG(
dbgs() <<
"LV: MaxInterleaveFactor for the target is "
3428 << MaxInterleaveCount <<
"\n");
3431 MaxInterleaveCount = (VF.
isScalar() ? Opts.force_target_max_scalar_interleave
3432 : Opts.force_target_max_vector_interleave)
3433 .value_or(MaxInterleaveCount);
3440 CM->isEpilogueAllowed());
3443 if (BestKnownTC && (BestKnownTC->isFixed() || VF.
isScalable())) {
3445 unsigned AvailableTC =
3447 unsigned EstimatedVF =
3455 unsigned InterleaveCountLB =
bit_floor(std::max(
3456 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
3470 unsigned InterleaveCountUB =
bit_floor(std::max(
3471 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
3472 MaxInterleaveCount = InterleaveCountLB;
3474 if (InterleaveCountUB != InterleaveCountLB) {
3475 unsigned TailTripCountUB =
3476 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3477 unsigned TailTripCountLB =
3478 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3481 if (TailTripCountUB == TailTripCountLB)
3482 MaxInterleaveCount = InterleaveCountUB;
3490 MaxInterleaveCount = InterleaveCountLB;
3494 assert(MaxInterleaveCount > 0 &&
3495 "Maximum interleave count must be greater than 0");
3499 if (IC > MaxInterleaveCount)
3500 IC = MaxInterleaveCount;
3503 IC = std::max(1u, IC);
3505 assert(IC > 0 &&
"Interleave count must be greater than 0.");
3509 if (Opts.enable_interleave_to_reduce_stalls && VF.
isVector()) {
3510 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving to reduce stall cycles due to "
3511 "instruction latency.\n");
3517 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving due to replicator region.\n");
3527 unsigned StallsIC = 1;
3534 if (Opts.force_target_load_latency > 0 && R.mayReadFromMemory())
3535 StallCycles = Opts.force_target_load_latency - 1;
3543 StallsIC = std::max(StallsIC, ThisIC);
3547 if (StallsIC <= 1) {
3549 dbgs() <<
"LV: Not interleaving as it wouldn't reduce stalls.\n");
3552 IC = std::min(IC, StallsIC);
3558 if (VF.
isVector() && HasReductions) {
3559 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving because of reductions.\n");
3567 bool ScalarInterleavingRequiresPredication =
3569 return Legal->blockNeedsPredication(BB);
3571 bool ScalarInterleavingRequiresRuntimePointerCheck =
3572 (VF.
isScalar() && Legal->getRuntimePointerChecking()->Need);
3577 <<
"LV: IC is " << IC <<
'\n'
3578 <<
"LV: VF is " << VF <<
'\n');
3579 const bool AggressivelyInterleave =
3580 TTI.enableAggressiveInterleaving(HasReductions);
3581 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3582 !ScalarInterleavingRequiresPredication &&
3583 LoopCost < Opts.small_loop_cost) {
3593 unsigned NumStores = 0;
3594 unsigned NumLoads = 0;
3608 if (
unsigned StoreOps = InterleaveR->getNumStoreOperands())
3609 NumStores += StoreOps;
3611 NumLoads += InterleaveR->getNumDefinedValues();
3626 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3627 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3633 bool HasSelectCmpReductions =
3638 return RecurrenceDescriptor::isAnyOfRecurrenceKind(
3639 RedR.getRecurrenceKind()) ||
3640 RecurrenceDescriptor::isFindIVRecurrenceKind(
3641 RedR.getRecurrenceKind());
3643 if (HasSelectCmpReductions) {
3644 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving select-cmp reductions.\n");
3653 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3654 bool HasOrderedReductions =
3658 if (HasOrderedReductions) {
3660 dbgs() <<
"LV: Not interleaving scalar ordered reductions.\n");
3664 unsigned F = Opts.max_nested_scalar_reduction_interleave;
3665 SmallIC = std::min(SmallIC,
F);
3666 StoresIC = std::min(StoresIC,
F);
3667 LoadsIC = std::min(LoadsIC,
F);
3670 if (Opts.enable_loadstore_runtime_interleave &&
3671 std::max(StoresIC, LoadsIC) > SmallIC) {
3673 dbgs() <<
"LV: Interleaving to saturate store or load ports.\n");
3674 return std::max(StoresIC, LoadsIC);
3679 if (VF.
isScalar() && AggressivelyInterleave) {
3683 return std::max(IC / 2, SmallIC);
3686 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving to reduce branch cost.\n");
3692 if (AggressivelyInterleave) {
3712 "Expecting a scalar emulated instruction");
3714 (
isa<StoreInst>(
I) && NumPredStores > Opts.vectorize_num_stores_pred);
3719 assert(!InstsToScalarize.contains(VF) &&
3720 !PredicatedBBsAfterVectorization.contains(VF) &&
3721 "Routine expected to be called once per VF");
3726 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
3736 ScalarCostsTy ScalarCosts;
3745 !
Legal->isCompressedLoadOrStore(&
I) &&
3746 computePredInstDiscount(&
I, ScalarCosts, VF) >= 0) {
3747 for (
const auto &[
I, IC] : ScalarCosts)
3748 ScalarCostsVF.
insert({
I, IC});
3751 PredicatedBBsAfterVectorization[VF].insert(BB);
3753 if (Pred->getSingleSuccessor() == BB)
3754 PredicatedBBsAfterVectorization[VF].insert(Pred);
3763 "Instruction marked uniform-after-vectorization will be predicated");
3781 if (!
I->hasOneUse() || PredInst->
getParent() !=
I->getParent() ||
3800 for (
Use &U :
I->operands())
3813 while (!Worklist.
empty()) {
3817 if (ScalarCosts.contains(
I))
3840 ScalarCost +=
TTI.getScalarizationOverhead(
3846 TTI.getCFInstrCost(Instruction::PHI, Config.CostKind);
3853 for (Use &U :
I->operands())
3856 "Instruction has non-scalar type");
3857 if (CanBeScalarized(J))
3859 else if (needsExtract(J, VF)) {
3862 ScalarCost +=
TTI.getScalarizationOverhead(
3865 true, Config.CostKind);
3875 Discount += VectorCost - ScalarCost;
3876 ScalarCosts[
I] = ScalarCost;
3900 if (
C.isValid() && Opts.force_target_instruction_cost)
3904 LLVM_DEBUG(
dbgs() <<
"LV: Found an estimated cost of " <<
C <<
" for VF "
3905 << VF <<
" For instruction: " <<
I <<
'\n');
3926 const Loop *TheLoop) {
3933LoopVectorizationCostModel::getMemInstScalarizationCost(
Instruction *
I,
3936 "Scalarization cost of instruction implies vectorization.");
3941 auto *SE =
PSE.getSE();
3956 TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV, Config.CostKind);
3964 AS, Config.CostKind, OpInfo);
3968 Cost += getScalarizationOverhead(
I, VF);
3979 Cost +=
TTI.getScalarizationOverhead(
3981 false,
true, Config.CostKind);
3982 Cost +=
TTI.getCFInstrCost(Instruction::CondBr, Config.CostKind);
3996 "Expected a consecutive widening decision");
4006 if (
Legal->isCompressedLoadOrStore(
I)) {
4007 LoadIID = Intrinsic::masked_expandload;
4008 StoreIID = Intrinsic::masked_compressstore;
4011 unsigned IID =
I->getOpcode() == Instruction::Load ? LoadIID : StoreIID;
4012 Cost +=
TTI.getMemIntrinsicInstrCost(
4013 MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4017 Cost +=
TTI.getMemoryOpCost(
I->getOpcode(), VectorTy, Alignment, AS,
4018 Config.CostKind, OpInfo,
I);
4023 VectorTy, Config.CostKind, {}, 0);
4028LoopVectorizationCostModel::getUniformMemOpCost(
Instruction *
I,
4030 assert(isUniformMemOp(*
I, VF));
4038 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
4040 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
4043 VectorTy, Config.CostKind);
4047 bool IsLoopInvariantStoreValue =
Legal->isInvariant(
SI->getValueOperand());
4053 TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr, Config.CostKind) +
4054 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
4056 if (!IsLoopInvariantStoreValue)
4057 Cost +=
TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
4058 VectorTy, Config.CostKind, 0);
4063LoopVectorizationCostModel::getGatherScatterCost(
Instruction *
I,
4071 if (!isUniform(Ptr, VF))
4074 unsigned IID =
I->getOpcode() == Instruction::Load
4075 ? Intrinsic::masked_gather
4076 : Intrinsic::masked_scatter;
4077 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
4079 TTI.getMemIntrinsicInstrCost(
4086LoopVectorizationCostModel::getInterleaveGroupCost(
Instruction *
I,
4089 assert(Group &&
"Fail to get an interleaved access group.");
4096 unsigned InterleaveFactor = Group->getFactor();
4100 SmallVector<unsigned, 4> Indices;
4101 for (
unsigned IF = 0; IF < InterleaveFactor; IF++)
4102 if (Group->getMember(IF))
4106 bool UseMaskForGaps =
4110 InsertPos->
getOpcode(), WideVecTy, Group->getFactor(), Indices,
4114 if (Group->isReverse()) {
4117 "Reverse masked interleaved access not supported.");
4118 Cost += Group->getNumMembers() *
4120 VectorTy, Config.CostKind, {}, 0);
4126LoopVectorizationCostModel::getMemoryInstructionCost(
Instruction *
I,
4137 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
4139 TTI.getMemoryOpCost(
I->getOpcode(), ValTy, Alignment, AS,
4140 Config.CostKind, OpInfo,
I);
4146LoopVectorizationCostModel::getScalarizationOverhead(
Instruction *
I,
4164 VIC = TTI::VectorInstrContext::Load;
4166 VIC = TTI::VectorInstrContext::Store;
4169 Cost +=
TTI.getScalarizationOverhead(
4171 true,
false, Config.CostKind,
4191 SmallPtrSet<Value *, 4> SeenOps;
4193 if (needsExtract(
Op, VF) && SeenOps.
insert(
Op).second)
4197 ? TTI::VectorInstrContext::Store
4200 TTI.getOperandsScalarizationOverhead(Tys, Config.CostKind, OperandVIC);
4227 LLVM_DEBUG(
dbgs() <<
"LV: Memory widening: calculating best strategy for "
4229 if (isUniformMemOp(
I, VF)) {
4230 auto IsLegalToScalarize = [&]() {
4250 return TheLoop->isLoopInvariant(
SI.getValueOperand());
4262 IsLegalToScalarize() ? getUniformMemOpCost(&
I, VF)
4268 LLVM_DEBUG(
dbgs() <<
"LV: Memory widening: uniform memory op has "
4269 "GatherScatterCost = "
4270 << GatherScatterCost <<
", ScalarizationCost = "
4271 << ScalarizationCost <<
'\n');
4272 if (GatherScatterCost < ScalarizationCost)
4280 if (std::optional<InstWidening> Decision =
4284 dbgs() <<
"LV: Memory widening: can be widened normally with cost "
4285 << WidenCost <<
'\n');
4291 "compressed loads/stores should be handled in "
4292 "memoryInstructionCanBeWidened");
4296 unsigned NumAccesses = 1;
4299 assert(Group &&
"Fail to get an interleaved access group.");
4301 NumAccesses = Group->getNumMembers();
4303 InterleaveCost = getInterleaveGroupCost(&
I, VF);
4308 ? getGatherScatterCost(&
I, VF) * NumAccesses
4312 getMemInstScalarizationCost(&
I, VF) * NumAccesses;
4318 if (InterleaveCost <= GatherScatterCost &&
4319 InterleaveCost < ScalarizationCost) {
4321 Cost = InterleaveCost;
4322 }
else if (GatherScatterCost < ScalarizationCost) {
4324 Cost = GatherScatterCost;
4327 Cost = ScalarizationCost;
4330 dbgs() <<
"LV: Memory widening: InterleaveCost = " << InterleaveCost
4331 <<
", GatherScatterCost = " << GatherScatterCost
4332 <<
", ScalarizationCost = " << ScalarizationCost <<
'\n');
4341 getMemInstScalarizationCost(
I, VF));
4355 if (
TTI.prefersVectorizedAddressing())
4364 if (PtrDef &&
TheLoop->contains(PtrDef) &&
4372 while (!Worklist.
empty()) {
4374 for (
auto &
Op :
I->operands())
4381 auto UpdateMemOpUserCost = [
this, VF](
LoadInst *
LI) {
4385 for (
User *U :
LI->users()) {
4392 dbgs() <<
"LV: Memory widening: updating decision for load user "
4399 for (
auto *
I : AddrDefs) {
4411 LLVM_DEBUG(
dbgs() <<
"LV: Memory widening: updating decision for load "
4425 getMemoryInstructionCost(
4427 : getMemInstScalarizationCost(Member, VF);
4430 <<
"LV: Memory widening: updating decision for interleave member "
4431 << *Member <<
'\n');
4443 ForcedScalars[VF].insert(
I);
4454 return !OpI || !
TheLoop->contains(OpI) ||
4458 [
this](
Value *
Op) { return shouldConsiderInvariant(Op); }));
4470 return InstsToScalarize[VF][
I];
4473 auto ForcedScalar = ForcedScalars.find(VF);
4474 if (VF.
isVector() && ForcedScalar != ForcedScalars.end()) {
4475 auto InstSet = ForcedScalar->second;
4476 if (InstSet.count(
I))
4481 const auto &MinBWs = Config.getMinimalBitwidths();
4482 uint64_t InstrMinBWs = MinBWs.lookup(
I);
4483 Type *RetTy =
I->getType();
4486 auto *SE =
PSE.getSE();
4490 [[maybe_unused]]
auto HasSingleCopyAfterVectorization =
4495 auto Scalarized = InstsToScalarize.find(VF);
4496 assert(Scalarized != InstsToScalarize.end() &&
4497 "VF not yet analyzed for scalarization profitability");
4498 return !Scalarized->second.count(
I) &&
4500 auto *UI = cast<Instruction>(U);
4501 return !Scalarized->second.count(UI);
4510 assert(
I->getOpcode() == Instruction::GetElementPtr ||
4511 I->getOpcode() == Instruction::PHI ||
4512 (
I->getOpcode() == Instruction::BitCast &&
4513 I->getType()->isPointerTy()) ||
4514 HasSingleCopyAfterVectorization(
I, VF));
4520 !
TTI.getNumberOfParts(VectorTy))
4524 switch (
I->getOpcode()) {
4525 case Instruction::GetElementPtr:
4531 case Instruction::UncondBr:
4532 case Instruction::CondBr: {
4539 bool ScalarPredicatedBB =
false;
4542 (PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(0)) ||
4543 PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(1))) &&
4545 ScalarPredicatedBB =
true;
4547 if (ScalarPredicatedBB) {
4554 return (
TTI.getScalarizationOverhead(
4556 false,
true, Config.CostKind) +
4557 (
TTI.getCFInstrCost(Instruction::CondBr, Config.CostKind) *
4563 return TTI.getCFInstrCost(Instruction::UncondBr, Config.CostKind);
4571 case Instruction::Switch: {
4573 return TTI.getCFInstrCost(Instruction::Switch, Config.CostKind);
4575 return Switch->getNumCases() *
4576 TTI.getCmpSelInstrCost(
4578 toVectorTy(Switch->getCondition()->getType(), VF),
4582 case Instruction::PHI: {
4587 return TTI.getShuffleCost(
4596 Type *ResultTy = Phi->getType();
4601 Phi->users(), [
this](
User *U,
bool) ->
PHINode * {
4602 auto *Phi = dyn_cast<PHINode>(U);
4603 if (Phi && Phi->getParent() == TheLoop->getHeader())
4608 auto &ReductionVars =
Legal->getReductionVars();
4609 auto Iter = ReductionVars.find(HeaderUser);
4610 if (Iter != ReductionVars.end() &&
4612 Iter->second.getRecurrenceKind()))
4615 return (Phi->getNumIncomingValues() - 1) *
4616 TTI.getCmpSelInstrCost(
4617 Instruction::Select,
toVectorTy(ResultTy, VF),
4625 Legal->getReductionVars().contains(Phi) &&
4626 !Config.isInLoopReduction(Phi)) {
4628 Intrinsic::vp_merge,
toVectorTy(Phi->getType(), VF),
4629 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
4630 return TTI.getIntrinsicInstrCost(ICA, Config.CostKind);
4633 return TTI.getCFInstrCost(Instruction::PHI, Config.CostKind);
4635 case Instruction::UDiv:
4636 case Instruction::SDiv:
4637 case Instruction::URem:
4638 case Instruction::SRem:
4646 case Instruction::Add:
4647 case Instruction::Sub: {
4654 if (!RHS || RHS->getZExtValue() != 1)
4655 MulCost =
TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
4660 Type *ScalarTy =
I->getType();
4664 {PtrTy, ScalarTy, MaskTy});
4667 return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
4668 TTI.getArithmeticInstrCost(
I->getOpcode(), VectorTy,
4673 case Instruction::FAdd:
4674 case Instruction::FSub:
4675 case Instruction::Mul:
4676 case Instruction::FMul:
4677 case Instruction::FDiv:
4678 case Instruction::FRem:
4679 case Instruction::Shl:
4680 case Instruction::LShr:
4681 case Instruction::AShr:
4682 case Instruction::And:
4683 case Instruction::Or:
4684 case Instruction::Xor: {
4688 if (
I->getOpcode() == Instruction::Mul &&
4689 ((
TheLoop->isLoopInvariant(
I->getOperand(0)) &&
4690 PSE.getSCEV(
I->getOperand(0))->isOne()) ||
4691 (
TheLoop->isLoopInvariant(
I->getOperand(1)) &&
4692 PSE.getSCEV(
I->getOperand(1))->isOne())))
4697 Value *Op2 =
I->getOperand(1);
4703 auto Op2Info =
TTI.getOperandInfo(Op2);
4709 return TTI.getArithmeticInstrCost(
4710 I->getOpcode(), VectorTy, Config.CostKind,
4711 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4714 case Instruction::FNeg: {
4715 return TTI.getArithmeticInstrCost(
4716 I->getOpcode(), VectorTy, Config.CostKind,
4717 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4718 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4719 I->getOperand(0),
I);
4721 case Instruction::Select: {
4726 const Value *Op0, *Op1;
4737 return TTI.getArithmeticInstrCost(
4739 VectorTy, Config.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1},
4743 Type *CondTy =
SI->getCondition()->getType();
4749 Pred = Cmp->getPredicate();
4750 return TTI.getCmpSelInstrCost(
4751 I->getOpcode(), VectorTy, CondTy, Pred, Config.CostKind,
4752 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
I);
4754 case Instruction::ICmp:
4755 case Instruction::FCmp: {
4756 Type *ValTy =
I->getOperand(0)->getType();
4762 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
4763 "if both the operand and the compare are marked for "
4764 "truncation, they must have the same bitwidth");
4769 return TTI.getCmpSelInstrCost(
4772 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
I);
4774 case Instruction::Store:
4775 case Instruction::Load: {
4780 "CM decision should be taken at this point");
4787 return getMemoryInstructionCost(
I, VF);
4789 case Instruction::BitCast:
4790 if (
I->getType()->isPointerTy())
4793 case Instruction::ZExt:
4794 case Instruction::SExt:
4795 case Instruction::FPToUI:
4796 case Instruction::FPToSI:
4797 case Instruction::FPExt:
4798 case Instruction::PtrToInt:
4799 case Instruction::IntToPtr:
4800 case Instruction::SIToFP:
4801 case Instruction::UIToFP:
4802 case Instruction::Trunc:
4803 case Instruction::FPTrunc: {
4807 "Expected a load or a store!");
4820 if (
Legal->isCompressedLoadOrStore(
I))
4835 unsigned Opcode =
I->getOpcode();
4838 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
4841 CCH = ComputeCCH(
Store);
4844 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
4845 Opcode == Instruction::FPExt) {
4847 CCH = ComputeCCH(
Load);
4855 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
4856 Trunc->getSrcTy(), CCH, Config.CostKind,
4860 Type *SrcScalarTy =
I->getOperand(0)->getType();
4864 MinBWs.lookup(Op0AsInstruction));
4872 (
I->getOpcode() == Instruction::ZExt ||
4873 I->getOpcode() == Instruction::SExt))
4877 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
4878 Config.CostKind,
I);
4880 case Instruction::Call:
4882 case Instruction::ExtractValue:
4883 return TTI.getInstructionCost(
I, Config.CostKind);
4884 case Instruction::Alloca:
4889 return TTI.getArithmeticInstrCost(Instruction::Mul, RetTy, Config.CostKind);
4890 case Instruction::Freeze:
4894 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
4910 auto IsLiveOutDead = [
this, RequiresScalarEpilogue](
User *U) {
4911 return RequiresScalarEpilogue &&
4921 : Group->getInsertPos() ==
I;
4934 all_of(
I.users(), [
this, IsLiveOutDead](
User *U) {
4935 return VecValuesToIgnore.contains(U) ||
4936 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
4944 if (IsGroupAddress(&
I))
4947 DeadInterleavePointerOps.
push_back(PointerOp);
4958 for (
unsigned I = 0;
I != DeadInterleavePointerOps.
size(); ++
I) {
4962 Instruction *UI = cast<Instruction>(U);
4963 return !VecValuesToIgnore.contains(U) &&
4964 (!isAccessInterleaved(UI) || IsGroupAddress(UI));
4985 for (
unsigned I = 0;
I != DeadOps.
size(); ++
I) {
4997 if ((ThenEmpty && ElseEmpty) ||
4999 ElseBB->
phis().empty()) ||
5001 ThenBB->
phis().empty())) {
5013 return !VecValuesToIgnore.contains(U) &&
5014 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
5021 bool BecameScalarDead =
false;
5023 [
this](
User *U) { return ValuesToIgnore.contains(U); }))
5030 if (ProcessedDeadOps.
insert(
Op).second || BecameScalarDead)
5037 Legal->getReductionVars().values()) {
5047 CM->collectValuesToIgnore();
5048 Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
5054 Config.collectInLoopReductions();
5059 Legal->collectUnitStridePredicates();
5061 auto VPlan1 = tryToBuildVPlan1(*CM);
5065 LLVM_DEBUG(
dbgs() <<
"LV: VPlan created successfully. Loop can be "
5068 if (!OrigLoop->isInnermost()) {
5073 buildVPlans(*VPlan1, VF, VF, *CM);
5080 Config.computeMinimalBitwidths();
5083 if (CM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
5085 LLVM_DEBUG(
dbgs() <<
"LV: Invalidate all interleaved groups due to "
5086 "predication which requires masked-interleaved "
5088 if (CM->InterleaveInfo.invalidateGroups())
5092 CM->invalidateCostModelingDecisions();
5095 if (CM->foldTailByMasking())
5096 Legal->prepareToFoldTailByMasking();
5103 "UserVF ignored because it may be larger than the maximal safe VF",
5104 "InvalidUserVF", ORE, OrigLoop);
5109 CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
5110 buildVPlans(*VPlan1, UserVF, UserVF, *CM);
5114 if (!VPlans.empty() && (VPlans.front()->getSingleVF() == UserVF) &&
5116 cost(*VPlans.front(), UserVF,
nullptr, *CM).isValid())) {
5121 if (!planForEpilogueTF()) {
5125 CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
5126 buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
5135 "InvalidCost", ORE, OrigLoop);
5148 for (
const auto &VF : VFCandidates) {
5150 CM->collectNonVectorizedAndSetWideningDecisions(VF);
5160bool LoopVectorizationPlanner::planForEpilogueTF() {
5162 Opts, *CM, OrigLoop, ORE, *Legal, Config.
getHints(), &
TTI);
5163 if (EpilogueTailLoweringStatus !=
5169 EpilogueTailLoweringStatus, OrigLoop, PSE, LI, Legal,
TTI,
TLI, CM->AC,
5170 ORE, CM->GetBFI, CM->TheFunction, CM->InterleaveInfo, Config);
5172 assert(!EpilogueTfCM.isEpilogueAllowed() &&
5173 "Epilogue tail-folding is expected to be enabled");
5177 EpilogueTfCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
5178 EpilogueTfCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
5186 if (!MaxFactors || !EpilogueTfCM.foldTailByMasking()) {
5189 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5193 if (EpilogueTfCM.getTailFoldingStyle() !=
5196 "for DataAndControlFlow tail-folding style.",
5197 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5201 auto VPlan1 = tryToBuildVPlan1(EpilogueTfCM);
5205 "Failed to build initial tail-folded epilogue VPlan",
5206 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5210 Legal->prepareToFoldTailByMasking();
5214 EpilogueTfCM.collectNonVectorizedAndSetWideningDecisions(
5217 [[maybe_unused]]
size_t NumPlansBefore = VPlans.size();
5222 assert(VPlans.size() == NumPlansBefore + 1 &&
5223 "Expected to build a single tail-folded VPlan");
5224 VPlan &Plan = *VPlans.back();
5227 "Expected to build a valid tail-folded VPlan with epilogue VF");
5234 "Tail-folded epilogue has no active-lane-mask exit, so it may not "
5235 "terminate, can't apply tail-folded epilogue",
5236 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5246 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5249 LLVM_DEBUG(
dbgs() <<
"LV: Tail-folded epilogue VPlan is created\n");
5256 bool ReusePrintingSlotTracker)
5260#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5261 if (ReusePrintingSlotTracker)
5262 PlanForSlotTracker = &Plan;
5269 if (
Cost.isValid() &&
Opts.force_target_instruction_cost)
5275 return CM.ValuesToIgnore.contains(UI) ||
5276 (IsVector &&
CM.VecValuesToIgnore.contains(UI)) ||
5282 CM.setWideningDecision(
I, VF,
5287 return CM.isScalarWithPredication(
I, VF) ||
5288 CM.isUniformAfterVectorization(
I, VF) ||
CM.isForcedScalar(
I, VF) ||
5289 (VF.
isVector() &&
CM.isProfitableToScalarize(
I, VF));
5293 return CM.isMaskRequired(
I);
5319 if (Opts.force_target_instruction_cost)
5327 for (
Instruction *ForcedScalar : CostCtx.
CM.ForcedScalars[VF]) {
5333 dbgs() <<
"Cost of " << ForcedCost <<
" for VF " << VF
5334 <<
": forced scalar " << *ForcedScalar <<
"\n";
5345 switch (
I->getOpcode()) {
5346 case Instruction::SDiv:
5347 case Instruction::UDiv:
5348 case Instruction::SRem:
5349 case Instruction::URem:
5356 SmallPtrSet<Instruction *, 8> Replicated;
5360 Replicated.
insert(RepR.getUnderlyingInstr());
5361 for (
const auto &[Scalarized, ScalarCost] : CostCtx.
CM.InstsToScalarize[VF]) {
5362 if (!Replicated.
contains(Scalarized) || UseVPlanCostModel(Scalarized) ||
5367 dbgs() <<
"Cost of " << ScalarCost <<
" for VF " << VF
5368 <<
": profitable to scalarize " << *Scalarized <<
"\n";
5379static std::optional<VPExecutionFrequency>
5382 return std::nullopt;
5391 VPCostContext CostCtx(*TLI, Plan, PlanCM, Config,
5401 if (RU && Config.shouldConsiderRegPressureForVF(VF)) {
5403 TTI, Config.CostKind, Opts.force_target_num_vector_regs.value_or(0));
5404 LLVM_DEBUG(
dbgs() <<
"Spill costs for VF " << VF <<
": " << SpillCost
5410 unsigned EstimatedWidth =
5413 <<
" (Estimated cost per lane: ");
5419 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5423 SmallString<16> Str;
5424 CostPerLane.toString(Str, 3);
5433std::pair<VectorizationFactor, VPlan *>
5438 VPlan &FirstPlan = *VPlans[0];
5441 if (VPlans.size() == 1) {
5446 "must have a single scalar VF, UserVF or an outer loop");
5451 assert(VPlans[0]->getSingleVF() == UserVF &&
5452 "expected second plan to be for the forced UserVF");
5454 "expected first plan to be for the forced epilogue VF");
5460 ?
"Reciprocal Throughput\n"
5462 ?
"Instruction Latency\n"
5465 ?
"Code Size and Latency\n"
5470 "More than a single plan/VF w/o any plan having scalar VF");
5474 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ScalarCost <<
".\n");
5478 bool ForceVectorization =
5480 if (ForceVectorization) {
5487 VPlan *PlanForBestVF = &FirstPlan;
5490 for (
auto &
P : VPlans) {
5492 P->vectorFactors().end());
5498 if (!ForceVectorization &&
P->hasScalarTail() && ExactTC.
isFixed() &&
5500 ExactTC.
getFixedValue() <= TTI.getMinTripCountTailFoldingThreshold()) {
5506 return Config.shouldConsiderRegPressureForVF(VF);
5508 if (ConsiderRegPressure)
5511 for (
unsigned I = 0;
I < VFs.
size();
I++) {
5518 <<
"LV: Not considering vector loop of width " << VF
5519 <<
" because it will not generate any vector instructions.\n");
5525 <<
"LV: Not considering vector loop of width " << VF
5526 <<
" because it would cause replicated blocks to be generated,"
5527 <<
" which isn't allowed when optimizing for size.\n");
5532 cost(*
P, VF, ConsiderRegPressure ? &RUs[
I] :
nullptr, *CM);
5535 if (isMoreProfitable(CurrentFactor, BestFactor,
P->hasScalarTail())) {
5536 BestFactor = CurrentFactor;
5537 PlanForBestVF =
P.get();
5541 if (isMoreProfitable(CurrentFactor, ScalarFactor,
P->hasScalarTail()))
5542 ProfitableVFs.push_back(CurrentFactor);
5546 VPlan &BestPlan = *PlanForBestVF;
5549 "when vectorizing, the scalar cost must be computed.");
5552 return {BestFactor, &BestPlan};
5561 : Opts(Config.
getOpts()), OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
5562 Legal(Legal), CM(
std::
move(CM)), Config(Config), PSE(PSE), ORE(ORE),
5574 "Trying to execute plan with unsupported VF");
5576 "Trying to execute plan with unsupported UF");
5578 ++LoopsEarlyExitVectorized;
5581 *PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
5582 if (TTI.hasMultiVectorLoadStore(
5594 bool HasBranchWeights =
5596 if (HasBranchWeights) {
5597 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5599 BestVPlan, BestVF, VScale);
5605 *Legal->getRuntimePointerChecking()->getDiffChecks(),
5606 *PSE.getSE(), HasBranchWeights);
5607 ++LoopsPartialAliasVectorized;
5614 BestVF, BestUF, PSE);
5643 BestVF * BestUF, *OrigLoop->getHeader()->getParent());
5645 assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
5646 "loops not exiting via the latch without required epilogue?");
5648 VectorPH, HasTailFolded, RequiresScalarEpilogue,
5649 &BestVPlan.
getVFxUF(), MaxRuntimeStep);
5672 OrigLoop->getStartLoc(),
5673 OrigLoop->getHeader())
5674 <<
"Created vector loop never executes due to insufficient trip "
5688 Builder, &BestVPlan, OrigLoop->getParentLoop());
5690#ifdef EXPENSIVE_CHECKS
5691 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
5696 State.CFG.PrevBB = OrigLoop->getLoopPreheader();
5697 assert(State.CFG.PrevBB &&
"Invalid loop structure");
5699 State.CFG.PrevBB, State.CFG.PrevBB->getTerminator(), DT, LI,
nullptr,
5701 ?
"vec.epilog.scalar.ph"
5714 if (!Exit->hasPredecessors())
5725 TTI.getUnrollingPreferences(OrigLoop, SE, UP, ORE);
5742 MDNode *LID = OrigLoop->getLoopID();
5743 unsigned OrigLoopInvocationWeight = 0;
5744 std::optional<unsigned> OrigAverageTripCount =
5756 bool DisableRuntimeUnroll = !HasRuntimeChecks && !BestVF.
isScalar();
5758 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
5760 HeaderVPBB, BestVPlan,
5762 OrigAverageTripCount, OrigLoopInvocationWeight,
5764 DisableRuntimeUnroll, UnrollVectorizedLoop);
5768 legacyCSE(State.CFG.VPBB2IRBB.lookup(HeaderVPBB));
5777 VPBB, State.CFG.VPBB2IRBB.at(VPBB), &BestVPlan);
5778 replace(TrackedBlocks, VPBB, IRVPBB);
5781 return ExpandedSCEVs;
5785 return CM.isPredicatedInst(
I);
5789 return CM.TTI.prefersVectorizedAddressing();
5795 VPI->
getOpcode() == Instruction::Store) &&
5796 "Must be called with either a load or store");
5801 CM.getWideningDecision(
I, VF);
5803 "CM decision should be taken at this point.");
5806 if (CM.isScalarAfterVectorization(
I, VF) ||
5807 CM.isProfitableToScalarize(
I, VF))
5822 CM.getWideningDecision(
I,
Range.Start);
5829 Builder.setInsertPoint(VPI);
5838 if (VPI->
getOpcode() == Instruction::Load) {
5841 Builder.createWidenLoad(*
Load, Ptr, Mask, Consecutive,
Load->getAlign(),
5842 *VPI,
Load->getDebugLoc());
5845 LoadR->getDebugLoc());
5853 Store->getDebugLoc());
5854 return Builder.createWidenStore(*
Store, Ptr, StoredVal, Mask, Consecutive,
5855 Store->getAlign(), *VPI,
5856 Store->getDebugLoc());
5861 "Instruction should have been handled earlier");
5865 return CM.isScalarAfterVectorization(
I, VF) ||
5866 CM.isProfitableToScalarize(
I, VF) ||
5867 CM.isScalarWithPredication(
I, VF);
5878 case Instruction::SDiv:
5879 case Instruction::UDiv:
5880 case Instruction::SRem:
5881 case Instruction::URem:
5883 if (CM.isPredicatedInst(
I))
5884 return new VPWidenIntrinsicRecipe(
5888 case Instruction::Add:
5889 case Instruction::And:
5890 case Instruction::AShr:
5891 case Instruction::FAdd:
5892 case Instruction::FCmp:
5893 case Instruction::FDiv:
5894 case Instruction::FMul:
5895 case Instruction::FNeg:
5896 case Instruction::FRem:
5897 case Instruction::FSub:
5898 case Instruction::ICmp:
5899 case Instruction::LShr:
5900 case Instruction::Mul:
5901 case Instruction::Or:
5902 case Instruction::Select:
5903 case Instruction::Shl:
5904 case Instruction::Sub:
5905 case Instruction::Xor:
5906 case Instruction::Freeze:
5907 case Instruction::ExtractValue:
5914 if (VPI->
getOpcode() != Instruction::Store)
5937std::optional<CompressedPtrInfo>
5940 return std::nullopt;
5949 VPBuilder::InsertPointGuard Guard(Builder);
5950 Builder.setInsertPoint(VPI);
5958 Ptr = Builder.createConsecutiveVectorPointer(Ptr, AccessTy,
5963 if (VPI->
getOpcode() == Instruction::Load)
5965 Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(AccessTy)},
5970 {StoredValue, Ptr, Mask}, AccessTy,
5978 Legal->isInvariantAddressOfReduction(
SI->getPointerOperand())) {
5980 if (Legal->isInvariantStoreOfReduction(
SI)) {
5987 [[maybe_unused]]
auto *Rdx =
5990 "Store of reduction thats not the backedge value?");
5992 SI, {Val, Addr},
true ,
nullptr , *VPI, *VPI,
5994 FinalRedStoresBuilder.
insert(Recipe);
6007 [&](
ElementCount VF) {
return CM.isUniformAfterVectorization(
I, VF); },
6010 bool IsPredicated = CM.isPredicatedInst(
I);
6018 case Intrinsic::assume:
6019 case Intrinsic::lifetime_start:
6020 case Intrinsic::lifetime_end:
6042 VPValue *BlockInMask =
nullptr;
6043 if (!IsPredicated) {
6047 LLVM_DEBUG(
dbgs() <<
"LV: Scalarizing and predicating:" << *
I <<
"\n");
6058 assert((
Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6060 "Should not predicate a uniform recipe");
6075 assert(!R->isPhi() &&
"phis must be handled earlier");
6078 "Call should have been handled by makeCallWideningDecisions");
6088 "Should have been handled prior to this!");
6092 if (VPI->
getOpcode() == Instruction::ExtractValue &&
6096 return tryToWiden(VPI);
6097 if (VPI->
getOpcode() == Instruction::ExtractValue)
6100 nullptr, *VPI, *VPI,
6103 if (!shouldWiden(Instr,
Range))
6106 if (VPI->
getOpcode() == Instruction::GetElementPtr) {
6120 return tryToWiden(VPI);
6143 "loop body and original loop must have the same blocks");
6151 if (HeaderFreq == 0)
6158 Edges += VPBB->getNumSuccessors();
6161 for (
const auto &[VPBB, BB] :
6164 std::optional<VPExecutionFrequency> Freq =
6173 std::min(
BBFreq, HeaderFreq), HeaderFreq);
6178 errs() <<
"Block frequency mismatch for " << VPBB->getName() <<
": VPlan "
6179 << Computed <<
", BlockFrequencyInfo " <<
Expected <<
"\n";
6188 bool IsInnerLoop = OrigLoop->isInnermost();
6193 std::optional<LoopVersioning> LVer;
6195 const LoopAccessInfo *LAI = Legal->getLAI();
6197 LI, DT, PSE.getSE());
6202 LVer->prepareNoAliasMetadata();
6209 OrigLoop, *LI, Legal->getWidestInductionType(), PSE,
6210 LVer ? &*LVer :
nullptr, GetBPI);
6212 VPDominatorTree VPDT(*VPlan0);
6213 if (
const LoopAccessInfo *LAI = Legal->getLAI())
6221 "execution frequencies do not match the loop's block frequencies");
6228 VPDT, Legal->getInductionVars(), Legal->getReductionVars(),
6229 Legal->getConditionalInductions(), Legal->getFixedOrderRecurrences(),
6230 Config.getInLoopReductions(), Config.getHints().allowReordering())) {
6234 if (
const LoopAccessInfo *LAI = Legal->getLAI())
6240 if (!IsInnerLoop && Opts.vplan_outer_loop_legality_checks &&
6244 "Cannot prove outer loop memory accesses safe",
6245 "cannot prove memory accesses in outer loop are safe to vectorize",
6246 "CantProveOuterLoopMemSafety", ORE, OrigLoop);
6251 bool ForceVectorization =
6253 bool OptForSize = !ForceVectorization && PlanCM.EpilogueLoweringStatus ==
6255 unsigned SCEVCheckThreshold = ForceVectorization
6256 ? Opts.pragma_vectorize_scev_check_threshold
6257 : Opts.vectorize_scev_check_threshold;
6259 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6268 if (Legal->hasUncountableEarlyExit()) {
6276 ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
6300 auto MaxVFTimes2 = MaxVF * 2;
6302 VFRange SubRange = {VF, MaxVFTimes2};
6303 auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.
duplicate()),
6314 Config.getMinimalBitwidths());
6319 Config.getMaxSafeElements());
6321 VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
6327 VPlans.push_back(std::move(
P));
6336 VPlans.push_back(std::move(Plan));
6347 if (Plan->isOuterLoop()) {
6348 for (ElementCount VF :
Range)
6351 *Plan, *TLI, PSE, OrigLoop))
6358 using namespace llvm::VPlanPatternMatch;
6359 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6366 bool RequiresScalarEpilogueCheck =
6368 [&PlanCM](ElementCount VF) {
6373 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6374 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6376 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6377 "second successor must be scalar preheader");
6378 BranchOnCond->setOperand(0, Plan->getFalse());
6385 bool IVUpdateMayOverflow =
false;
6386 for (ElementCount VF :
Range)
6394 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6400 m_VPInstruction<Instruction::Add>(
6402 "Did not find the canonical IV increment");
6415 for (InterleaveGroup<Instruction> *IG :
6417 auto ApplyIG = [IG, &PlanCM](ElementCount VF) ->
bool {
6424 "Unsupported interleave factor for scalable vectors");
6429 InterleaveGroups.
insert(IG);
6436 VPRecipeBuilder RecipeBuilder(*Plan, Legal, PlanCM, Builder);
6439 RecipeBuilder, *PSE.getSE()))
6445 VPCostContext CostCtx(*TLI, *Plan, PlanCM, Config);
6448 RecipeBuilder, CostCtx);
6454 RecipeBuilder, CostCtx))
6468 make_range(VPBB->getFirstNonPhi(), VPBB->end()),
6469 IsaPred<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6470 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6471 VPWidenCallRecipe, VPWidenIntrinsicRecipe,
6472 VPVectorPointerRecipe, VPVectorEndPointerRecipe,
6473 VPHistogramRecipe, VPInstruction>) &&
6474 "Unexpected recipe");
6475 for (VPInstruction &VPI :
6486 Builder.setInsertPoint(&VPI);
6488 VPRecipeBase *Recipe =
6489 RecipeBuilder.tryToCreateWidenNonPhiRecipe(&VPI,
Range);
6491 Recipe = RecipeBuilder.handleReplication(&VPI,
Range);
6492 Builder.insert(Recipe);
6498 "Unexpected multidef recipe");
6506 "entry block must be set to a VPRegionBlock having a non-empty entry "
6517 addReductionResultComputation(Plan,
Range.Start, PlanCM);
6558 *OrigLoop, CostCtx,
Range);
6560 if (Legal->hasUncountableExitWithSideEffects())
6565 if (
Range.Start.isScalar())
6568 for (ElementCount VF :
Range)
6570 Plan->setName(
"Initial VPlan");
6581void LoopVectorizationPlanner::addReductionResultComputation(
6583 using namespace VPlanPatternMatch;
6584 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6585 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6587 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->
end())));
6589 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6591 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
6604 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6610 if (Blend->getNumIncomingValues() == 2 &&
6611 Blend->getMask(0) == HeaderMask) {
6613 Blend->getMask(0), Blend->getIncomingValue(0),
6614 Blend->getIncomingValue(1), {},
"", *Blend);
6616 Blend->eraseFromParent();
6621 auto *NewExitingVPV = OrigExitingVPV;
6638 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6644 VPInstruction *FinalReductionResult;
6645 VPBuilder::InsertPointGuard Guard(Builder);
6646 Builder.setInsertPoint(IP);
6651 auto *AnyOfSelect = findSingleUser<VPSingleDefRecipe>(
6655 bool TrueValIsPhi = AnyOfSelect->getOperand(1) == PhiR;
6657 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(2)
6658 : AnyOfSelect->getOperand(1);
6664 VPValue *
Cmp = AnyOfSelect->getOperand(0);
6667 if (VPRecipeBase *CmpR =
Cmp->getDefiningRecipe())
6669 Builder.setInsertPoint(AnyOfSelect);
6674 Cmp = Builder.createNot(Cmp);
6681 VPValue *NewExiting = Builder.createOr(NewPhiR, Cmp);
6688 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6690 std::function<void(VPSingleDefRecipe *)> CloneChain =
6691 [&](VPSingleDefRecipe *Old) {
6695 for (VPValue *
Op : Old->operands()) {
6701 VPSingleDefRecipe *
New;
6703 New =
B->cloneWithOperands(NewOps);
6705 New =
W->cloneWithOperands(NewOps);
6707 New = Rep->cloneWithOperands(NewOps);
6710 New->insertBefore(Old);
6711 Substitutions[Old] =
New;
6714 if (OrigExitingVPV != AnyOfSelect) {
6716 NewExiting = Substitutions.
lookup(OrigExitingVPV);
6718 NewPhiR->setOperand(1, NewExiting);
6721 Builder.setInsertPoint(IP);
6722 FinalReductionResult =
6723 Builder.createAnyOfReduction(NewExiting, NewVal, Start, ExitDL);
6728 VPValue *ReductionOp = NewExitingVPV;
6731 assert(!PhiR->
isInLoop() &&
"Unexpected truncated inloop reduction!");
6733 "Unexpected truncated min-max recurrence!");
6735 ExtendOpc = RdxDesc.
isSigned() ? Instruction::SExt : Instruction::ZExt;
6737 VPBuilder::InsertPointGuard Guard(Builder);
6738 Builder.setInsertPoint(
6739 std::next(NewExitingVPV->getDefiningRecipe()->getIterator()));
6741 Builder.createWidenCast(Instruction::Trunc, NewExitingVPV, RdxTy);
6742 VPWidenCastRecipe *Extnd =
6743 Builder.createWidenCast(ExtendOpc, ReductionOp, PhiTy);
6751 FinalReductionResult = Builder.createNaryOp(
6753 if (ExtendOpc != Instruction::CastOpsEnd)
6754 FinalReductionResult = Builder.createScalarCast(
6755 ExtendOpc, FinalReductionResult, PhiTy, {});
6760 for (
auto *U :
to_vector(OrigExitingVPV->users())) {
6762 if (FinalReductionResult == U || Parent->getParent())
6766 if (
match(U, m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
6768 match(U, m_VPInstruction<Instruction::ICmp>())))
6770 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
6786 VPBuilder PHBuilder(Plan->getVectorPreheader());
6787 VPValue *Iden = Plan->getOrAddLiveIn(
6789 auto *ScaleFactorVPV = Plan->getConstantInt(32, 1);
6790 VPValue *StartV = PHBuilder.createNaryOp(
6802 bool HasBranchWeights)
const {
6803 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
6804 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
6805 assert((!Config.OptForSize ||
6807 "Cannot SCEV check stride or overflow when optimizing for size");
6809 SCEVCheckBlock, HasBranchWeights);
6811 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
6812 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
6816 "Memory runtime checks are not supported for outer loops yet");
6818 if (Config.OptForSize) {
6821 "Cannot emit memory checks when optimizing for size, unless forced "
6825 OrigLoop->getStartLoc(),
6826 OrigLoop->getHeader())
6827 <<
"Code-size may be reduced by not forcing "
6828 "vectorization, or by source-code modifications "
6829 "eliminating the need for runtime checks "
6830 "(e.g., adding 'restrict').";
6833 const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
6835 RTChecks.eraseMemCheckBlock();
6837 OrigLoop, *PSE.getSE(), VF, UF, HasBranchWeights);
6843 ElementCount MinProfitableTripCount,
bool TailFoldedEpilogue)
const {
6855 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
6877 if (Opts.tail_folding_policy) {
6878 switch (*Opts.tail_folding_policy) {
6898 if (
TTI->preferTailFoldingOverEpilogue(&TFI))
6913 if (S->getValueOperand()->getType()->isFloatTy())
6923 while (!Worklist.
empty()) {
6925 if (!L->contains(
I))
6927 if (!Visited.
insert(
I).second)
6937 I->getDebugLoc(), L->getHeader())
6938 <<
"floating point conversion changes vector width. "
6939 <<
"Mixed floating point precision requires an up/down "
6940 <<
"cast that will negatively impact performance.";
6943 for (
Use &
Op :
I->operands())
6960 for (
auto *PredVPBB : ExitVPBB->getPredecessors()) {
6966 << PredVPBB->getName() <<
":\n");
6967 Cost = std::max(Cost, PredVPBB->cost(VF, CostCtx));
6987 std::optional<unsigned> VScale) {
6999 <<
"LV: Interleaving only is not profitable due to runtime checks\n");
7016 InstructionCost TotalCost = RtC + std::max(EarlyExitCost, MiddleCost);
7067 uint64_t MinTC = std::max(MinTC1, MinTC2);
7069 MinTC =
alignTo(MinTC, IntVF);
7073 dbgs() <<
"LV: Minimum required TC for runtime checks to be profitable:"
7080 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is not beneficial: expected "
7081 "trip count < minimum profitable VF ("
7092 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced),
7093 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {}
7120 VPInstruction *Freeze = Builder.createFreeze(OrigStart, {},
"fr");
7128 [[maybe_unused]]
bool MatchedTC =
7130 assert(MatchedTC &&
"must match vector trip count");
7144 : MiddleVPBB->
end());
7147 auto *ResumePhi = cast<VPPhi>(R.getOperand(0));
7148 assert(ResumePhi->getNumIncoming() == 2 &&
7149 ResumePhi->getIncomingBlock(0) == MiddleVPBB &&
7150 "expected resume phi merging middle block and bypass value");
7151 return MiddleBuilder.createNaryOp(VPInstruction::ResumeForEpilogue,
7152 ResumePhi->operands());
7157 ResumePhi->replaceAllUsesWith(ResumePhi->getOperand(1));
7158 ResumePhi->eraseFromParent();
7160 return {VectorTCMarker, std::move(ResumeValues)};
7166 VPlan &MainPlan,
VPlan &Plan,
const SCEV2ValueTy &ExpandedSCEVs,
7174 for (
auto [HeaderPhi, ResumeForEpi] :
7176 IRPhiToResumeForEpi[&
cast<VPIRPhi>(HeaderPhi).getIRPhi()] = ResumeForEpi;
7181 Header->
setName(
"vec.epilog.vector.body");
7185 for (
auto [R, ResumeForEpi] :
7194 "expected resume phis unless the epilogue is tail-folded");
7197 ResumePhi->setIncomingValueForBlock(
7198 EpilogueCheck, Plan.
getOrAddLiveIn(ResumeForEpi->getUnderlyingValue()));
7204 "vector preheader must be entered from the epilogue check first");
7214 if (MainV == BypassV)
7225 VPValue *VPV = GetResumeValue(
Markers.VectorTC,
"vec.epilog.resume.val");
7228 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7230 auto *VPI = dyn_cast<VPInstruction>(U);
7239 unsigned Opc = VPI->getOpcode();
7240 return Instruction::isCast(Opc) ||
7241 is_contained({Instruction::Add, Instruction::Mul,
7246 "the canonical IV should only be used by its increment, casts, "
7247 "stride scaling, ScalarIVSteps or DerivedIV when resetting the start "
7249 VPBuilder Builder(Header->getFirstNonPhi());
7254 assert(
Increment &&
"Must have a canonical IV increment at this point");
7255 IV->replaceUsesWithIf(
7259 Increment->replaceAllUsesWith(OffsetIVInc);
7263 VPBuilder PHBuilder(VectorPH->getFirstNonPhi());
7279 assert(RdxResult &&
"expected to find reduction result");
7287 VPValue *SentinelVPV =
nullptr;
7288 bool IsFindIV =
any_of(RdxResult->users(), [&](
VPUser *U) {
7289 return match(U, VPlanPatternMatch::m_SpecificICmp(
7290 ICmpInst::ICMP_NE, m_Specific(RdxResult),
7291 m_VPValue(SentinelVPV)));
7294 RecurKind RK = ReductionPhi->getRecurrenceKind();
7295 ResumeVPV = GetResumeValue(ResumeForEpi,
"bc.merge.rdx");
7301 m_VPInstruction<Instruction::Freeze>(
m_VPValue()))) &&
7302 "expected live-in or Freeze");
7311 "sentinel must be a live-in to be used in the preheader");
7315 for (
VPUser *U : RdxResult->users()) {
7325 ResumeVPV = PHBuilder.createSelect(Cmp, SentinelVPV, ResumeVPV);
7329 ReductionPhi->setStartValue(
7335 "unexpected start value");
7339 if (ReductionPhi->getVFScaleFactor() > 1 &&
7342 assert((
Sub->getOpcode() == Instruction::Sub ||
7343 Sub->getOpcode() == Instruction::FSub) &&
7344 "Unexpected opcode");
7346 "Expected operand to match the original start value of the "
7350 [[maybe_unused]]
auto StartValueIsIdentity = [&] {
7351 Value *IdentityValue =
7353 ReductionPhi->getFastMathFlagsOrNone());
7355 return StartValue && StartValue->getValue() == IdentityValue;
7357 assert(StartValueIsIdentity() &&
7358 "Expected start value for partial sub-reduction to be zero "
7359 "(or negative zero)");
7361 Sub->setOperand(0, ResumeVPV);
7366 NewStart->setOperand(0, ResumeVPV);
7367 NewStart->insertBefore(VPI);
7368 ReductionPhi->setStartValue(NewStart);
7377 VPBuilder EntryBuilder(Plan.getVectorPreheader());
7378 auto *EntryALM = EntryBuilder.createNaryOp(
7380 R.getDebugLoc(),
"active.lane.mask.entry");
7390 GetResumeValue(IRPhiToResumeForEpi.at(IndPhi),
"bc.resume.val");
7392 if (IntOrFpIV && IntOrFpIV->isTruncated()) {
7398 VPValue *OrigStart = Plan.getOrAddLiveIn(
7399 IntOrFpIV->getInductionDescriptor().getStartValue());
7400 VPValue *NarrowStart = IntOrFpIV->getStartValue();
7401 if (NarrowStart != OrigStart) {
7403 VPValue *
Offset = OffsetBuilder.createSub(NarrowStart, OrigStart);
7404 ResumeVPV = OffsetBuilder.createAdd(ResumeVPV,
Offset);
7408 assert(ResumeVPV &&
"Must have a resume value");
7417 assert(ExpandedSCEVs.contains(ExpandR.getSCEV()) &&
7418 "Epilogue plan needs a SCEV not expanded for the main loop");
7420 Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR.getSCEV()));
7422 if (Plan.getTripCount() == &ExpandR)
7423 Plan.resetTripCount(ExpandedVal);
7424 ExpandR.eraseFromParent();
7431 Plan.getOrAddLiveIn(
Markers.VectorTC->getUnderlyingValue()),
7432 Plan.requiresScalarEpilogue(), EpilogueVF, MainLoopStep,
7433 EpilogueLoopStep, SE);
7438 assert((Opts.enable_vplan_native_path || L->isInnermost()) &&
7439 "VPlan-native path is not enabled. Only process inner loops.");
7442 << L->getHeader()->getParent()->getName() <<
"' from "
7443 << L->getLocStr() <<
"\n");
7446 Opts, L, InterleaveOnlyWhenForced || !Opts.interleave_loops, *
ORE,
TTI);
7449 dbgs() <<
"LV: Loop hints:"
7460 Function *
F = L->getHeader()->getParent();
7471 !Opts.vectorize_loops)) {
7481 L->getHeader(),
PSI,
7488 LoopVectorizationLegality LVL(Opts, L, PSE,
DT,
TTI,
TLI,
F, *
LAIs,
LI,
ORE,
7489 &Requirements, &Hints,
DB,
AC,
7491 if (!LVL.
canVectorize(Opts.enable_vplan_native_path)) {
7492 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Cannot prove legality.\n");
7497 bool IsInnerLoop = L->isInnermost();
7501 LLVM_DEBUG(
dbgs() <<
"LV: cannot compute the outer-loop trip count\n");
7506 if (!Opts.enable_early_exit_vectorization) {
7508 "early exit is not enabled",
7509 "UncountableEarlyExitLoopsDisabled",
ORE, L);
7513 !Opts.enable_early_exit_vectorization_with_side_effects) {
7515 "early exit and side effects is not enabled",
7516 "UncountableEarlyExitSideEffectLoopsDisabled",
7524 bool UseInterleaved =
7525 IsInnerLoop &&
valueOr(Opts.enable_interleaved_mem_accesses,
7526 TTI->enableInterleavedAccessVectorization());
7529 if (UseInterleaved) {
7534 if (L->getExitingBlock() != L->getLoopLatch() &&
7544 "requiring a scalar epilogue is unsupported",
7545 "UncountableEarlyExitUnsupported",
ORE, L);
7556 if (ExpectedTC && ExpectedTC->isFixed() &&
7557 ExpectedTC->getFixedValue() < Opts.vectorizer_min_trip_count) {
7558 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop with a very small trip count. "
7559 <<
"This loop is worth vectorizing only if no scalar "
7560 <<
"epilogue is required.");
7562 LLVM_DEBUG(
dbgs() <<
" But vectorizing was explicitly forced.\n");
7581 ExpectedTC->getFixedValue() <=
7582 TTI->getMinTripCountTailFoldingThreshold()))
7589 if (
F->hasFnAttribute(Attribute::NoImplicitFloat)) {
7591 "Can't vectorize when the NoImplicitFloat attribute is used",
7592 "loop not vectorized due to NoImplicitFloat attribute",
7593 "NoImplicitFloat",
ORE, L);
7603 TTI->isFPVectorizationPotentiallyUnsafe()) {
7605 "Potentially unsafe FP op prevents vectorization",
7606 "loop not vectorized due to unsafe FP support.",
"UnsafeFP",
ORE, L);
7612 bool AllowOrderedReductions =
7613 valueOr(Opts.force_ordered_reductions,
TTI->enableOrderedReductions());
7618 ExactFPMathInst->getDebugLoc(),
7619 ExactFPMathInst->getParent())
7620 <<
"loop not vectorized: cannot prove it is safe to reorder "
7621 "floating-point operations";
7623 LLVM_DEBUG(
dbgs() <<
"LV: loop not vectorized: cannot prove it is safe to "
7624 "reorder floating-point operations\n");
7635 std::make_unique<LoopVectorizationCostModel>(
7636 SEL, L, PSE,
LI, &LVL, *
TTI,
TLI,
AC,
ORE,
GetBFI,
F, IAI, Config),
7650 LVP.
plan(UserVF, UserIC);
7656 if (!IsInnerLoop && Opts.vplan_build_outerloop_stress_test)
7659 if (IsInnerLoop &&
ORE->allowExtraAnalysis(
LV_NAME))
7663 "Did not expect to alias-mask outer loop");
7672 unsigned SelectedIC = UserIC > 0 ? UserIC : IC;
7675 if (VF.Width.
isVector() || SelectedIC > 1) {
7682 if ((Checks.getSCEVChecks().first &&
7683 match(Checks.getSCEVChecks().first,
m_One())) ||
7684 (Checks.getMemRuntimeChecks().first &&
7685 match(Checks.getMemRuntimeChecks().first,
m_One()))) {
7687 "runtime checks are known to fail, so we will never enter the "
7689 "RuntimeChecksNeverEnterVectorLoop",
ORE, L);
7695 bool ForceVectorization =
7697 assert((IsInnerLoop || ForceVectorization) &&
7698 "Outer loops must be forced to vectorize");
7701 if (!ForceVectorization &&
7706 DEBUG_TYPE,
"CantReorderMemOps", L->getStartLoc(),
7708 <<
"loop not vectorized: cannot prove it is safe to reorder "
7709 "memory operations";
7718 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
7719 bool VectorizeLoop =
true, InterleaveLoop =
true;
7723 dbgs() <<
"LV: Vectorization is possible but not beneficial.\n");
7725 "VectorizationNotBeneficial",
7726 "the cost-model indicates that vectorization is not beneficial"};
7728 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is not possible. Failed to "
7729 "create any vector VPlans.\n");
7730 VecDiagMsg = {
"VectorizationNotPossible",
7731 "vectorization is not possible"};
7733 VectorizeLoop =
false;
7738 "UserIC should only be ignored due to unsafe dependencies");
7739 LLVM_DEBUG(
dbgs() <<
"LV: Ignoring user-specified interleave count.\n");
7740 IntDiagMsg = {
"InterleavingUnsafe",
7741 "Ignoring user-specified interleave count due to possibly "
7742 "unsafe dependencies in the loop."};
7743 InterleaveLoop =
false;
7747 LLVM_DEBUG(
dbgs() <<
"LV: Ignoring UserIC, because vectorization and "
7748 "interleaving should be avoided up front\n");
7749 IntDiagMsg = {
"InterleavingAvoided",
7750 "Ignoring UserIC, because interleaving was avoided up front"};
7751 InterleaveLoop =
false;
7752 }
else if (IC == 1 && UserIC <= 1) {
7757 "InterleavingNotBeneficial",
7758 "the cost-model indicates that interleaving is not beneficial"};
7760 IntDiagMsg.first =
"InterleavingNotBeneficialAndDisabled";
7761 IntDiagMsg.second +=
7762 " and is explicitly disabled or interleave count is set to 1";
7765 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving is not possible. Failed to create"
7766 <<
" any vplans\n");
7767 IntDiagMsg = {
"InterleavingNotPossible",
"interleaving is not possible"};
7769 InterleaveLoop =
false;
7770 }
else if (IC > 1 && UserIC == 1) {
7772 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving is beneficial but is explicitly "
7774 IntDiagMsg = {
"InterleavingBeneficialButDisabled",
7775 "the cost-model indicates that interleaving is beneficial "
7776 "but is explicitly disabled or interleave count is set to 1"};
7777 InterleaveLoop =
false;
7783 if (!VectorizeLoop && InterleaveLoop && LVL.
hasHistograms()) {
7784 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving without vectorization due "
7785 <<
"to histogram operations.\n");
7787 "HistogramPreventsScalarInterleaving",
7788 "Unable to interleave without vectorization due to constraints on "
7789 "the order of histogram operations"};
7790 InterleaveLoop =
false;
7794 IC = UserIC > 0 ? UserIC : IC;
7799 <<
"LV: Not interleaving due to partial aliasing vectorization.\n");
7801 "PartialAliasingVectorization",
7802 "Unable to interleave due to partial aliasing vectorization."};
7803 InterleaveLoop =
false;
7809 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving due to EE with side effects.\n");
7810 IntDiagMsg = {
"EEWithSideEffectsPreventsInterleaving",
7811 "Unable to interleave due to early exit with side effects."};
7812 InterleaveLoop =
false;
7818 dbgs() <<
"LV: Not interleaving due to conditional inductions.\n");
7819 IntDiagMsg = {
"ConditionalInductionsPreventInterleaving",
7820 "Unable to interleave due to conditional inductions."};
7821 InterleaveLoop =
false;
7826 if (!VectorizeLoop && !InterleaveLoop) {
7830 L->getStartLoc(), L->getHeader())
7831 << VecDiagMsg.second;
7835 L->getStartLoc(), L->getHeader())
7836 << IntDiagMsg.second;
7841 if (!VectorizeLoop && InterleaveLoop) {
7845 L->getStartLoc(), L->getHeader())
7846 << VecDiagMsg.second;
7848 }
else if (VectorizeLoop && !InterleaveLoop) {
7849 LLVM_DEBUG(
dbgs() <<
"LV: Found a vectorizable loop (" << VF.Width
7850 <<
") in " << L->getLocStr() <<
'\n');
7853 L->getStartLoc(), L->getHeader())
7854 << IntDiagMsg.second;
7856 }
else if (VectorizeLoop && InterleaveLoop) {
7857 LLVM_DEBUG(
dbgs() <<
"LV: Found a vectorizable loop (" << VF.Width
7858 <<
") in " << L->getLocStr() <<
'\n');
7864 using namespace ore;
7869 <<
"interleaved loop (interleaved count: "
7870 << NV(
"InterleaveCount", IC) <<
")";
7891 VPlan &BestPlan = *BestPlanPtr;
7893 std::unique_ptr<VPlan> EpiPlan =
7895 bool HasBranchWeights =
7898 VPlan &BestEpiPlan = *EpiPlan;
7899 VPlan &BestMainPlan = BestPlan;
7919 L->getLoopPredecessor()->getTerminator()->getDebugLoc(),
7923 dbgs() <<
"Create Skeleton for epilogue vectorized loop (first pass)\n"
7924 <<
"Main Loop VF:" << VF.Width <<
", Main Loop UF:" << IC
7925 <<
", Epilogue Loop VF:" << EpilogueVF <<
", Epilogue Loop UF:1\n";
7931 VF.Width, IC, BestMainPlan, Checks.hasChecks(),
7937 dbgs() <<
"intermediate fn:\n" << *L->getHeader()->getParent() <<
"\n";
7942 EntryBB->
setName(
"iter.check");
7945 ->setName(
"vec.epilog.iter.check");
7950 VF.Width, IC, EpilogueVF, Config,
7954 dbgs() <<
"Create Skeleton for epilogue vectorized loop (second pass)\n"
7955 <<
"Epilogue Loop VF:" << EpilogueVF <<
", Epilogue Loop UF:1\n";
7958 EpilogueVF, 1, BestEpiPlan, Checks.hasChecks(),
7961 dbgs() <<
"final fn:\n" << *L->getHeader()->getParent() <<
"\n";
7963 ++LoopsEpilogueVectorized;
7966 VF.MinProfitableTripCount,
7973 LVP.
executePlan(VF.Width, IC, BestPlan, Checks.hasChecks());
7977#ifdef EXPENSIVE_CHECKS
7978 assert(
DT->verify(DominatorTree::VerificationLevel::Fast) &&
7979 "DT not preserved correctly");
7995 if (!
TTI->getNumberOfRegisters(
TTI->getRegisterClassForType(
true)) &&
8007 for (
const auto &L : *
LI)
8020 LoopsAnalyzed += Worklist.
size();
8023 while (!Worklist.
empty()) {
8044 assert((!
Changed ||
DT->verify(DominatorTree::VerificationLevel::Fast)) &&
8045 "DT not preserved correctly");
8047 "Invalid IR produced by LoopVectorize");
8075 auto ClearStaleCycleInfo = [
this, &AM, &
F] {
8080 ClearStaleCycleInfo();
8084 ClearStaleCycleInfo();
8088 if (!Result.MadeAnyChange)
8102 if (Result.MadeCFGChange) {
8117 static_cast<PassInfoMixin<LoopVectorizePass> *
>(
this)->
printPipeline(
8118 OS, MapClassName2PassName);
8121 OS << (InterleaveOnlyWhenForced ?
"" :
"no-") <<
"interleave-forced-only;";
8122 OS << (VectorizeOnlyWhenForced ?
"" :
"no-") <<
"vectorize-forced-only;";
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Lower Kernel Arguments
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static InstructionCost getCost(const AnalysisOptions &Opts, Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI)
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
This file defines the DenseMap class.
This is the interface for a simple mod/ref and alias analysis over globals.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static cl::opt< ElementCount, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE, Loop *L)
Get the maximum trip count for L from the SCEV unsigned range, excluding zero from the range.
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static void legacyCSE(BasicBlock *BB)
FIXME: This legacy common-subexpression-elimination routine is scheduled for removal,...
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode)
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static EpilogueLowering getEpilogueTailLowering(const VectorizeOptions &Opts, const LoopVectorizationCostModel &MainCM, const Loop *L, OptimizationRemarkEmitter *ORE, LoopVectorizationLegality &LVL, const LoopVectorizeHints &Hints, const TargetTransformInfo *TTI)
Determine how to lower the epilogue for the vector epilogue loop.
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static std::optional< VPExecutionFrequency > getRecordedExecutionFrequency(const VPBasicBlock *VPBB)
Returns the frequency with which VPBB executes, as recorded on its recipes.
static void collectSupportedLoops(const VectorizeOptions &Opts, Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF, bool MaskRequired, const TargetLibraryInfo *TLI)
Returns true iff CI has a library vector variant usable at VF.
static constexpr uint32_t MinItersBypassWeights[]
static Function * getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF, bool MaskRequired, const TargetLibraryInfo *TLI)
Returns the vector library variant function of CI usable at VF, respecting MaskRequired,...
static bool isExplicitVecOuterLoop(const VectorizeOptions &Opts, Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static void preparePlanForEpilogueVectorLoop(VPlan &MainPlan, VPlan &Plan, const SCEV2ValueTy &ExpandedSCEVs, ElementCount MainLoopVF, unsigned MainLoopUF, ElementCount EpilogueVF, VFSelectionContext &Config, ScalarEvolution &SE, const MainPlanResumeMarkers &Markers, VPBasicBlock *MainCheck)
Prepare Plan for vectorizing the epilogue loop.
static bool hasForcedEpilogueVF()
static bool hasConditionalInductionPhi(VPlan &Plan)
Returns true if the VPlan contains a VPConditionalInductionPHIRecipe.
static void printOptimizedVPlan(VPlan &)
static bool isIndvarOverflowCheckKnownFalse(const VectorizeOptions &Opts, const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static const SCEV * getAddressAccessSCEV(Value *Ptr, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets the address access SCEV for Ptr, if it should be used for cost modeling according to isAddressSC...
static std::optional< ElementCount > getSmallBestKnownTC(const VectorizeOptions &Opts, PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true, bool CanExcludeZeroTrips=false, bool ComputeUpperBoundOnly=false)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static bool useMaskedInterleavedAccesses(const VectorizeOptions &Opts, const TargetTransformInfo &TTI)
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static bool hasFindLastReductionPhi(VPlan &Plan)
Returns true if the VPlan contains a VPReductionPHIRecipe with FindLast recurrence kind.
static EpilogueLowering getEpilogueLowering(const VectorizeOptions &Opts, LoopVectorizeHints &Hints, bool OptForSize, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop, LoopInfo *LI, LoopVectorizationCostModel &CM)
Cross-check the execution frequencies recorded in Plan against BlockFrequencyInfo for the blocks of O...
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, EpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static cl::opt< ElementCount > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(ElementCount::getFixed(1)), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops. Note: This allows all scalable VFs >= vscale x 1."))
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
This file implements a map that provides insertion order iteration.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
static const CodeGenOptions & getOpts(const TargetMachine &TM)
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
A manager for alias analyses.
static constexpr roundingMode rmTowardZero
static const fltSemantics & IEEEdouble()
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned getActiveBits() const
Compute the number of active bits in the value.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
void clearAnalysis(IRUnitT &IR)
Directly clear a cached analysis for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
ArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
LLVM_ABI BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
static LLVM_ABI BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
static uint32_t getDenominator()
uint32_t getNumerator() const
Represents analyses that only rely on functions' control flow.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
Conditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
This class represents a range of values.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
Analysis pass which computes a CycleInfo.
static DebugLoc getTemporary()
static DebugLoc getUnknown()
An analysis that produces DemandedBits for a function.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
void insert_range(Range &&R)
Inserts a range of 'std::pair<KeyT, ValueT>' values into the map.
ValueT lookup_or(const_arg_type_t< KeyT > Val, U &&Default) const
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
constexpr bool isPowerOf2() const
Tagged union holding either a T or a Error.
Convenience struct for specifying and reasoning about fast-math flags.
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Common base class shared among various IRBuilders.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
@ IK_PtrInduction
Pointer induction var. Step = C.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
iterator_range< user_iterator > users()
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
The group of interleaved loads/stores sharing the same stride and close to each other.
auto members() const
Return an iterator range over the non-null members of this group, in index order.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool hasGroups() const
Returns true if we have any interleave groups.
iterator_range< SmallPtrSetIterator< llvm::InterleaveGroup< Instruction > * > > getInterleaveGroups()
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
LLVM_ABI void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
const SymbolicStrideMap & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
unsigned getNumBlocks() const
Get the number of blocks in this loop in constant time.
BlockT * getHeader() const
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
LLVM_ABI void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
BlockFrequencyInfo * BFI
The BlockFrequencyInfo returned from GetBFI.
BlockFrequencyInfo & getBFI()
Returns the BlockFrequencyInfo for the function if cached, otherwise fetches it via GetBFI.
bool isForcedScalar(Instruction *I, ElementCount VF) const
Returns true if I has been forced to be scalarized at VF.
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
static constexpr StringLiteral getInstWideningStr(InstWidening W)
bool isMaskRequired(Instruction *I) const
Wrapper function for LoopVectorizationLegality::isMaskRequired, that passes the Instruction I and if ...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const TargetTransformInfo & TTI
Vector target information.
friend class LoopVectorizationPlanner
const Function * TheFunction
LoopVectorizationLegality * Legal
Vectorization legality.
uint64_t getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB)
A helper function that returns how much we should divide the cost of a predicated block by.
std::optional< InstWidening > memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
If I is a memory instruction with a consecutive pointer that can be widened, returns the widening kin...
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool maskPartialAliasing() const
Returns true if all loop blocks should have partial aliases masked.
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const
Returns true if the target machine supports gather or scatter for I's data type and alignment.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
Loop * TheLoop
The loop that we evaluate.
void tryToEnablePartialAliasMasking()
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, std::function< BlockFrequencyInfo &()> GetBFI, const Function *F, InterleavedAccessInfo &IAI, VFSelectionContext &Config)
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool usePredicatedReductionSelect(RecurKind RecurrenceKind, bool HasUsesOutsideReductionChain) const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const
Returns true if an artificially high cost for emulated masked memrefs should be used.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const
Returns true if the target machine supports masked loads or stores for I's data type and alignment.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
bool isEpilogueAllowed() const
Returns true if an epilogue is allowed (e.g., not prevented by optsize or a loop hint annotation).
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
@ CM_InvalidatedDecision
A widening decision that has been invalidated after replacing the corresponding recipe during VPlan t...
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF)
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool isScalarWithPredication(Instruction *I, ElementCount VF)
Returns true if I is an instruction which requires predication and for which our chosen predication s...
std::function< BlockFrequencyInfo &()> GetBFI
A function to lazily fetch BlockFrequencyInfo.
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost MaskedCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
TailFoldingStyle getTailFoldingStyle() const
Returns the TailFoldingStyle that is best for the current loop.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
RecurrenceSet & getFixedOrderRecurrences()
Return the fixed-order recurrences found in the loop.
LLVM_ABI bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool hasUncountableExitWithSideEffects() const
Returns true if this is an early exit loop with state-changing or potentially-faulting operations and...
LLVM_ABI bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
LLVM_ABI bool isFixedOrderRecurrence(const PHINode *Phi) const
Returns True if Phi is a fixed-order recurrence in this loop.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
bool hasConditionalInductions() const
const HistogramInfo * getHistogramInfo(Instruction *I) const
Returns a HistogramInfo* for the given instruction if it was determined to be part of a load -> updat...
const ReductionList & getReductionVars() const
Returns the reduction variables found in the loop.
bool isSafeForAnyVectorWidth() const
bool hasUncountableEarlyExit() const
Returns true if the loop has uncountable early exits, i.e.
LLVM_ABI bool isUniformMemOp(Instruction &I, std::optional< ElementCount > VF) const
A uniform memory op is a load or store which accesses the same memory location on all VF lanes,...
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
LLVM_ABI bool isUniform(Value *V, std::optional< ElementCount > VF) const
Returns true if value V is uniform across VF lanes, when VF is provided, and otherwise if V is invari...
Planner drives the vectorization process after having passed Legality checks.
EpilogueVectorizationKind
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
@ None
Not part of epilogue vectorization.
@ Epilogue
Vectorizing the epilogue loop.
@ MainLoop
Vectorizing the main loop of epilogue vectorization.
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool TailFoldedEpilogue) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
void clearCostModel()
Destroy the cost model.
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll, bool UnrollVectorizedLoop)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
bool hasAPlan() const
Returns true if there is at least one VPlan.
LoopVectorizationCostModel & getCostModel()
Return the cost model. Must not be called after clearCostModel().
LoopVectorizationPlanner(Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal, std::unique_ptr< LoopVectorizationCostModel > CM, VFSelectionContext &Config, PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE, std::function< const BranchProbabilityInfo &()> GetBPI)
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks, ElementCount VF, unsigned UF, bool HasBranchWeights) const
Attach the runtime checks of RTChecks to Plan.
bool hasVectorPlan() const
Return true if there is at least one VPlan for a vector VF.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
~LoopVectorizationPlanner()
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, bool HasRuntimeChecks, EpilogueVectorizationKind EpilogueVecKind=EpilogueVectorizationKind::None, MutableArrayRef< VPBasicBlock * > TrackedBlocks={})
void printPlans(raw_ostream &O)
std::unique_ptr< VPlan > selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC, bool ScalarEpilogueAllowed)
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
std::pair< VectorizationFactor, VPlan * > computeBestVF()
Compute and return the most profitable vectorization factor and the corresponding best VPlan.
This holds vectorization requirements that must be verified late in the process.
Instruction * getExactFPInst()
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
enum ForceKind getForce() const
LLVM_ABI void emitRemarkWithHints() const
Dumps all the hint information.
LLVM_ABI bool allowVectorization(Loop *L, bool VectorizeOnlyWhenForced) const
bool isPotentiallyUnsafe() const
ElementCount getWidth() const
@ FK_Enabled
Forcing enabled.
@ FK_Undefined
Not selected.
@ FK_Disabled
Forcing disabled.
unsigned getPredicate() const
unsigned getInterleave() const
Represents a single loop in the control flow graph.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
unsigned getOpcode() const
Type * getRecurrenceType() const
Returns the type of the recurrence.
static bool isFindLastRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static LLVM_ABI bool isSubRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is for a sub operation.
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
bool Need
This flag indicates if we need to add the runtime check.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
LLVM_ABI void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEVFlags Flags=SCEV::FlagNone)
LLVM_ABI const SCEV * getURemExpr(SCEVUse LHS, SCEVUse RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI SCEVUse getAddExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI SCEVUse getMulExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
void insert_range(Range &&R)
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
bool contains(const_arg_type key) const
Check if the SetVector contains the given key.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
A Use represents the edge between a Value definition and its users.
iterator_range< op_iterator > op_range
Value * getOperand(unsigned i) const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
Holds state needed to make cost decisions before computing costs per-VF, including the maximum VFs.
const TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
const LoopVectorizeHints & getHints() const
bool isEpilogueVectorizationProfitable(ElementCount VF, unsigned IC) const
Returns true if epilogue vectorization is considered profitable for a main loop with vectorization fa...
std::optional< unsigned > getVScaleForTuning() const
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy::iterator iterator
Instruction iterators...
iterator begin()
Recipe iterator methods.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override
Return the cost of this VPBasicBlock.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
const VPRecipeBase & front() const
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPBasicBlock * getExitingBasicBlock() const
void setName(const Twine &newName)
size_t getNumPredecessors() const
const VPBlocksTy & getPredecessors() const
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
static auto blocksAs(T &&Range)
Return an iterator range over Range with each block cast to BlockTy.
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
static auto blocksOnly(T &&Range)
Return an iterator range over Range which only includes BlockTy blocks.
static std::pair< VPBasicBlock *, VPBasicBlock * > getPlainCFGHeaderAndLatch(const VPlan &Plan)
Returns the header and latch of the outermost loop of Plan in plain CFG form (before regions are form...
A recipe for generating conditional branches on the bits of a mask.
static VPBuilderBase getToInsertAfter(VPRecipeBase *R)
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", std::optional< VPIRFlags > Flags=std::nullopt, Type *ResultTy=nullptr)
Create a phi with IncomingValues, using the default flags for the result type, unless Flags is set.
T * insert(T *R)
Insert R at the current insertion point. Returns R unchanged.
VPInstruction * createAdd(VPValue *LHS, VPValue *RHS, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", VPRecipeWithIRFlags::WrapFlagsTy WrapFlags={false, false})
VPInstruction * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", std::optional< VPIRFlags > Flags=std::nullopt)
Create a select of TrueVal and FalseVal based on Cond, using the default flags for the result type,...
static VPSingleDefRecipe * createSingleScalarOp(unsigned Opcode, ArrayRef< VPValue * > Operands, VPValue *Mask, const VPIRFlags &Flags, const VPIRMetadata &Metadata, DebugLoc DL, Type *ResultTy, Instruction *UV)
void setInsertPoint(const VPInsertPoint &IP)
Set the current insert point.
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", Type *ResultTy=nullptr, const VPIRAttributes &Attrs={})
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
Recipe to expand a SCEV expression.
A recipe representing a sequence of load -> update -> store as part of a histogram operation.
A special type of VPBasicBlock that wraps an existing IR basic block.
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlagsOrNone() const
This is a concrete Recipe that models a single VPlan-level instruction.
iterator_range< operand_iterator > operandsWithoutMask()
Returns an iterator range over the operands excluding the mask operand if present.
VPInstruction * clone() override
Clone the current recipe.
@ ResumeForEpilogue
Explicit user for values in the main VPlan, used by the epilogue vector loop.
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
@ ComputeReductionResult
Reduce the operands to the final reduction result using the operation specified via the operation's V...
unsigned getOpcode() const
VPValue * getMask() const
Returns the mask for the VPInstruction.
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
RecipeListTy::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
VPRecipeBase * tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for a non-phi recipe R if one can be created within the given VF R...
VPHistogramRecipe * widenIfHistogram(VPInstruction *VPI)
If VPI represents a histogram operation (as determined by LoopVectorizationLegality) make that safe f...
std::optional< CompressedPtrInfo > getCompressedPtrInfo(VPInstruction *VPI)
Returns the CompressedPtrInfo for VPI if it is a compressed load or store (as determined by LoopVecto...
bool prefersVectorizedAddressing() const
Returns true if the target prefers vectorized addressing.
VPRecipeBase * tryToWidenMemory(VPInstruction *VPI, VFRange &Range)
Check if the load or store instruction VPI should widened for Range.Start and potentially masked.
bool replaceWithFinalIfReductionStore(VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder)
If VPI is a store of a reduction into an invariant address, delete it.
VPSingleDefRecipe * handleReplication(VPInstruction *VPI, VFRange &Range)
Build a replicating or single-scalar recipe for VPI.
VPWidenMemIntrinsicRecipe * widenCompressedLoadOrStore(VPInstruction *VPI)
Build a llvm.masked.expandload or llvm.masked.compressstore intrinsic for VPI.
bool isPredicatedInst(Instruction *I) const
Returns true if I needs to be predicated (i.e.
Type * getScalarType() const
Returns the scalar type of this VPRecipeValue.
A recipe for handling reduction phis.
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
bool hasUsesOutsideReductionChain() const
Returns true, if the phi is part of a multi-use reduction.
bool isInLoop() const
Returns true if the phi is part of an in-loop reduction.
VPReductionPHIRecipe * cloneWithOperands(VPValue *Start, VPValue *BackedgeValue)
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
A recipe to represent inloop, ordered or partial reduction operations.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void clearCanonicalIVNUW(VPInstruction *Increment)
Unsets NUW for the canonical IV increment Increment, for loop regions.
VPRegionValue * getCanonicalIV()
Return the canonical induction variable of the region, null for replicating regions.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPSingleDefRecipe is a base class for recipes that model a sequence of one or more output IR that def...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
VPValue * getOperand(unsigned N) const
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
Type * getScalarType() const
Returns the scalar type of this VPValue, dispatching based on the concrete subclass.
Value * getLiveInIRValue() const
Return the underlying IR value for a VPIRValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser *U)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
void replaceAllUsesWith(VPValue *New)
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
A recipe for widening vector memory intrinsics.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
bool hasVF(ElementCount VF) const
ElementCount getSingleVF() const
Returns the single VF of the plan, asserting that the plan has exactly one VF.
VPBasicBlock * getEntry()
VPValue * getTripCount() const
The trip count of the original loop.
VPSymbolicValue & getVFxUF()
Returns VF * UF of the vector loop region.
bool hasUF(unsigned UF) const
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
VPIRValue * getOrAddLiveIn(Value *V)
Gets the live-in VPIRValue for V or adds a new live-in (if none exists yet) for V.
VPIRValue * getZero(Type *Ty)
Return a VPIRValue wrapping the null value of type Ty.
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
LLVM_ABI_FOR_TEST bool isOuterLoop() const
Returns true if this VPlan is for an outer loop, i.e., its vector loop region contains a nested loop ...
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
VPBasicBlock * getVectorPreheader() const
Returns the preheader of the vector loop region, if one exists, or null otherwise.
bool requiresScalarEpilogue() const
Returns true if the plan requires a scalar epilogue after the vector loop.
VPSymbolicValue & getUF()
Returns the UF of the vector loop region.
bool hasScalarVFOnly() const
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
bool hasTailFolded() const
Returns true if the vector loop region is tail-folded.
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
LLVM_ABI_FOR_TEST VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isNonZero() const
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ BasicBlock
Various leaf nodes.
void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, const Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, const Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, ElementCount VFWidth, unsigned IC)
Report successful vectorization of the loop.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
match_unless< Pattern > m_Unless(const Pattern &P)
Match if the inner matcher does NOT match.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
match_bind< const SCEVMulExpr > m_scev_Mul(const SCEVMulExpr *&V)
bool match(const SCEV *S, const Pattern &P)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagNone, true > m_scev_c_Mul(const Op0_t &Op0, const Op1_t &Op1)
RecipeTy * findSingleUser(VPValue *V, const MatchT &P)
Return the sole user of V matching pattern P, casting it to RecipeTy, or nullptr.
bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start)
Match FindIV result pattern: select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),...
VPInstruction_match< VPInstruction::ExtractLastLane, Op0_t > m_ExtractLastLane(const Op0_t &Op0)
VPInstruction_match< VPInstruction::BranchOnCount > m_BranchOnCount()
auto m_VPValue()
Match an arbitrary VPValue and ignore it.
VPInstruction_match< VPInstruction::ExtractLastPart, Op0_t > m_ExtractLastPart(const Op0_t &Op0)
bool match(Val *V, const Pattern &P)
VPInstruction_match< VPInstruction::ExtractLane, Op0_t, Op1_t > m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1)
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
BranchProbability getExecutionProbability(BlockFrequency Freq)
Returns Freq as a BranchProbability, relative to the full mass.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
bool isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE, const Loop *L)
Returns true if Addr is an address SCEV that can be passed to TTI::getAddressComputationCost,...
VPInstruction * findCanonicalIVIncrement(VPlan &Plan)
Find the canonical IV increment of Plan's vector loop region.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used by all users.
VPValue * findIncomingAliasMask(const VPlan &Plan)
Finds the incoming alias-mask within the vector preheader.
Type * getLoadStoreValueType(VPSingleDefRecipe *R)
Get the value type of a VPInstruction or VPReplicateRecipe load or store.
bool doesGeneratePerAllLanes(const VPRecipeBase *R)
Returns true if R produces scalar values for all VF lanes.
VPRecipeBase * findRecipe(VPValue *Start, PredT Pred)
Search Start's users for a recipe satisfying Pred, looking through recipes with definitions.
LLVM_ABI_FOR_TEST const SCEV * getSCEVExprForVPValue(const VPValue *V, PredicatedScalarEvolution &PSE, const Loop *L=nullptr)
Return the SCEV expression for V.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
SmallVector< VPBasicBlock * > vp_rpo_plain_cfg_loop_body(VPBasicBlock *Header)
Returns the VPBasicBlocks forming the loop body of a plain (pre-region) VPlan in reverse post-order s...
constexpr auto not_equal_to(T &&Arg)
Functor variant of std::not_equal_to that can be used as a UnaryPredicate in functional algorithms li...
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
VPBuilderBase<> VPBuilder
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
auto make_isa_range(RangeT &&Range)
Return a range over Range containing only elements for which isa<T> holds, casting each of them to T.
auto dyn_cast_or_null(const Y &Val)
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA, const TargetTransformInfo *TTI=nullptr)
Simplify each loop in a loop nest recursively.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
std::optional< uint64_t > getMaxRuntimeElementCount(ElementCount EC, const Function &F)
void sort(IteratorTy Start, IteratorTy End)
bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
UncountableExitStyle
Different methods of handling early exits.
@ ReadOnly
No side effects to worry about, so we can process any uncountable exits in the loop and branch either...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
@ CM_EpilogueNotAllowedLowTripLoop
@ CM_EpilogueNotNeededFoldTail
@ CM_EpilogueNotAllowedFoldTail
@ CM_EpilogueNotAllowedOptSize
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
void replace(R &&Range, const T &OldValue, const T &NewValue)
Provide wrappers to std::replace which take ranges instead of having to pass begin/end explicitly.
RecurKind
These are the kinds of recurrences that we support.
@ Sub
Subtraction of integers.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
constexpr T AbsoluteDifference(U X, V Y)
Subtract two unsigned integers, X and Y, of type T and return the absolute value of the result.
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool proveOuterLoopMemorySafety(VPlan &Plan, PredicatedScalarEvolution &PSE, AAResults &AA, const VPDominatorTree &VPDT, const Loop *OuterLoop)
Returns true if the memory accesses in outer loop plain CFG Plan can be vectorized.
auto sum_of(R &&Range, E Init=E{0})
Returns the sum of all values in Range with Init initial value.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
const VectorizeOptions & getVectorizeOptions(const LLVMContext &C)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
constexpr bool valueOr(BoolOrDefault X, bool Default)
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
bool pred_empty(const BasicBlock *BB)
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
LLVM_ABI std::optional< int64_t > getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy, Value *Ptr, PredicatedScalarEvolution &PSE)
If AR is an affine AddRec for Lp with a constant step, return the step in units of AccessTy's allocat...
@ Increment
Incrementally increasing token ID.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
@ Disabled
Don't do any conversion of .debug_str_offsets tables.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, ElementCount VF, unsigned IC)
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Implement std::hash so that hash_code can be used in STL containers.
ResumeForEpilogue markers in the main plan, used by the epilogue plan.
SmallVector< VPInstruction * > ResumeValues
This struct is a compact representation of a valid (non-zero power of two) alignment.
A special type used by analysis passes to provide an address that identifies that particular analysis...
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
HistogramUpdateKind UpdateKind
The update operation applied to the histogram buckets.
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
LoopAccessInfoManager * LAIs
std::function< const BranchProbabilityInfo &()> GetBPI
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
std::function< BlockFrequencyInfo &()> GetBFI
TargetTransformInfo * TTI
Storage for information about made changes.
Holds the VFShape for a specific scalar to vector function mapping.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Struct to hold various analysis needed for cost computations.
const VFSelectionContext & Config
LoopVectorizationCostModel & CM
VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan, LoopVectorizationCostModel &CM, VFSelectionContext &Config, bool ReusePrintingSlotTracker=false)
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
bool isMaskRequired(Instruction *I) const
Forwards to LoopVectorizationCostModel::isMaskRequired.
void invalidateWideningDecision(Instruction *I, ElementCount VF)
Mark the widening decision for I at VF as invalidated since a VPlan transform replaced the original r...
PredicatedScalarEvolution & PSE
bool willBeScalarized(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalarized at VF.
static bool executesAtMostOnce(const VPlan &Plan, ElementCount VF)
Returns true if the vector loop body of Plan is known to execute at most once at VF,...
TargetTransformInfo::TargetCostKind CostKind
const TargetLibraryInfo & TLI
const TargetTransformInfo & TTI
SmallPtrSet< Instruction *, 8 > SkipCostComputation
const VectorizeOptions & Opts
A pure-virtual common base class for recipes defining a single VPValue and using IR flags.
A struct that represents some properties of the register usage of a loop.
InstructionCost spillCost(const TargetTransformInfo &TTI, TargetTransformInfo::TargetCostKind CostKind, unsigned OverrideMaxNumRegs=0) const
Calculate the estimated cost of any spills due to using more registers than the number available for ...
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
A recipe for widening load operations, using the address to load from and an optional mask.
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
A recipe for widening store operations, using the stored value, the address to store to and an option...
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI unsigned VectorizeMemoryCheckThreshold
The maximum allowed number of runtime memory checks.
static LLVM_ABI bool HoistRuntimeChecks