161#define LV_NAME "loop-vectorize"
162#define DEBUG_TYPE LV_NAME
168STATISTIC(LoopsVectorized,
"Number of loops vectorized");
169STATISTIC(LoopsAnalyzed,
"Number of loops analyzed for vectorization");
170STATISTIC(LoopsEpilogueVectorized,
"Number of epilogues vectorized");
171STATISTIC(LoopsEarlyExitVectorized,
"Number of early exit loops vectorized");
175 cl::desc(
"Enable vectorization of epilogue loops."));
179 cl::desc(
"When epilogue vectorization is enabled, and a value greater than "
180 "1 is specified, forces the given VF for all applicable epilogue "
184 "epilogue-vectorization-minimum-VF",
cl::Hidden,
185 cl::desc(
"Only loops with vectorization factor equal to or larger than "
186 "the specified value are considered for epilogue vectorization."));
192 cl::desc(
"Loops with a constant trip count that is smaller than this "
193 "value are vectorized only if no scalar iteration overheads "
198 cl::desc(
"The maximum allowed number of runtime memory checks"));
214 "prefer-predicate-over-epilogue",
217 cl::desc(
"Tail-folding and predication preferences over creating a scalar "
221 "Don't tail-predicate loops, create scalar epilogue"),
223 "predicate-else-scalar-epilogue",
224 "prefer tail-folding, create scalar epilogue if tail "
227 "predicate-dont-vectorize",
228 "prefers tail-folding, don't attempt vectorization if "
229 "tail-folding fails.")));
232 "force-tail-folding-style",
cl::desc(
"Force the tail folding style"),
238 "Create lane mask for data only, using active.lane.mask intrinsic"),
240 "data-without-lane-mask",
241 "Create lane mask with compare/stepvector"),
243 "Create lane mask using active.lane.mask intrinsic, and use "
244 "it for both data and control flow"),
246 "data-and-control-without-rt-check",
247 "Similar to data-and-control, but remove the runtime check"),
249 "Use predicated EVL instructions for tail folding. If EVL "
250 "is unsupported, fallback to data-without-lane-mask.")));
254 cl::desc(
"Maximize bandwidth when selecting vectorization factor which "
255 "will be determined by the smallest type in loop."));
259 cl::desc(
"Enable vectorization on interleaved memory accesses in a loop"));
265 cl::desc(
"Enable vectorization on masked interleaved memory accesses in a loop"));
269 cl::desc(
"A flag that overrides the target's number of scalar registers."));
273 cl::desc(
"A flag that overrides the target's number of vector registers."));
277 cl::desc(
"A flag that overrides the target's max interleave factor for "
282 cl::desc(
"A flag that overrides the target's max interleave factor for "
283 "vectorized loops."));
287 cl::desc(
"A flag that overrides the target's expected cost for "
288 "an instruction to a single constant value. Mostly "
289 "useful for getting consistent testing."));
294 "Pretend that scalable vectors are supported, even if the target does "
295 "not support them. This flag should only be used for testing."));
300 "The cost of a loop that is considered 'small' by the interleaver."));
304 cl::desc(
"Enable the use of the block frequency analysis to access PGO "
305 "heuristics minimizing code growth in cold regions and being more "
306 "aggressive in hot regions."));
312 "Enable runtime interleaving until load/store ports are saturated"));
317 cl::desc(
"Max number of stores to be predicated behind an if."));
321 cl::desc(
"Count the induction variable only once when interleaving"));
325 cl::desc(
"Enable if predication of stores during vectorization."));
329 cl::desc(
"The maximum interleave count to use when interleaving a scalar "
330 "reduction in a nested loop."));
335 cl::desc(
"Prefer in-loop vector reductions, "
336 "overriding the targets preference."));
340 cl::desc(
"Enable the vectorisation of loops with in-order (strict) "
346 "Prefer predicating a reduction operation over an after loop select."));
350 cl::desc(
"Enable VPlan-native vectorization path with "
351 "support for outer loop vectorization."));
355#ifdef EXPENSIVE_CHECKS
361 cl::desc(
"Verfiy VPlans after VPlan transforms."));
370 "Build VPlan for every supported loop nest in the function and bail "
371 "out right after the build (stress test the VPlan H-CFG construction "
372 "in the VPlan-native vectorization path)."));
376 cl::desc(
"Enable loop interleaving in Loop vectorization passes"));
379 cl::desc(
"Run the Loop vectorization passes"));
382 "force-widen-divrem-via-safe-divisor",
cl::Hidden,
384 "Override cost based safe divisor widening for div/rem instructions"));
387 "vectorizer-maximize-bandwidth-for-vector-calls",
cl::init(
true),
389 cl::desc(
"Try wider VFs if they enable the use of vector variants"));
394 "Enable vectorization of early exit loops with uncountable exits."));
398 cl::desc(
"Discard VFs if their register pressure is too high."));
411 return DL.getTypeAllocSizeInBits(Ty) !=
DL.getTypeSizeInBits(Ty);
446static std::optional<ElementCount>
448 bool CanUseConstantMax =
true) {
458 if (!CanUseConstantMax)
470class GeneratedRTChecks;
503 VF(VecWidth),
UF(UnrollFactor),
Builder(
PSE.getSE()->getContext()),
506 Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
616 "A high UF for the epilogue loop is likely not beneficial.");
667 EPI.MainLoopVF,
EPI.MainLoopUF) {}
704 EPI.EpilogueVF,
EPI.EpilogueUF) {}
721 if (
I->getDebugLoc() !=
Empty)
722 return I->getDebugLoc();
725 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
726 if (OpInst->getDebugLoc() != Empty)
727 return OpInst->getDebugLoc();
730 return I->getDebugLoc();
739 dbgs() <<
"LV: " << Prefix << DebugMsg;
755static OptimizationRemarkAnalysis
761 if (
I &&
I->getDebugLoc())
762 DL =
I->getDebugLoc();
766 return OptimizationRemarkAnalysis(
PassName, RemarkName,
DL, CodeRegion);
774 assert(Ty->isIntegerTy() &&
"Expected an integer step");
782 return B.CreateElementCount(Ty, VFxStep);
787 return B.CreateElementCount(Ty, VF);
798 <<
"loop not vectorized: " << OREMsg);
821 "Vectorizing: ", TheLoop->
isInnermost() ?
"innermost loop" :
"outer loop",
827 <<
"vectorized " << LoopType <<
"loop (vectorization width: "
829 <<
", interleaved count: " <<
ore::NV(
"InterleaveCount", IC) <<
")";
885 initializeVScaleForTuning();
900 bool runtimeChecksRequired();
919 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
938 void collectValuesToIgnore();
941 void collectElementTypesForWidening();
945 void collectInLoopReductions();
966 "Profitable to scalarize relevant only for VF > 1.");
969 "cost-model should not be used for outer loops (in VPlan-native path)");
971 auto Scalars = InstsToScalarize.find(VF);
972 assert(Scalars != InstsToScalarize.end() &&
973 "VF not yet analyzed for scalarization profitability");
974 return Scalars->second.contains(
I);
981 "cost-model should not be used for outer loops (in VPlan-native path)");
991 auto UniformsPerVF = Uniforms.find(VF);
992 assert(UniformsPerVF != Uniforms.end() &&
993 "VF not yet analyzed for uniformity");
994 return UniformsPerVF->second.count(
I);
1001 "cost-model should not be used for outer loops (in VPlan-native path)");
1005 auto ScalarsPerVF = Scalars.find(VF);
1006 assert(ScalarsPerVF != Scalars.end() &&
1007 "Scalar values are not calculated for VF");
1008 return ScalarsPerVF->second.count(
I);
1016 I->getType()->getScalarSizeInBits() < MinBWs.lookup(
I))
1018 return VF.
isVector() && MinBWs.contains(
I) &&
1040 WideningDecisions[{
I, VF}] = {W,
Cost};
1059 for (
unsigned Idx = 0; Idx < Grp->
getFactor(); ++Idx) {
1062 WideningDecisions[{
I, VF}] = {W, InsertPosCost};
1064 WideningDecisions[{
I, VF}] = {W, OtherMemberCost};
1076 "cost-model should not be used for outer loops (in VPlan-native path)");
1078 std::pair<Instruction *, ElementCount> InstOnVF(
I, VF);
1079 auto Itr = WideningDecisions.find(InstOnVF);
1080 if (Itr == WideningDecisions.end())
1082 return Itr->second.first;
1089 std::pair<Instruction *, ElementCount> InstOnVF(
I, VF);
1090 assert(WideningDecisions.contains(InstOnVF) &&
1091 "The cost is not calculated");
1092 return WideningDecisions[InstOnVF].second;
1105 std::optional<unsigned> MaskPos,
1108 CallWideningDecisions[{CI, VF}] = {Kind, Variant, IID, MaskPos,
Cost};
1114 auto I = CallWideningDecisions.find({CI, VF});
1115 if (
I == CallWideningDecisions.end())
1138 Value *
Op = Trunc->getOperand(0);
1139 if (
Op !=
Legal->getPrimaryInduction() &&
TTI.isTruncateFree(SrcTy, DestTy))
1143 return Legal->isInductionPhi(
Op);
1159 if (VF.
isScalar() || Uniforms.contains(VF))
1162 collectLoopUniforms(VF);
1164 collectLoopScalars(VF);
1172 return Legal->isConsecutivePtr(DataType,
Ptr) &&
1180 return Legal->isConsecutivePtr(DataType,
Ptr) &&
1195 return (
LI &&
TTI.isLegalMaskedGather(Ty,
Align)) ||
1202 return (
all_of(
Legal->getReductionVars(), [&](
auto &Reduction) ->
bool {
1203 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1204 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1215 return ScalarCost < SafeDivisorCost;
1239 std::pair<InstructionCost, InstructionCost>
1267 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
1274 LLVM_DEBUG(
dbgs() <<
"LV: Loop requires scalar epilogue: not exiting "
1275 "from latch block\n");
1280 "interleaved group requires scalar epilogue\n");
1283 LLVM_DEBUG(
dbgs() <<
"LV: Loop does not require scalar epilogue\n");
1295 if (!ChosenTailFoldingStyle)
1297 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1298 : ChosenTailFoldingStyle->second;
1306 assert(!ChosenTailFoldingStyle &&
"Tail folding must not be selected yet.");
1307 if (!
Legal->canFoldTailByMasking()) {
1313 ChosenTailFoldingStyle = {
1314 TTI.getPreferredTailFoldingStyle(
true),
1315 TTI.getPreferredTailFoldingStyle(
false)};
1325 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1339 dbgs() <<
"LV: Preference for VP intrinsics indicated. Will "
1340 "not try to generate VP Intrinsics "
1342 ?
"since interleave count specified is greater than 1.\n"
1343 :
"due to non-interleaving reasons.\n"));
1377 return InLoopReductions.contains(Phi);
1388 TTI.preferPredicatedReductionSelect();
1403 WideningDecisions.clear();
1404 CallWideningDecisions.clear();
1422 bool isEpilogueVectorizationProfitable(
const ElementCount VF,
1423 const unsigned IC)
const;
1431 std::optional<InstructionCost> getReductionPatternCost(
Instruction *
I,
1433 Type *VectorTy)
const;
1437 bool shouldConsiderInvariant(
Value *
Op);
1443 unsigned NumPredStores = 0;
1447 std::optional<unsigned> VScaleForTuning;
1452 void initializeVScaleForTuning() {
1457 auto Max = Attr.getVScaleRangeMax();
1458 if (Max && Min == Max) {
1459 VScaleForTuning = Max;
1472 FixedScalableVFPair computeFeasibleMaxVF(
unsigned MaxTripCount,
1473 ElementCount UserVF,
1474 bool FoldTailByMasking);
1478 ElementCount clampVFByMaxTripCount(ElementCount VF,
unsigned MaxTripCount,
1479 bool FoldTailByMasking)
const;
1484 ElementCount getMaximizedVFForTarget(
unsigned MaxTripCount,
1485 unsigned SmallestType,
1486 unsigned WidestType,
1487 ElementCount MaxSafeVF,
1488 bool FoldTailByMasking);
1492 bool isScalableVectorizationAllowed();
1496 ElementCount getMaxLegalScalableVF(
unsigned MaxSafeElements);
1502 InstructionCost getMemInstScalarizationCost(Instruction *
I, ElementCount VF);
1523 ElementCount VF)
const;
1527 bool useEmulatedMaskMemRefHack(Instruction *
I, ElementCount VF);
1532 MapVector<Instruction *, uint64_t> MinBWs;
1537 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1541 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1542 PredicatedBBsAfterVectorization;
1555 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1556 ChosenTailFoldingStyle;
1559 std::optional<bool> IsScalableVectorizationAllowed;
1565 std::optional<unsigned> MaxSafeElements;
1571 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1575 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1579 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1583 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1586 SmallPtrSet<PHINode *, 4> InLoopReductions;
1591 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1599 ScalarCostsTy &ScalarCosts,
1611 void collectLoopUniforms(ElementCount VF);
1620 void collectLoopScalars(ElementCount VF);
1624 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1625 std::pair<InstWidening, InstructionCost>>;
1627 DecisionList WideningDecisions;
1629 using CallDecisionList =
1630 DenseMap<std::pair<CallInst *, ElementCount>, CallWideningDecision>;
1632 CallDecisionList CallWideningDecisions;
1636 bool needsExtract(
Value *V, ElementCount VF)
const {
1640 getWideningDecision(
I, VF) == CM_Scalarize ||
1651 return !Scalars.
contains(VF) || !isScalarAfterVectorization(
I, VF);
1656 ElementCount VF)
const {
1658 SmallPtrSet<const Value *, 4> UniqueOperands;
1662 !needsExtract(
Op, VF))
1734class GeneratedRTChecks {
1740 Value *SCEVCheckCond =
nullptr;
1747 Value *MemRuntimeCheckCond =
nullptr;
1756 bool CostTooHigh =
false;
1758 Loop *OuterLoop =
nullptr;
1769 : DT(DT), LI(LI),
TTI(
TTI),
1770 SCEVExp(*PSE.
getSE(),
DL,
"scev.check",
false),
1771 MemCheckExp(*PSE.
getSE(),
DL,
"scev.check",
false),
1779 void create(Loop *L,
const LoopAccessInfo &LAI,
1780 const SCEVPredicate &UnionPred, ElementCount VF,
unsigned IC) {
1800 nullptr,
"vector.scevcheck");
1807 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1808 SCEVCleaner.cleanup();
1813 if (RtPtrChecking.Need) {
1814 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1815 MemCheckBlock =
SplitBlock(Pred, Pred->getTerminator(), DT, LI,
nullptr,
1818 auto DiffChecks = RtPtrChecking.getDiffChecks();
1820 Value *RuntimeVF =
nullptr;
1823 [VF, &RuntimeVF](IRBuilderBase &
B,
unsigned Bits) {
1825 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1831 MemCheckBlock->
getTerminator(), L, RtPtrChecking.getChecks(),
1834 assert(MemRuntimeCheckCond &&
1835 "no RT checks generated although RtPtrChecking "
1836 "claimed checks are required");
1841 if (!MemCheckBlock && !SCEVCheckBlock)
1851 if (SCEVCheckBlock) {
1854 auto *UI =
new UnreachableInst(Preheader->
getContext(), SCEVCheckBlock);
1858 if (MemCheckBlock) {
1861 auto *UI =
new UnreachableInst(Preheader->
getContext(), MemCheckBlock);
1867 if (MemCheckBlock) {
1871 if (SCEVCheckBlock) {
1877 OuterLoop =
L->getParentLoop();
1881 if (SCEVCheckBlock || MemCheckBlock)
1893 for (Instruction &
I : *SCEVCheckBlock) {
1894 if (SCEVCheckBlock->getTerminator() == &
I)
1900 if (MemCheckBlock) {
1902 for (Instruction &
I : *MemCheckBlock) {
1903 if (MemCheckBlock->getTerminator() == &
I)
1915 ScalarEvolution *SE = MemCheckExp.
getSE();
1920 const SCEV *
Cond = SE->
getSCEV(MemRuntimeCheckCond);
1925 unsigned BestTripCount = 2;
1929 PSE, OuterLoop,
false))
1930 if (EstimatedTC->isFixed())
1931 BestTripCount = EstimatedTC->getFixedValue();
1936 NewMemCheckCost = std::max(NewMemCheckCost.
getValue(),
1937 (InstructionCost::CostType)1);
1939 if (BestTripCount > 1)
1941 <<
"We expect runtime memory checks to be hoisted "
1942 <<
"out of the outer loop. Cost reduced from "
1943 << MemCheckCost <<
" to " << NewMemCheckCost <<
'\n');
1945 MemCheckCost = NewMemCheckCost;
1949 RTCheckCost += MemCheckCost;
1952 if (SCEVCheckBlock || MemCheckBlock)
1953 LLVM_DEBUG(
dbgs() <<
"Total cost of runtime checks: " << RTCheckCost
1961 ~GeneratedRTChecks() {
1962 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1963 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1964 bool SCEVChecksUsed = !SCEVCheckBlock || !
pred_empty(SCEVCheckBlock);
1965 bool MemChecksUsed = !MemCheckBlock || !
pred_empty(MemCheckBlock);
1967 SCEVCleaner.markResultUsed();
1969 if (MemChecksUsed) {
1970 MemCheckCleaner.markResultUsed();
1972 auto &SE = *MemCheckExp.
getSE();
1979 I.eraseFromParent();
1982 MemCheckCleaner.cleanup();
1983 SCEVCleaner.cleanup();
1985 if (!SCEVChecksUsed)
1986 SCEVCheckBlock->eraseFromParent();
1988 MemCheckBlock->eraseFromParent();
1993 std::pair<Value *, BasicBlock *> getSCEVChecks()
const {
1994 using namespace llvm::PatternMatch;
1996 return {
nullptr,
nullptr};
1998 return {SCEVCheckCond, SCEVCheckBlock};
2003 std::pair<Value *, BasicBlock *> getMemRuntimeChecks()
const {
2004 using namespace llvm::PatternMatch;
2005 if (MemRuntimeCheckCond &&
match(MemRuntimeCheckCond,
m_ZeroInt()))
2006 return {
nullptr,
nullptr};
2007 return {MemRuntimeCheckCond, MemCheckBlock};
2011 bool hasChecks()
const {
2012 return getSCEVChecks().first || getMemRuntimeChecks().first;
2055 LLVM_DEBUG(
dbgs() <<
"LV: Loop hints prevent outer loop vectorization.\n");
2061 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Interleave is not supported for "
2091 for (
Loop *InnerL : L)
2114 ?
B.CreateSExtOrTrunc(Index, StepTy)
2115 :
B.CreateCast(Instruction::SIToFP, Index, StepTy);
2116 if (CastedIndex != Index) {
2118 Index = CastedIndex;
2128 assert(
X->getType() ==
Y->getType() &&
"Types don't match!");
2133 return B.CreateAdd(
X,
Y);
2139 assert(
X->getType()->getScalarType() ==
Y->getType() &&
2140 "Types don't match!");
2147 Y =
B.CreateVectorSplat(XVTy->getElementCount(),
Y);
2148 return B.CreateMul(
X,
Y);
2151 switch (InductionKind) {
2154 "Vector indices not supported for integer inductions yet");
2156 "Index type does not match StartValue type");
2158 return B.CreateSub(StartValue, Index);
2163 return B.CreatePtrAdd(StartValue,
CreateMul(Index, Step));
2166 "Vector indices not supported for FP inductions yet");
2169 (InductionBinOp->
getOpcode() == Instruction::FAdd ||
2170 InductionBinOp->
getOpcode() == Instruction::FSub) &&
2171 "Original bin op should be defined for FP induction");
2173 Value *MulExp =
B.CreateFMul(Step, Index);
2174 return B.CreateBinOp(InductionBinOp->
getOpcode(), StartValue, MulExp,
2185 if (std::optional<unsigned> MaxVScale =
TTI.getMaxVScale())
2188 if (
F.hasFnAttribute(Attribute::VScaleRange))
2189 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2191 return std::nullopt;
2200 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2202 unsigned MaxUF = UF ? *UF : Cost->TTI.getMaxInterleaveFactor(VF);
2204 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
2210 if (
unsigned TC = Cost->PSE.getSmallConstantMaxTripCount()) {
2213 std::optional<unsigned> MaxVScale =
2217 MaxVF *= *MaxVScale;
2220 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2234 return TTI.enableMaskedInterleavedAccessVectorization();
2247 PreVectorPH = CheckVPIRBB;
2257 "must have incoming values for all operands");
2258 R.addOperand(R.getOperand(NumPredecessors - 2));
2284 auto CreateStep = [&]() ->
Value * {
2291 if (!
VF.isScalable())
2293 return Builder.CreateBinaryIntrinsic(
2299 Value *Step = CreateStep();
2308 CheckMinIters =
Builder.getTrue();
2310 TripCountSCEV, SE.
getSCEV(Step))) {
2313 CheckMinIters =
Builder.CreateICmp(
P,
Count, Step,
"min.iters.check");
2315 }
else if (
VF.isScalable() && !
TTI->isVScaleKnownToBeAPowerOfTwo() &&
2323 Value *MaxUIntTripCount =
2330 return CheckMinIters;
2339 VPlan *Plan =
nullptr) {
2343 auto IP = IRVPBB->
begin();
2345 R.moveBefore(*IRVPBB, IP);
2349 R.moveBefore(*IRVPBB, IRVPBB->
end());
2358 assert(VectorPH &&
"Invalid loop structure");
2360 Cost->requiresScalarEpilogue(
VF.isVector())) &&
2361 "loops not exiting via the latch without required epilogue?");
2368 Twine(Prefix) +
"scalar.ph");
2374 const SCEV2ValueTy &ExpandedSCEVs) {
2375 const SCEV *Step =
ID.getStep();
2377 return C->getValue();
2379 return U->getValue();
2380 Value *V = ExpandedSCEVs.lookup(Step);
2381 assert(V &&
"SCEV must be expanded at this point");
2391 auto *Cmp = L->getLatchCmpInst();
2393 InstsToIgnore.
insert(Cmp);
2394 for (
const auto &KV : IL) {
2403 [&](
const User *U) { return U == IV || U == Cmp; }))
2404 InstsToIgnore.
insert(IVInst);
2416struct CSEDenseMapInfo {
2427 return DenseMapInfo<Instruction *>::getTombstoneKey();
2430 static unsigned getHashValue(
const Instruction *
I) {
2431 assert(canHandle(
I) &&
"Unknown instruction!");
2436 static bool isEqual(
const Instruction *
LHS,
const Instruction *
RHS) {
2437 if (
LHS == getEmptyKey() ||
RHS == getEmptyKey() ||
2438 LHS == getTombstoneKey() ||
RHS == getTombstoneKey())
2440 return LHS->isIdenticalTo(
RHS);
2452 if (!CSEDenseMapInfo::canHandle(&In))
2458 In.replaceAllUsesWith(V);
2459 In.eraseFromParent();
2472 std::optional<unsigned> VScale) {
2476 EstimatedVF *= *VScale;
2477 assert(EstimatedVF >= 1 &&
"Estimated VF shouldn't be less than 1");
2495 for (
auto &ArgOp : CI->
args())
2506 return ScalarCallCost;
2519 assert(
ID &&
"Expected intrinsic call!");
2523 FMF = FPMO->getFastMathFlags();
2529 std::back_inserter(ParamTys),
2530 [&](
Type *Ty) { return maybeVectorizeType(Ty, VF); });
2535 return TTI.getIntrinsicInstrCost(CostAttrs,
CostKind);
2549 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2564 Builder.SetInsertPoint(NewPhi);
2566 NewPhi->
addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
2571void LoopVectorizationCostModel::collectLoopScalars(
ElementCount VF) {
2576 "This function should not be visited twice for the same VF");
2599 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
2600 assert(WideningDecision != CM_Unknown &&
2601 "Widening decision should be ready at this moment");
2603 if (
Ptr == Store->getValueOperand())
2604 return WideningDecision == CM_Scalarize;
2606 "Ptr is neither a value or pointer operand");
2607 return WideningDecision != CM_GatherScatter;
2612 auto IsLoopVaryingGEP = [&](
Value *
V) {
2623 if (!IsLoopVaryingGEP(
Ptr))
2635 if (IsScalarUse(MemAccess,
Ptr) &&
2639 PossibleNonScalarPtrs.
insert(
I);
2655 for (
auto *BB : TheLoop->
blocks())
2656 for (
auto &
I : *BB) {
2658 EvaluatePtrUse(Load,
Load->getPointerOperand());
2660 EvaluatePtrUse(Store,
Store->getPointerOperand());
2661 EvaluatePtrUse(Store,
Store->getValueOperand());
2664 for (
auto *
I : ScalarPtrs)
2665 if (!PossibleNonScalarPtrs.
count(
I)) {
2673 auto ForcedScalar = ForcedScalars.
find(VF);
2674 if (ForcedScalar != ForcedScalars.
end())
2675 for (
auto *
I : ForcedScalar->second) {
2676 LLVM_DEBUG(
dbgs() <<
"LV: Found (forced) scalar instruction: " << *
I <<
"\n");
2685 while (Idx != Worklist.
size()) {
2687 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
2691 auto *J = cast<Instruction>(U);
2692 return !TheLoop->contains(J) || Worklist.count(J) ||
2693 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
2694 IsScalarUse(J, Src));
2697 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Src <<
"\n");
2703 for (
const auto &Induction :
Legal->getInductionVars()) {
2704 auto *Ind = Induction.first;
2709 if (Ind ==
Legal->getPrimaryInduction() && foldTailByMasking())
2714 auto IsDirectLoadStoreFromPtrIndvar = [&](
Instruction *Indvar,
2716 return Induction.second.getKind() ==
2724 bool ScalarInd =
all_of(Ind->users(), [&](User *U) ->
bool {
2725 auto *I = cast<Instruction>(U);
2726 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2727 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2736 if (IndUpdatePhi &&
Legal->isFixedOrderRecurrence(IndUpdatePhi))
2741 bool ScalarIndUpdate =
all_of(IndUpdate->users(), [&](User *U) ->
bool {
2742 auto *I = cast<Instruction>(U);
2743 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
2744 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2746 if (!ScalarIndUpdate)
2751 Worklist.
insert(IndUpdate);
2752 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *Ind <<
"\n");
2753 LLVM_DEBUG(
dbgs() <<
"LV: Found scalar instruction: " << *IndUpdate
2767 switch(
I->getOpcode()) {
2770 case Instruction::Call:
2774 case Instruction::Load:
2775 case Instruction::Store: {
2784 TTI.isLegalMaskedGather(VTy, Alignment))
2786 TTI.isLegalMaskedScatter(VTy, Alignment));
2788 case Instruction::UDiv:
2789 case Instruction::SDiv:
2790 case Instruction::SRem:
2791 case Instruction::URem: {
2812 if (
Legal->blockNeedsPredication(
I->getParent()))
2824 switch(
I->getOpcode()) {
2827 "instruction should have been considered by earlier checks");
2828 case Instruction::Call:
2832 "should have returned earlier for calls not needing a mask");
2834 case Instruction::Load:
2837 case Instruction::Store: {
2845 case Instruction::UDiv:
2846 case Instruction::SDiv:
2847 case Instruction::SRem:
2848 case Instruction::URem:
2850 return !
Legal->isInvariant(
I->getOperand(1));
2854std::pair<InstructionCost, InstructionCost>
2857 assert(
I->getOpcode() == Instruction::UDiv ||
2858 I->getOpcode() == Instruction::SDiv ||
2859 I->getOpcode() == Instruction::SRem ||
2860 I->getOpcode() == Instruction::URem);
2869 ScalarizationCost = 0;
2875 ScalarizationCost +=
2879 ScalarizationCost +=
2881 TTI.getArithmeticInstrCost(
I->getOpcode(),
I->getType(),
CostKind);
2898 TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2903 SafeDivisorCost +=
TTI.getArithmeticInstrCost(
2905 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2906 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2908 return {ScalarizationCost, SafeDivisorCost};
2915 "Decision should not be set yet.");
2917 assert(Group &&
"Must have a group.");
2918 unsigned InterleaveFactor = Group->getFactor();
2922 auto &
DL =
I->getDataLayout();
2934 bool ScalarNI =
DL.isNonIntegralPointerType(ScalarTy);
2935 for (
unsigned Idx = 0; Idx < InterleaveFactor; Idx++) {
2940 bool MemberNI =
DL.isNonIntegralPointerType(MemberTy);
2942 if (MemberNI != ScalarNI)
2945 if (MemberNI && ScalarNI &&
2946 ScalarTy->getPointerAddressSpace() !=
2947 MemberTy->getPointerAddressSpace())
2956 bool PredicatedAccessRequiresMasking =
2958 Legal->isMaskRequired(
I);
2959 bool LoadAccessWithGapsRequiresEpilogMasking =
2962 bool StoreAccessWithGapsRequiresMasking =
2964 if (!PredicatedAccessRequiresMasking &&
2965 !LoadAccessWithGapsRequiresEpilogMasking &&
2966 !StoreAccessWithGapsRequiresMasking)
2973 "Masked interleave-groups for predicated accesses are not enabled.");
2975 if (Group->isReverse())
2979 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2980 StoreAccessWithGapsRequiresMasking;
2988 :
TTI.isLegalMaskedStore(Ty, Alignment, AS);
3000 if (!
Legal->isConsecutivePtr(ScalarTy,
Ptr))
3010 auto &
DL =
I->getDataLayout();
3017void LoopVectorizationCostModel::collectLoopUniforms(
ElementCount VF) {
3024 "This function should not be visited twice for the same VF");
3028 Uniforms[VF].
clear();
3036 auto IsOutOfScope = [&](
Value *V) ->
bool {
3048 auto AddToWorklistIfAllowed = [&](
Instruction *
I) ->
void {
3049 if (IsOutOfScope(
I)) {
3054 if (isPredicatedInst(
I)) {
3056 dbgs() <<
"LV: Found not uniform due to requiring predication: " << *
I
3060 LLVM_DEBUG(
dbgs() <<
"LV: Found uniform instruction: " << *
I <<
"\n");
3070 for (BasicBlock *
E : Exiting) {
3074 if (Cmp && TheLoop->
contains(Cmp) &&
Cmp->hasOneUse())
3075 AddToWorklistIfAllowed(Cmp);
3084 if (PrevVF.isVector()) {
3085 auto Iter = Uniforms.
find(PrevVF);
3086 if (Iter != Uniforms.
end() && !Iter->second.contains(
I))
3089 if (!
Legal->isUniformMemOp(*
I, VF))
3099 auto IsUniformDecision = [&](
Instruction *
I, ElementCount VF) {
3100 InstWidening WideningDecision = getWideningDecision(
I, VF);
3101 assert(WideningDecision != CM_Unknown &&
3102 "Widening decision should be ready at this moment");
3104 if (IsUniformMemOpUse(
I))
3107 return (WideningDecision == CM_Widen ||
3108 WideningDecision == CM_Widen_Reverse ||
3109 WideningDecision == CM_Interleave);
3119 (IsUniformDecision(
I, VF) ||
Legal->isInvariant(
Ptr));
3127 SetVector<Value *> HasUniformUse;
3131 for (
auto *BB : TheLoop->
blocks())
3132 for (
auto &
I : *BB) {
3134 switch (
II->getIntrinsicID()) {
3135 case Intrinsic::sideeffect:
3136 case Intrinsic::experimental_noalias_scope_decl:
3137 case Intrinsic::assume:
3138 case Intrinsic::lifetime_start:
3139 case Intrinsic::lifetime_end:
3141 AddToWorklistIfAllowed(&
I);
3149 if (IsOutOfScope(EVI->getAggregateOperand())) {
3150 AddToWorklistIfAllowed(EVI);
3156 "Expected aggregate value to be call return value");
3169 if (IsUniformMemOpUse(&
I))
3170 AddToWorklistIfAllowed(&
I);
3172 if (IsVectorizedMemAccessUse(&
I,
Ptr))
3179 for (
auto *V : HasUniformUse) {
3180 if (IsOutOfScope(V))
3183 bool UsersAreMemAccesses =
all_of(
I->users(), [&](User *U) ->
bool {
3184 auto *UI = cast<Instruction>(U);
3185 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
3187 if (UsersAreMemAccesses)
3188 AddToWorklistIfAllowed(
I);
3195 while (Idx != Worklist.
size()) {
3198 for (
auto *OV :
I->operand_values()) {
3200 if (IsOutOfScope(OV))
3205 if (
OP &&
Legal->isFixedOrderRecurrence(
OP))
3211 auto *J = cast<Instruction>(U);
3212 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
3214 AddToWorklistIfAllowed(OI);
3225 for (
const auto &Induction :
Legal->getInductionVars()) {
3226 auto *Ind = Induction.first;
3231 bool UniformInd =
all_of(Ind->users(), [&](User *U) ->
bool {
3232 auto *I = cast<Instruction>(U);
3233 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3234 IsVectorizedMemAccessUse(I, Ind);
3241 bool UniformIndUpdate =
all_of(IndUpdate->users(), [&](User *U) ->
bool {
3242 auto *I = cast<Instruction>(U);
3243 return I == Ind || Worklist.count(I) ||
3244 IsVectorizedMemAccessUse(I, IndUpdate);
3246 if (!UniformIndUpdate)
3250 AddToWorklistIfAllowed(Ind);
3251 AddToWorklistIfAllowed(IndUpdate);
3260 if (
Legal->getRuntimePointerChecking()->Need) {
3262 "runtime pointer checks needed. Enable vectorization of this "
3263 "loop with '#pragma clang loop vectorize(enable)' when "
3264 "compiling with -Os/-Oz",
3265 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3269 if (!
PSE.getPredicate().isAlwaysTrue()) {
3271 "runtime SCEV checks needed. Enable vectorization of this "
3272 "loop with '#pragma clang loop vectorize(enable)' when "
3273 "compiling with -Os/-Oz",
3274 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3279 if (!
Legal->getLAI()->getSymbolicStrides().empty()) {
3281 "runtime stride == 1 checks needed. Enable vectorization of "
3282 "this loop without such check by compiling with -Os/-Oz",
3283 "CantVersionLoopWithOptForSize",
ORE,
TheLoop);
3290bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
3291 if (IsScalableVectorizationAllowed)
3292 return *IsScalableVectorizationAllowed;
3294 IsScalableVectorizationAllowed =
false;
3298 if (Hints->isScalableVectorizationDisabled()) {
3300 "ScalableVectorizationDisabled", ORE, TheLoop);
3304 LLVM_DEBUG(
dbgs() <<
"LV: Scalable vectorization is available\n");
3307 std::numeric_limits<ElementCount::ScalarTy>::max());
3316 if (!canVectorizeReductions(MaxScalableVF)) {
3318 "Scalable vectorization not supported for the reduction "
3319 "operations found in this loop.",
3320 "ScalableVFUnfeasible", ORE, TheLoop);
3326 if (
any_of(ElementTypesInLoop, [&](
Type *Ty) {
3331 "for all element types found in this loop.",
3332 "ScalableVFUnfeasible", ORE, TheLoop);
3338 "for safe distance analysis.",
3339 "ScalableVFUnfeasible", ORE, TheLoop);
3343 IsScalableVectorizationAllowed =
true;
3348LoopVectorizationCostModel::getMaxLegalScalableVF(
unsigned MaxSafeElements) {
3349 if (!isScalableVectorizationAllowed())
3353 std::numeric_limits<ElementCount::ScalarTy>::max());
3354 if (
Legal->isSafeForAnyVectorWidth())
3355 return MaxScalableVF;
3363 "Max legal vector width too small, scalable vectorization "
3365 "ScalableVFUnfeasible", ORE, TheLoop);
3367 return MaxScalableVF;
3370FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
3371 unsigned MaxTripCount, ElementCount UserVF,
bool FoldTailByMasking) {
3373 unsigned SmallestType, WidestType;
3374 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
3380 unsigned MaxSafeElementsPowerOf2 =
3382 if (!
Legal->isSafeForAnyStoreLoadForwardDistances()) {
3383 unsigned SLDist =
Legal->getMaxStoreLoadForwardSafeDistanceInBits();
3384 MaxSafeElementsPowerOf2 =
3385 std::min(MaxSafeElementsPowerOf2, SLDist / WidestType);
3388 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
3390 if (!
Legal->isSafeForAnyVectorWidth())
3391 this->MaxSafeElements = MaxSafeElementsPowerOf2;
3393 LLVM_DEBUG(
dbgs() <<
"LV: The max safe fixed VF is: " << MaxSafeFixedVF
3395 LLVM_DEBUG(
dbgs() <<
"LV: The max safe scalable VF is: " << MaxSafeScalableVF
3400 auto MaxSafeUserVF =
3401 UserVF.
isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
3403 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
3406 return FixedScalableVFPair(
3412 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF));
3418 <<
" is unsafe, clamping to max safe VF="
3419 << MaxSafeFixedVF <<
".\n");
3421 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationFactor",
3424 <<
"User-specified vectorization factor "
3425 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3426 <<
" is unsafe, clamping to maximum safe vectorization factor "
3427 <<
ore::NV(
"VectorizationFactor", MaxSafeFixedVF);
3429 return MaxSafeFixedVF;
3434 <<
" is ignored because scalable vectors are not "
3437 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationFactor",
3440 <<
"User-specified vectorization factor "
3441 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3442 <<
" is ignored because the target does not support scalable "
3443 "vectors. The compiler will pick a more suitable value.";
3447 <<
" is unsafe. Ignoring scalable UserVF.\n");
3449 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationFactor",
3452 <<
"User-specified vectorization factor "
3453 <<
ore::NV(
"UserVectorizationFactor", UserVF)
3454 <<
" is unsafe. Ignoring the hint to let the compiler pick a "
3455 "more suitable value.";
3460 LLVM_DEBUG(
dbgs() <<
"LV: The Smallest and Widest types: " << SmallestType
3461 <<
" / " << WidestType <<
" bits.\n");
3466 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3467 MaxSafeFixedVF, FoldTailByMasking))
3471 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3472 MaxSafeScalableVF, FoldTailByMasking))
3473 if (MaxVF.isScalable()) {
3474 Result.ScalableVF = MaxVF;
3475 LLVM_DEBUG(
dbgs() <<
"LV: Found feasible scalable VF = " << MaxVF
3484 if (
Legal->getRuntimePointerChecking()->Need &&
TTI.hasBranchDivergence()) {
3488 "Not inserting runtime ptr check for divergent target",
3489 "runtime pointer checks needed. Not enabled for divergent target",
3490 "CantVersionLoopWithDivergentTarget",
ORE,
TheLoop);
3496 unsigned MaxTC =
PSE.getSmallConstantMaxTripCount();
3499 LLVM_DEBUG(
dbgs() <<
"LV: Found maximum trip count: " << MaxTC <<
'\n');
3502 "loop trip count is one, irrelevant for vectorization",
3513 Legal->getWidestInductionType()->getScalarSizeInBits() &&
3517 "Trip count computation wrapped",
3518 "backedge-taken count is -1, loop trip count wrapped to 0",
3523 switch (ScalarEpilogueStatus) {
3525 return computeFeasibleMaxVF(MaxTC, UserVF,
false);
3530 dbgs() <<
"LV: vector predicate hint/switch found.\n"
3531 <<
"LV: Not allowing scalar epilogue, creating predicated "
3532 <<
"vector loop.\n");
3539 dbgs() <<
"LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
3541 LLVM_DEBUG(
dbgs() <<
"LV: Not allowing scalar epilogue due to low trip "
3557 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
3558 "No decisions should have been taken at this point");
3568 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3572 if (MaxVScale &&
TTI.isVScaleKnownToBeAPowerOfTwo()) {
3573 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3574 *MaxPowerOf2RuntimeVF,
3577 MaxPowerOf2RuntimeVF = std::nullopt;
3580 auto NoScalarEpilogueNeeded = [
this, &UserIC](
unsigned MaxVF) {
3584 !
Legal->hasUncountableEarlyExit())
3586 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3591 const SCEV *BackedgeTakenCount =
PSE.getSymbolicMaxBackedgeTakenCount();
3593 BackedgeTakenCount ==
PSE.getBackedgeTakenCount()) &&
3594 "Invalid loop count");
3596 BackedgeTakenCount, SE->
getOne(BackedgeTakenCount->
getType()));
3603 if (MaxPowerOf2RuntimeVF > 0u) {
3605 "MaxFixedVF must be a power of 2");
3606 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3608 LLVM_DEBUG(
dbgs() <<
"LV: No tail will remain for any chosen VF.\n");
3614 if (ExpectedTC && ExpectedTC->isFixed() &&
3615 ExpectedTC->getFixedValue() <=
3616 TTI.getMinTripCountTailFoldingThreshold()) {
3617 if (MaxPowerOf2RuntimeVF > 0u) {
3623 LLVM_DEBUG(
dbgs() <<
"LV: Picking a fixed-width so that no tail will "
3624 "remain for any chosen VF.\n");
3631 "The trip count is below the minial threshold value.",
3632 "loop trip count is too low, avoiding vectorization",
"LowTripCount",
3647 <<
"LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3648 "try to generate VP Intrinsics with scalable vector "
3653 assert(ContainsScalableVF &&
"Expected scalable vector factor.");
3663 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking: vectorize with a "
3664 "scalar epilogue instead.\n");
3670 LLVM_DEBUG(
dbgs() <<
"LV: Can't fold tail by masking: don't vectorize\n");
3676 "unable to calculate the loop count due to complex control flow",
3682 "Cannot optimize for size and vectorize at the same time.",
3683 "cannot optimize for size and vectorize at the same time. "
3684 "Enable vectorization of this loop with '#pragma clang loop "
3685 "vectorize(enable)' when compiling with -Os/-Oz",
3697 if (
TTI.shouldConsiderVectorizationRegPressure())
3713 (
TTI.shouldMaximizeVectorBandwidth(RegKind) ||
3715 Legal->hasVectorCallVariants())));
3718ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
3719 ElementCount VF,
unsigned MaxTripCount,
bool FoldTailByMasking)
const {
3721 if (VF.
isScalable() && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
3722 auto Attr = TheFunction->getFnAttribute(Attribute::VScaleRange);
3723 auto Min = Attr.getVScaleRangeMin();
3730 if (MaxTripCount > 0 && requiresScalarEpilogue(
true))
3733 if (MaxTripCount && MaxTripCount <= EstimatedVF &&
3741 LLVM_DEBUG(
dbgs() <<
"LV: Clamping the MaxVF to maximum power of two not "
3742 "exceeding the constant trip count: "
3743 << ClampedUpperTripCount <<
"\n");
3745 FoldTailByMasking ? VF.
isScalable() :
false);
3750ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
3751 unsigned MaxTripCount,
unsigned SmallestType,
unsigned WidestType,
3752 ElementCount MaxSafeVF,
bool FoldTailByMasking) {
3753 bool ComputeScalableMaxVF = MaxSafeVF.
isScalable();
3759 auto MinVF = [](
const ElementCount &
LHS,
const ElementCount &
RHS) {
3761 "Scalable flags must match");
3769 ComputeScalableMaxVF);
3770 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
3772 << (MaxVectorElementCount * WidestType) <<
" bits.\n");
3774 if (!MaxVectorElementCount) {
3776 << (ComputeScalableMaxVF ?
"scalable" :
"fixed")
3777 <<
" vector registers.\n");
3781 ElementCount MaxVF = clampVFByMaxTripCount(MaxVectorElementCount,
3782 MaxTripCount, FoldTailByMasking);
3785 if (MaxVF != MaxVectorElementCount)
3793 MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
3795 MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
3797 if (useMaxBandwidth(RegKind)) {
3800 ComputeScalableMaxVF);
3801 MaxVF = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
3803 if (ElementCount MinVF =
3805 if (ElementCount::isKnownLT(MaxVF, MinVF)) {
3807 <<
") with target's minimum: " << MinVF <<
'\n');
3812 MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, FoldTailByMasking);
3814 if (MaxVectorElementCount != MaxVF) {
3818 invalidateCostModelingDecisions();
3826 const unsigned MaxTripCount,
3828 bool IsEpilogue)
const {
3834 unsigned EstimatedWidthB =
B.Width.getKnownMinValue();
3835 if (std::optional<unsigned> VScale = CM.getVScaleForTuning()) {
3836 if (
A.Width.isScalable())
3837 EstimatedWidthA *= *VScale;
3838 if (
B.Width.isScalable())
3839 EstimatedWidthB *= *VScale;
3846 return CostA < CostB ||
3847 (CostA == CostB && EstimatedWidthA > EstimatedWidthB);
3853 A.Width.isScalable() && !
B.Width.isScalable();
3864 return CmpFn(CostA * EstimatedWidthB, CostB * EstimatedWidthA);
3866 auto GetCostForTC = [MaxTripCount, HasTail](
unsigned VF,
3878 return VectorCost * (MaxTripCount / VF) +
3879 ScalarCost * (MaxTripCount % VF);
3880 return VectorCost *
divideCeil(MaxTripCount, VF);
3883 auto RTCostA = GetCostForTC(EstimatedWidthA, CostA,
A.ScalarCost);
3884 auto RTCostB = GetCostForTC(EstimatedWidthB, CostB,
B.ScalarCost);
3885 return CmpFn(RTCostA, RTCostB);
3891 bool IsEpilogue)
const {
3893 return LoopVectorizationPlanner::isMoreProfitable(
A,
B, MaxTripCount, HasTail,
3899 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3901 for (
const auto &Plan : VPlans) {
3910 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind,
3912 precomputeCosts(*Plan, VF, CostCtx);
3915 for (
auto &R : *VPBB) {
3916 if (!R.cost(VF, CostCtx).isValid())
3922 if (InvalidCosts.
empty())
3930 for (
auto &Pair : InvalidCosts)
3935 sort(InvalidCosts, [&Numbering](RecipeVFPair &
A, RecipeVFPair &
B) {
3936 unsigned NA = Numbering[
A.first];
3937 unsigned NB = Numbering[
B.first];
3952 Subset =
Tail.take_front(1);
3959 [](
const auto *R) {
return Instruction::PHI; })
3960 .Case<VPWidenSelectRecipe>(
3961 [](
const auto *R) {
return Instruction::Select; })
3962 .Case<VPWidenStoreRecipe>(
3963 [](
const auto *R) {
return Instruction::Store; })
3964 .Case<VPWidenLoadRecipe>(
3965 [](
const auto *R) {
return Instruction::Load; })
3966 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3967 [](
const auto *R) {
return Instruction::Call; })
3970 [](
const auto *R) {
return R->getOpcode(); })
3972 return R->getStoredValues().empty() ? Instruction::Load
3973 : Instruction::Store;
3981 if (Subset ==
Tail ||
Tail[Subset.size()].first != R) {
3982 std::string OutString;
3984 assert(!Subset.empty() &&
"Unexpected empty range");
3985 OS <<
"Recipe with invalid costs prevented vectorization at VF=(";
3986 for (
const auto &Pair : Subset)
3987 OS << (Pair.second == Subset.front().second ?
"" :
", ") << Pair.second;
3989 if (Opcode == Instruction::Call) {
3992 Name =
Int->getIntrinsicName();
3996 WidenCall ? WidenCall->getCalledScalarFunction()
3998 ->getLiveInIRValue());
4001 OS <<
" call to " << Name;
4006 Tail =
Tail.drop_front(Subset.size());
4010 Subset =
Tail.take_front(Subset.size() + 1);
4011 }
while (!
Tail.empty());
4033 switch (R.getVPDefID()) {
4034 case VPDef::VPDerivedIVSC:
4035 case VPDef::VPScalarIVStepsSC:
4036 case VPDef::VPReplicateSC:
4037 case VPDef::VPInstructionSC:
4038 case VPDef::VPCanonicalIVPHISC:
4039 case VPDef::VPVectorPointerSC:
4040 case VPDef::VPVectorEndPointerSC:
4041 case VPDef::VPExpandSCEVSC:
4042 case VPDef::VPEVLBasedIVPHISC:
4043 case VPDef::VPPredInstPHISC:
4044 case VPDef::VPBranchOnMaskSC:
4046 case VPDef::VPReductionSC:
4047 case VPDef::VPActiveLaneMaskPHISC:
4048 case VPDef::VPWidenCallSC:
4049 case VPDef::VPWidenCanonicalIVSC:
4050 case VPDef::VPWidenCastSC:
4051 case VPDef::VPWidenGEPSC:
4052 case VPDef::VPWidenIntrinsicSC:
4053 case VPDef::VPWidenSC:
4054 case VPDef::VPWidenSelectSC:
4055 case VPDef::VPBlendSC:
4056 case VPDef::VPFirstOrderRecurrencePHISC:
4057 case VPDef::VPHistogramSC:
4058 case VPDef::VPWidenPHISC:
4059 case VPDef::VPWidenIntOrFpInductionSC:
4060 case VPDef::VPWidenPointerInductionSC:
4061 case VPDef::VPReductionPHISC:
4062 case VPDef::VPInterleaveEVLSC:
4063 case VPDef::VPInterleaveSC:
4064 case VPDef::VPWidenLoadEVLSC:
4065 case VPDef::VPWidenLoadSC:
4066 case VPDef::VPWidenStoreEVLSC:
4067 case VPDef::VPWidenStoreSC:
4073 auto WillGenerateTargetVectors = [&
TTI, VF](
Type *VectorTy) {
4074 unsigned NumLegalParts =
TTI.getNumberOfParts(VectorTy);
4090 if (R.getNumDefinedValues() == 0 &&
4099 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
4101 if (!Visited.
insert({ScalarTy}).second)
4115 [](
auto *VPRB) { return VPRB->isReplicator(); });
4121 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ExpectedCost <<
".\n");
4122 assert(ExpectedCost.
isValid() &&
"Unexpected invalid cost for scalar loop");
4125 [](std::unique_ptr<VPlan> &
P) {
return P->hasScalarVFOnly(); }) &&
4126 "Expected Scalar VF to be a candidate");
4133 if (ForceVectorization &&
4134 (VPlans.size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
4138 ChosenFactor.
Cost = InstructionCost::getMax();
4141 for (
auto &
P : VPlans) {
4143 P->vectorFactors().end());
4146 if (
any_of(VFs, [
this](ElementCount VF) {
4147 return CM.shouldConsiderRegPressureForVF(VF);
4151 for (
unsigned I = 0;
I < VFs.size();
I++) {
4152 ElementCount VF = VFs[
I];
4160 if (CM.shouldConsiderRegPressureForVF(VF) &&
4168 VPCostContext CostCtx(CM.TTI, *CM.TLI, *
P, CM, CM.CostKind,
4170 VPRegionBlock *VectorRegion =
P->getVectorLoopRegion();
4171 assert(VectorRegion &&
"Expected to have a vector region!");
4174 for (VPRecipeBase &R : *VPBB) {
4178 switch (VPI->getOpcode()) {
4181 case Instruction::Select: {
4182 VPValue *VPV = VPI->getVPSingleValue();
4185 switch (WR->getOpcode()) {
4186 case Instruction::UDiv:
4187 case Instruction::SDiv:
4188 case Instruction::URem:
4189 case Instruction::SRem:
4196 C += VPI->cost(VF, CostCtx);
4200 unsigned Multiplier =
4203 C += VPI->cost(VF * Multiplier, CostCtx);
4207 C += VPI->cost(VF, CostCtx);
4219 <<
" costs: " << (Candidate.Cost / Width));
4222 << CM.getVScaleForTuning().value_or(1) <<
")");
4228 <<
"LV: Not considering vector loop of width " << VF
4229 <<
" because it will not generate any vector instructions.\n");
4236 <<
"LV: Not considering vector loop of width " << VF
4237 <<
" because it would cause replicated blocks to be generated,"
4238 <<
" which isn't allowed when optimizing for size.\n");
4242 if (isMoreProfitable(Candidate, ChosenFactor,
P->hasScalarTail()))
4243 ChosenFactor = Candidate;
4249 "There are conditional stores.",
4250 "store that is conditionally executed prevents vectorization",
4251 "ConditionalStore", ORE, OrigLoop);
4252 ChosenFactor = ScalarCost;
4256 !isMoreProfitable(ChosenFactor, ScalarCost,
4257 !CM.foldTailByMasking()))
dbgs()
4258 <<
"LV: Vectorization seems to be not beneficial, "
4259 <<
"but was forced by a user.\n");
4260 return ChosenFactor;
4264bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
4265 ElementCount VF)
const {
4268 if (
any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
4269 if (!Legal->isReductionVariable(&Phi))
4270 return Legal->isFixedOrderRecurrence(&Phi);
4271 return RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(
4272 Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind());
4278 for (
const auto &Entry :
Legal->getInductionVars()) {
4281 Entry.first->getIncomingValueForBlock(OrigLoop->getLoopLatch());
4282 for (User *U :
PostInc->users())
4286 for (User *U :
Entry.first->users())
4295 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
4309 if (!
TTI.preferEpilogueVectorization())
4314 if (
TTI.getMaxInterleaveFactor(VF) <= 1)
4319 :
TTI.getEpilogueVectorizationMinVF();
4327 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is disabled.\n");
4331 if (!CM.isScalarEpilogueAllowed()) {
4332 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because no "
4333 "epilogue is allowed.\n");
4339 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
4340 LLVM_DEBUG(
dbgs() <<
"LEV: Unable to vectorize epilogue because the loop "
4341 "is not a supported candidate.\n");
4346 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization factor is forced.\n");
4349 return {ForcedEC, 0, 0};
4351 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization forced factor is not "
4356 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
4358 dbgs() <<
"LEV: Epilogue vectorization skipped due to opt for size.\n");
4362 if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
4363 LLVM_DEBUG(
dbgs() <<
"LEV: Epilogue vectorization is not profitable for "
4375 Type *TCType = Legal->getWidestInductionType();
4376 const SCEV *RemainingIterations =
nullptr;
4377 unsigned MaxTripCount = 0;
4381 RemainingIterations =
4385 if (RemainingIterations->
isZero())
4395 << MaxTripCount <<
"\n");
4398 for (
auto &NextVF : ProfitableVFs) {
4405 if ((!NextVF.Width.isScalable() && MainLoopVF.
isScalable() &&
4407 (NextVF.Width.isScalable() &&
4409 (!NextVF.Width.isScalable() && !MainLoopVF.
isScalable() &&
4415 if (RemainingIterations && !NextVF.Width.isScalable()) {
4418 SE.
getConstant(TCType, NextVF.Width.getFixedValue()),
4419 RemainingIterations))
4423 if (Result.Width.isScalar() ||
4424 isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
4431 << Result.Width <<
"\n");
4435std::pair<unsigned, unsigned>
4437 unsigned MinWidth = -1U;
4438 unsigned MaxWidth = 8;
4444 for (
const auto &PhiDescriptorPair :
Legal->getReductionVars()) {
4448 MinWidth = std::min(
4452 MaxWidth = std::max(MaxWidth,
4457 MinWidth = std::min<unsigned>(
4458 MinWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
4459 MaxWidth = std::max<unsigned>(
4460 MaxWidth,
DL.getTypeSizeInBits(
T->getScalarType()).getFixedValue());
4463 return {MinWidth, MaxWidth};
4471 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
4485 if (!
Legal->isReductionVariable(PN))
4488 Legal->getRecurrenceDescriptor(PN);
4498 T = ST->getValueOperand()->getType();
4501 "Expected the load/store/recurrence type to be sized");
4525 if (!CM.isScalarEpilogueAllowed())
4530 LLVM_DEBUG(
dbgs() <<
"LV: Preference for VP intrinsics indicated. "
4531 "Unroll factor forced to be 1.\n");
4536 if (!Legal->isSafeForAnyVectorWidth())
4545 const bool HasReductions =
4551 if (LoopCost == 0) {
4553 LoopCost = CM.expectedCost(VF);
4555 LoopCost = cost(Plan, VF);
4556 assert(LoopCost.
isValid() &&
"Expected to have chosen a VF with valid cost");
4567 for (
auto &Pair : R.MaxLocalUsers) {
4568 Pair.second = std::max(Pair.second, 1U);
4582 unsigned IC = UINT_MAX;
4584 for (
const auto &Pair : R.MaxLocalUsers) {
4585 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
4588 << TTI.getRegisterClassName(Pair.first)
4589 <<
" register class\n");
4597 unsigned MaxLocalUsers = Pair.second;
4598 unsigned LoopInvariantRegs = 0;
4599 if (R.LoopInvariantRegs.contains(Pair.first))
4600 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
4602 unsigned TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
4606 TmpIC =
llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
4607 std::max(1U, (MaxLocalUsers - 1)));
4610 IC = std::min(IC, TmpIC);
4614 unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
4630 if (BestKnownTC && (BestKnownTC->isFixed() || VF.
isScalable())) {
4632 unsigned AvailableTC =
4638 if (CM.requiresScalarEpilogue(VF.
isVector()))
4641 unsigned InterleaveCountLB =
bit_floor(std::max(
4642 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
4656 unsigned InterleaveCountUB =
bit_floor(std::max(
4657 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
4658 MaxInterleaveCount = InterleaveCountLB;
4660 if (InterleaveCountUB != InterleaveCountLB) {
4661 unsigned TailTripCountUB =
4662 (AvailableTC % (EstimatedVF * InterleaveCountUB));
4663 unsigned TailTripCountLB =
4664 (AvailableTC % (EstimatedVF * InterleaveCountLB));
4667 if (TailTripCountUB == TailTripCountLB)
4668 MaxInterleaveCount = InterleaveCountUB;
4676 MaxInterleaveCount = InterleaveCountLB;
4680 assert(MaxInterleaveCount > 0 &&
4681 "Maximum interleave count must be greater than 0");
4685 if (IC > MaxInterleaveCount)
4686 IC = MaxInterleaveCount;
4689 IC = std::max(1u, IC);
4691 assert(IC > 0 &&
"Interleave count must be greater than 0.");
4695 if (VF.
isVector() && HasReductions) {
4696 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving because of reductions.\n");
4704 bool ScalarInterleavingRequiresPredication =
4706 return Legal->blockNeedsPredication(BB);
4708 bool ScalarInterleavingRequiresRuntimePointerCheck =
4709 (VF.
isScalar() && Legal->getRuntimePointerChecking()->Need);
4714 <<
"LV: IC is " << IC <<
'\n'
4715 <<
"LV: VF is " << VF <<
'\n');
4716 const bool AggressivelyInterleaveReductions =
4717 TTI.enableAggressiveInterleaving(HasReductions);
4718 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
4719 !ScalarInterleavingRequiresPredication && LoopCost <
SmallLoopCost) {
4728 unsigned NumStores = 0;
4729 unsigned NumLoads = 0;
4743 if (
unsigned StoreOps = InterleaveR->getNumStoreOperands())
4744 NumStores += StoreOps;
4746 NumLoads += InterleaveR->getNumDefinedValues();
4761 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
4762 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
4768 bool HasSelectCmpReductions =
4772 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4773 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
4774 RedR->getRecurrenceKind()) ||
4775 RecurrenceDescriptor::isFindIVRecurrenceKind(
4776 RedR->getRecurrenceKind()));
4778 if (HasSelectCmpReductions) {
4779 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving select-cmp reductions.\n");
4788 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
4789 bool HasOrderedReductions =
4792 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4794 return RedR && RedR->isOrdered();
4796 if (HasOrderedReductions) {
4798 dbgs() <<
"LV: Not interleaving scalar ordered reductions.\n");
4803 SmallIC = std::min(SmallIC,
F);
4804 StoresIC = std::min(StoresIC,
F);
4805 LoadsIC = std::min(LoadsIC,
F);
4809 std::max(StoresIC, LoadsIC) > SmallIC) {
4811 dbgs() <<
"LV: Interleaving to saturate store or load ports.\n");
4812 return std::max(StoresIC, LoadsIC);
4817 if (VF.
isScalar() && AggressivelyInterleaveReductions) {
4821 return std::max(IC / 2, SmallIC);
4824 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving to reduce branch cost.\n");
4830 if (AggressivelyInterleaveReductions) {
4839bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
Instruction *
I,
4849 assert((isPredicatedInst(
I)) &&
4850 "Expecting a scalar emulated instruction");
4863 if (InstsToScalarize.contains(VF) ||
4864 PredicatedBBsAfterVectorization.contains(VF))
4870 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4880 ScalarCostsTy ScalarCosts;
4887 !useEmulatedMaskMemRefHack(&
I, VF) &&
4888 computePredInstDiscount(&
I, ScalarCosts, VF) >= 0) {
4889 for (
const auto &[
I, IC] : ScalarCosts)
4890 ScalarCostsVF.
insert({
I, IC});
4893 for (
const auto &[
I,
Cost] : ScalarCosts) {
4895 if (!CI || !CallWideningDecisions.contains({CI, VF}))
4898 CallWideningDecisions[{CI, VF}].Cost =
Cost;
4902 PredicatedBBsAfterVectorization[VF].insert(BB);
4904 if (Pred->getSingleSuccessor() == BB)
4905 PredicatedBBsAfterVectorization[VF].insert(Pred);
4913 assert(!isUniformAfterVectorization(PredInst, VF) &&
4914 "Instruction marked uniform-after-vectorization will be predicated");
4932 if (!
I->hasOneUse() || PredInst->
getParent() !=
I->getParent() ||
4933 isScalarAfterVectorization(
I, VF))
4938 if (isScalarWithPredication(
I, VF))
4951 for (
Use &U :
I->operands())
4953 if (isUniformAfterVectorization(J, VF))
4964 while (!Worklist.
empty()) {
4968 if (ScalarCosts.contains(
I))
4988 if (isScalarWithPredication(
I, VF) && !
I->getType()->isVoidTy()) {
4991 ScalarCost +=
TTI.getScalarizationOverhead(
5004 for (Use &U :
I->operands())
5007 "Instruction has non-scalar type");
5008 if (CanBeScalarized(J))
5010 else if (needsExtract(J, VF)) {
5026 Discount += VectorCost - ScalarCost;
5027 ScalarCosts[
I] = ScalarCost;
5043 ValuesToIgnoreForVF);
5050 for (
Instruction &
I : BB->instructionsWithoutDebug()) {
5063 LLVM_DEBUG(
dbgs() <<
"LV: Found an estimated cost of " <<
C <<
" for VF "
5064 << VF <<
" For instruction: " <<
I <<
'\n');
5092 const Loop *TheLoop) {
5100 auto *SE = PSE.
getSE();
5101 unsigned NumOperands = Gep->getNumOperands();
5102 for (
unsigned Idx = 1; Idx < NumOperands; ++Idx) {
5103 Value *Opd = Gep->getOperand(Idx);
5105 !
Legal->isInductionVariable(Opd))
5114LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *
I,
5117 "Scalarization cost of instruction implies vectorization.");
5119 return InstructionCost::getInvalid();
5122 auto *SE = PSE.
getSE();
5153 if (isPredicatedInst(
I)) {
5158 VectorType::get(IntegerType::getInt1Ty(ValTy->
getContext()), VF);
5164 if (useEmulatedMaskMemRefHack(
I, VF))
5174LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *
I,
5180 int ConsecutiveStride =
Legal->isConsecutivePtr(ValTy,
Ptr);
5182 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5183 "Stride should be 1 or -1 for consecutive memory access");
5186 if (
Legal->isMaskRequired(
I)) {
5195 bool Reverse = ConsecutiveStride < 0;
5203LoopVectorizationCostModel::getUniformMemOpCost(Instruction *
I,
5221 bool IsLoopInvariantStoreValue =
Legal->isInvariant(
SI->getValueOperand());
5229 if (!IsLoopInvariantStoreValue)
5236LoopVectorizationCostModel::getGatherScatterCost(Instruction *
I,
5249 Legal->isMaskRequired(
I), Alignment,
5254LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *
I,
5256 const auto *Group = getInterleavedAccessGroup(
I);
5257 assert(Group &&
"Fail to get an interleaved access group.");
5264 unsigned InterleaveFactor = Group->getFactor();
5265 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
5268 SmallVector<unsigned, 4> Indices;
5269 for (
unsigned IF = 0;
IF < InterleaveFactor;
IF++)
5270 if (Group->getMember(IF))
5274 bool UseMaskForGaps =
5275 (Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed()) ||
5278 InsertPos->
getOpcode(), WideVecTy, Group->getFactor(), Indices,
5282 if (Group->isReverse()) {
5285 "Reverse masked interleaved access not supported.");
5286 Cost += Group->getNumMembers() *
5293std::optional<InstructionCost>
5300 return std::nullopt;
5318 return std::nullopt;
5329 Instruction *LastChain = InLoopReductionImmediateChains.lookup(RetI);
5331 return std::nullopt;
5337 ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
5346 BaseCost =
TTI.getMinMaxReductionCost(MinMaxID, VectorTy,
5349 BaseCost =
TTI.getArithmeticReductionCost(
5357 TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy,
CostKind);
5374 if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
5380 !
TheLoop->isLoopInvariant(Op0) && !
TheLoop->isLoopInvariant(Op1) &&
5392 TTI.getCastInstrCost(Op0->
getOpcode(), MulType, ExtType,
5395 TTI.getArithmeticInstrCost(Instruction::Mul, MulType,
CostKind);
5397 TTI.getCastInstrCost(RedOp->
getOpcode(), VectorTy, MulType,
5405 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
5406 return I == RetI ? RedCost : 0;
5408 !
TheLoop->isLoopInvariant(RedOp)) {
5417 TTI.getCastInstrCost(RedOp->
getOpcode(), VectorTy, ExtType,
5419 if (RedCost.
isValid() && RedCost < BaseCost + ExtCost)
5420 return I == RetI ? RedCost : 0;
5421 }
else if (RedOp && RdxDesc.
getOpcode() == Instruction::Add &&
5425 !
TheLoop->isLoopInvariant(Op0) && !
TheLoop->isLoopInvariant(Op1)) {
5444 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
5450 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
5451 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
5452 ExtraExtCost =
TTI.getCastInstrCost(
5459 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
5460 return I == RetI ? RedCost : 0;
5464 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
5470 if (RedCost.
isValid() && RedCost < MulCost + BaseCost)
5471 return I == RetI ? RedCost : 0;
5475 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
5479LoopVectorizationCostModel::getMemoryInstructionCost(
Instruction *
I,
5490 return TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
CostKind) +
5491 TTI.getMemoryOpCost(
I->getOpcode(), ValTy, Alignment, AS,
CostKind,
5494 return getWideningCost(
I, VF);
5498LoopVectorizationCostModel::getScalarizationOverhead(Instruction *
I,
5499 ElementCount VF)
const {
5504 return InstructionCost::getInvalid();
5532 Instruction::op_range
Ops = CI ? CI->
args() :
I->operands();
5537 for (
auto *V : filterExtractingOperands(
Ops, VF))
5560 if (
Legal->isUniformMemOp(
I, VF)) {
5561 auto IsLegalToScalarize = [&]() {
5581 return TheLoop->isLoopInvariant(
SI.getValueOperand());
5593 IsLegalToScalarize() ? getUniformMemOpCost(&
I, VF)
5599 if (GatherScatterCost < ScalarizationCost)
5609 int ConsecutiveStride =
Legal->isConsecutivePtr(
5611 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5612 "Expected consecutive stride.");
5621 unsigned NumAccesses = 1;
5624 assert(Group &&
"Fail to get an interleaved access group.");
5630 NumAccesses = Group->getNumMembers();
5632 InterleaveCost = getInterleaveGroupCost(&
I, VF);
5637 ? getGatherScatterCost(&
I, VF) * NumAccesses
5641 getMemInstScalarizationCost(&
I, VF) * NumAccesses;
5647 if (InterleaveCost <= GatherScatterCost &&
5648 InterleaveCost < ScalarizationCost) {
5650 Cost = InterleaveCost;
5651 }
else if (GatherScatterCost < ScalarizationCost) {
5653 Cost = GatherScatterCost;
5656 Cost = ScalarizationCost;
5663 for (
unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5664 if (
auto *
I = Group->getMember(Idx)) {
5666 getMemInstScalarizationCost(
I, VF));
5682 if (
TTI.prefersVectorizedAddressing())
5691 if (PtrDef &&
TheLoop->contains(PtrDef) &&
5699 while (!Worklist.
empty()) {
5701 for (
auto &
Op :
I->operands())
5704 AddrDefs.
insert(InstOp).second)
5708 auto UpdateMemOpUserCost = [
this, VF](
LoadInst *
LI) {
5712 for (
User *U :
LI->users()) {
5722 for (
auto *
I : AddrDefs) {
5741 for (
unsigned I = 0;
I < Group->getFactor(); ++
I) {
5758 ForcedScalars[VF].insert(
I);
5765 "Trying to set a vectorization decision for a scalar VF");
5767 auto ForcedScalar = ForcedScalars.find(VF);
5782 for (
auto &ArgOp : CI->
args())
5791 TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys,
CostKind);
5801 "Unexpected valid cost for scalarizing scalable vectors");
5808 if (VF.
isVector() && ((ForcedScalar != ForcedScalars.end() &&
5809 ForcedScalar->second.contains(CI)) ||
5817 bool MaskRequired =
Legal->isMaskRequired(CI);
5820 for (
Type *ScalarTy : ScalarTys)
5829 std::nullopt, *RedCost);
5840 if (Info.Shape.VF != VF)
5844 if (MaskRequired && !Info.isMasked())
5848 bool ParamsOk =
true;
5850 switch (Param.ParamKind) {
5856 if (!
PSE.getSE()->isLoopInvariant(
PSE.getSCEV(ScalarParam),
5893 VectorCost =
TTI.getCallInstrCost(
nullptr, RetTy, Tys,
CostKind);
5904 if (VectorCost <=
Cost) {
5926 return !OpI || !
TheLoop->contains(OpI) ||
5930 [
this](
Value *
Op) { return shouldConsiderInvariant(Op); }));
5942 return InstsToScalarize[VF][
I];
5945 auto ForcedScalar = ForcedScalars.find(VF);
5946 if (VF.
isVector() && ForcedScalar != ForcedScalars.end()) {
5947 auto InstSet = ForcedScalar->second;
5948 if (InstSet.count(
I))
5953 Type *RetTy =
I->getType();
5956 auto *SE =
PSE.getSE();
5960 [[maybe_unused]]
auto HasSingleCopyAfterVectorization =
5965 auto Scalarized = InstsToScalarize.find(VF);
5966 assert(Scalarized != InstsToScalarize.end() &&
5967 "VF not yet analyzed for scalarization profitability");
5968 return !Scalarized->second.count(
I) &&
5970 auto *UI = cast<Instruction>(U);
5971 return !Scalarized->second.count(UI);
5980 assert(
I->getOpcode() == Instruction::GetElementPtr ||
5981 I->getOpcode() == Instruction::PHI ||
5982 (
I->getOpcode() == Instruction::BitCast &&
5983 I->getType()->isPointerTy()) ||
5984 HasSingleCopyAfterVectorization(
I, VF));
5990 !
TTI.getNumberOfParts(VectorTy))
5994 switch (
I->getOpcode()) {
5995 case Instruction::GetElementPtr:
6001 case Instruction::Br: {
6008 bool ScalarPredicatedBB =
false;
6011 (PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(0)) ||
6012 PredicatedBBsAfterVectorization[VF].count(BI->
getSuccessor(1))) &&
6014 ScalarPredicatedBB =
true;
6016 if (ScalarPredicatedBB) {
6024 TTI.getScalarizationOverhead(
6032 return TTI.getCFInstrCost(Instruction::Br,
CostKind);
6040 case Instruction::Switch: {
6042 return TTI.getCFInstrCost(Instruction::Switch,
CostKind);
6044 return Switch->getNumCases() *
6045 TTI.getCmpSelInstrCost(
6047 toVectorTy(Switch->getCondition()->getType(), VF),
6051 case Instruction::PHI: {
6068 Type *ResultTy = Phi->getType();
6074 auto *Phi = dyn_cast<PHINode>(U);
6075 if (Phi && Phi->getParent() == TheLoop->getHeader())
6080 auto &ReductionVars =
Legal->getReductionVars();
6081 auto Iter = ReductionVars.find(HeaderUser);
6082 if (Iter != ReductionVars.end() &&
6084 Iter->second.getRecurrenceKind()))
6087 return (Phi->getNumIncomingValues() - 1) *
6088 TTI.getCmpSelInstrCost(
6089 Instruction::Select,
toVectorTy(ResultTy, VF),
6099 Intrinsic::vp_merge,
toVectorTy(Phi->getType(), VF),
6100 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
6104 return TTI.getCFInstrCost(Instruction::PHI,
CostKind);
6106 case Instruction::UDiv:
6107 case Instruction::SDiv:
6108 case Instruction::URem:
6109 case Instruction::SRem:
6113 ScalarCost : SafeDivisorCost;
6117 case Instruction::Add:
6118 case Instruction::Sub: {
6119 auto Info =
Legal->getHistogramInfo(
I);
6126 if (!RHS || RHS->getZExtValue() != 1)
6128 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
6132 Type *ScalarTy =
I->getType();
6136 {PtrTy, ScalarTy, MaskTy});
6139 return TTI.getIntrinsicInstrCost(ICA,
CostKind) + MulCost +
6140 TTI.getArithmeticInstrCost(
I->getOpcode(), VectorTy,
CostKind);
6144 case Instruction::FAdd:
6145 case Instruction::FSub:
6146 case Instruction::Mul:
6147 case Instruction::FMul:
6148 case Instruction::FDiv:
6149 case Instruction::FRem:
6150 case Instruction::Shl:
6151 case Instruction::LShr:
6152 case Instruction::AShr:
6153 case Instruction::And:
6154 case Instruction::Or:
6155 case Instruction::Xor: {
6159 if (
I->getOpcode() == Instruction::Mul &&
6160 ((
TheLoop->isLoopInvariant(
I->getOperand(0)) &&
6161 PSE.getSCEV(
I->getOperand(0))->isOne()) ||
6162 (
TheLoop->isLoopInvariant(
I->getOperand(1)) &&
6163 PSE.getSCEV(
I->getOperand(1))->isOne())))
6172 Value *Op2 =
I->getOperand(1);
6178 auto Op2Info =
TTI.getOperandInfo(Op2);
6184 return TTI.getArithmeticInstrCost(
6186 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6187 Op2Info, Operands,
I,
TLI);
6189 case Instruction::FNeg: {
6190 return TTI.getArithmeticInstrCost(
6192 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6193 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6194 I->getOperand(0),
I);
6196 case Instruction::Select: {
6201 const Value *Op0, *Op1;
6212 return TTI.getArithmeticInstrCost(
6214 VectorTy,
CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1},
I);
6217 Type *CondTy =
SI->getCondition()->getType();
6223 Pred = Cmp->getPredicate();
6224 return TTI.getCmpSelInstrCost(
I->getOpcode(), VectorTy, CondTy, Pred,
6225 CostKind, {TTI::OK_AnyValue, TTI::OP_None},
6226 {TTI::OK_AnyValue, TTI::OP_None},
I);
6228 case Instruction::ICmp:
6229 case Instruction::FCmp: {
6230 Type *ValTy =
I->getOperand(0)->getType();
6236 MinBWs[
I] == MinBWs[Op0AsInstruction]) &&
6237 "if both the operand and the compare are marked for "
6238 "truncation, they must have the same bitwidth");
6243 return TTI.getCmpSelInstrCost(
6246 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
I);
6248 case Instruction::Store:
6249 case Instruction::Load: {
6254 "CM decision should be taken at this point");
6261 return getMemoryInstructionCost(
I, VF);
6263 case Instruction::BitCast:
6264 if (
I->getType()->isPointerTy())
6267 case Instruction::ZExt:
6268 case Instruction::SExt:
6269 case Instruction::FPToUI:
6270 case Instruction::FPToSI:
6271 case Instruction::FPExt:
6272 case Instruction::PtrToInt:
6273 case Instruction::IntToPtr:
6274 case Instruction::SIToFP:
6275 case Instruction::UIToFP:
6276 case Instruction::Trunc:
6277 case Instruction::FPTrunc: {
6281 "Expected a load or a store!");
6307 unsigned Opcode =
I->getOpcode();
6310 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
6313 CCH = ComputeCCH(Store);
6316 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
6317 Opcode == Instruction::FPExt) {
6319 CCH = ComputeCCH(Load);
6327 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
6328 Trunc->getSrcTy(), CCH,
CostKind, Trunc);
6335 Type *SrcScalarTy =
I->getOperand(0)->getType();
6347 (
I->getOpcode() == Instruction::ZExt ||
6348 I->getOpcode() == Instruction::SExt))
6352 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
CostKind,
I);
6354 case Instruction::Call:
6356 case Instruction::ExtractValue:
6358 case Instruction::Alloca:
6366 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
CostKind);
6381 auto IsLiveOutDead = [
this, RequiresScalarEpilogue](
User *U) {
6382 return RequiresScalarEpilogue &&
6396 all_of(
I.users(), [
this, IsLiveOutDead](
User *U) {
6397 return VecValuesToIgnore.contains(U) ||
6398 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
6407 if (Group->getInsertPos() == &
I)
6410 DeadInterleavePointerOps.
push_back(PointerOp);
6416 if (Br->isConditional())
6423 for (
unsigned I = 0;
I != DeadInterleavePointerOps.
size(); ++
I) {
6426 Instruction *UI = cast<Instruction>(U);
6427 return !VecValuesToIgnore.contains(U) &&
6428 (!isAccessInterleaved(UI) ||
6429 getInterleavedAccessGroup(UI)->getInsertPos() == UI);
6449 for (
unsigned I = 0;
I != DeadOps.
size(); ++
I) {
6461 if ((ThenEmpty && ElseEmpty) ||
6463 ElseBB->
phis().empty()) ||
6465 ThenBB->
phis().empty())) {
6477 return !VecValuesToIgnore.contains(U) &&
6478 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
6486 [
this](
User *U) { return ValuesToIgnore.contains(U); }))
6495 for (
const auto &Reduction :
Legal->getReductionVars()) {
6502 for (
const auto &Induction :
Legal->getInductionVars()) {
6511 if (!InLoopReductions.empty())
6514 for (
const auto &Reduction :
Legal->getReductionVars()) {
6515 PHINode *Phi = Reduction.first;
6526 !
TTI.preferInLoopReduction(Kind, Phi->getType()))
6534 bool InLoop = !ReductionOperations.
empty();
6537 InLoopReductions.insert(Phi);
6540 for (
auto *
I : ReductionOperations) {
6541 InLoopReductionImmediateChains[
I] = LastChain;
6545 LLVM_DEBUG(
dbgs() <<
"LV: Using " << (InLoop ?
"inloop" :
"out of loop")
6546 <<
" reduction for phi: " << *Phi <<
"\n");
6559 unsigned WidestType;
6563 TTI.enableScalableVectorization()
6568 unsigned N =
RegSize.getKnownMinValue() / WidestType;
6579 if (!OrigLoop->isInnermost()) {
6589 <<
"overriding computed VF.\n");
6592 }
else if (UserVF.
isScalable() && !TTI.supportsScalableVectors() &&
6594 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing. Scalable VF requested, but "
6595 <<
"not supported by the target.\n");
6597 "Scalable vectorization requested but not supported by the target",
6598 "the scalable user-specified vectorization width for outer-loop "
6599 "vectorization cannot be used because the target does not support "
6600 "scalable vectors.",
6601 "ScalableVFUnfeasible", ORE, OrigLoop);
6606 "VF needs to be a power of two");
6608 <<
"VF " << VF <<
" to build VPlans.\n");
6618 return {VF, 0 , 0 };
6622 dbgs() <<
"LV: Not vectorizing. Inner loops aren't supported in the "
6623 "VPlan-native path.\n");
6628 assert(OrigLoop->isInnermost() &&
"Inner loop expected.");
6629 CM.collectValuesToIgnore();
6630 CM.collectElementTypesForWidening();
6637 if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
6641 <<
"LV: Invalidate all interleaved groups due to fold-tail by masking "
6642 "which requires masked-interleaved support.\n");
6643 if (CM.InterleaveInfo.invalidateGroups())
6647 CM.invalidateCostModelingDecisions();
6650 if (CM.foldTailByMasking())
6651 Legal->prepareToFoldTailByMasking();
6658 "UserVF ignored because it may be larger than the maximal safe VF",
6659 "InvalidUserVF", ORE, OrigLoop);
6662 "VF needs to be a power of two");
6665 CM.collectInLoopReductions();
6666 if (CM.selectUserVectorizationFactor(UserVF)) {
6668 buildVPlansWithVPRecipes(UserVF, UserVF);
6673 "InvalidCost", ORE, OrigLoop);
6686 CM.collectInLoopReductions();
6687 for (
const auto &VF : VFCandidates) {
6689 CM.collectNonVectorizedAndSetWideningDecisions(VF);
6708 return CM.isUniformAfterVectorization(
I, VF);
6712 return CM.ValuesToIgnore.contains(UI) ||
6713 (IsVector &&
CM.VecValuesToIgnore.contains(UI)) ||
6733 for (
const auto &[
IV, IndDesc] :
Legal->getInductionVars()) {
6735 IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
6737 for (
unsigned I = 0;
I != IVInsts.
size();
I++) {
6738 for (
Value *
Op : IVInsts[
I]->operands()) {
6740 if (
Op ==
IV || !OpI || !OrigLoop->contains(OpI) || !
Op->hasOneUse())
6746 for (User *U :
IV->users()) {
6759 if (TC == VF && !CM.foldTailByMasking())
6763 for (Instruction *IVInst : IVInsts) {
6768 dbgs() <<
"Cost of " << InductionCost <<
" for VF " << VF
6769 <<
": induction instruction " << *IVInst <<
"\n";
6771 Cost += InductionCost;
6781 CM.TheLoop->getExitingBlocks(Exiting);
6782 SetVector<Instruction *> ExitInstrs;
6784 for (BasicBlock *EB : Exiting) {
6789 ExitInstrs.
insert(CondI);
6793 for (
unsigned I = 0;
I != ExitInstrs.
size(); ++
I) {
6795 if (!OrigLoop->contains(CondI) ||
6800 dbgs() <<
"Cost of " << CondICost <<
" for VF " << VF
6801 <<
": exit condition instruction " << *CondI <<
"\n";
6807 any_of(OpI->users(), [&ExitInstrs,
this](User *U) {
6808 return OrigLoop->contains(cast<Instruction>(U)->getParent()) &&
6809 !ExitInstrs.contains(cast<Instruction>(U));
6821 for (BasicBlock *BB : OrigLoop->blocks()) {
6825 if (BB == OrigLoop->getLoopLatch())
6827 auto BranchCost = CostCtx.
getLegacyCost(BB->getTerminator(), VF);
6834 for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
6840 dbgs() <<
"Cost of " << ForcedCost <<
" for VF " << VF
6841 <<
": forced scalar " << *ForcedScalar <<
"\n";
6845 for (
const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
6850 dbgs() <<
"Cost of " << ScalarCost <<
" for VF " << VF
6851 <<
": profitable to scalarize " << *Scalarized <<
"\n";
6860 ElementCount VF)
const {
6861 VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, CM.CostKind, *PSE.
getSE());
6869 <<
" (Estimated cost per lane: ");
6871 double CostPerLane = double(
Cost.
getValue()) / EstimatedWidth;
6894 return &WidenMem->getIngredient();
6903 if (!VPI || VPI->getOpcode() != Instruction::Select ||
6904 VPI->getNumUsers() != 1)
6908 switch (WR->getOpcode()) {
6909 case Instruction::UDiv:
6910 case Instruction::SDiv:
6911 case Instruction::URem:
6912 case Instruction::SRem:
6925 auto *IG =
IR->getInterleaveGroup();
6926 unsigned NumMembers = IG->getNumMembers();
6927 for (
unsigned I = 0;
I != NumMembers; ++
I) {
6961 if (RepR->isSingleScalar() &&
6963 RepR->getUnderlyingInstr(), VF))
6966 if (
Instruction *UI = GetInstructionForCost(&R)) {
6971 if (
match(&R,
m_Cmp(Pred, m_VPValue(), m_VPValue())) &&
6983 return any_of(TheLoop->
blocks(), [&SeenInstrs, &CostCtx,
6985 return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
6988 if (isa<PHINode>(&I) && BB == TheLoop->getHeader() &&
6989 CostCtx.CM.Legal->isInductionPhi(cast<PHINode>(&I)))
6991 return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
7001 VPlan &FirstPlan = *VPlans[0];
7007 ?
"Reciprocal Throughput\n"
7009 ?
"Instruction Latency\n"
7012 ?
"Code Size and Latency\n"
7017 "More than a single plan/VF w/o any plan having scalar VF");
7021 LLVM_DEBUG(
dbgs() <<
"LV: Scalar loop costs: " << ScalarCost <<
".\n");
7026 if (ForceVectorization) {
7033 for (
auto &
P : VPlans) {
7035 P->vectorFactors().end());
7039 return CM.shouldConsiderRegPressureForVF(VF);
7043 for (
unsigned I = 0;
I < VFs.
size();
I++) {
7050 <<
"LV: Not considering vector loop of width " << VF
7051 <<
" because it will not generate any vector instructions.\n");
7057 <<
"LV: Not considering vector loop of width " << VF
7058 <<
" because it would cause replicated blocks to be generated,"
7059 <<
" which isn't allowed when optimizing for size.\n");
7066 if (CM.shouldConsiderRegPressureForVF(VF) &&
7068 LLVM_DEBUG(
dbgs() <<
"LV(REG): Not considering vector loop of width "
7069 << VF <<
" because it uses too many registers\n");
7073 if (isMoreProfitable(CurrentFactor, BestFactor,
P->hasScalarTail()))
7074 BestFactor = CurrentFactor;
7077 if (isMoreProfitable(CurrentFactor, ScalarFactor,
P->hasScalarTail()))
7078 ProfitableVFs.push_back(CurrentFactor);
7094 VPCostContext CostCtx(CM.TTI, *CM.TLI, BestPlan, CM, CM.CostKind,
7096 precomputeCosts(BestPlan, BestFactor.
Width, CostCtx);
7103 BestFactor.
Width) ||
7106 " VPlan cost model and legacy cost model disagreed");
7108 "when vectorizing, the scalar cost must be computed.");
7118 "RdxResult must be ComputeFindIVResult");
7136 if (!EpiRedResult ||
7142 auto *EpiRedHeaderPhi =
7144 RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
7145 Value *MainResumeValue;
7149 "unexpected start recipe");
7150 MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
7152 MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
7154 [[maybe_unused]]
Value *StartV =
7155 EpiRedResult->getOperand(1)->getLiveInIRValue();
7158 "AnyOf expected to start with ICMP_NE");
7159 assert(Cmp->getOperand(1) == StartV &&
7160 "AnyOf expected to start by comparing main resume value to original "
7162 MainResumeValue = Cmp->getOperand(0);
7165 Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
7167 Value *Cmp, *OrigResumeV, *CmpOp;
7168 [[maybe_unused]]
bool IsExpectedPattern =
7169 match(MainResumeValue,
7175 assert(IsExpectedPattern &&
"Unexpected reduction resume pattern");
7176 MainResumeValue = OrigResumeV;
7191 "Trying to execute plan with unsupported VF");
7193 "Trying to execute plan with unsupported UF");
7195 ++LoopsEarlyExitVectorized;
7203 bool HasBranchWeights =
7205 if (HasBranchWeights) {
7206 std::optional<unsigned> VScale = CM.getVScaleForTuning();
7208 BestVPlan, BestVF, VScale);
7213 attachRuntimeChecks(BestVPlan, ILV.
RTChecks, HasBranchWeights);
7226 OrigLoop->getStartLoc(),
7227 OrigLoop->getHeader())
7228 <<
"Created vector loop never executes due to insufficient trip "
7244 BestVPlan, VectorPH, CM.foldTailByMasking(),
7245 CM.requiresScalarEpilogue(BestVF.
isVector()));
7257 assert(VectorizingEpilogue &&
"should only re-use the existing trip "
7258 "count during epilogue vectorization");
7262 OrigLoop->getParentLoop(),
7263 Legal->getWidestInductionType());
7265#ifdef EXPENSIVE_CHECKS
7266 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
7277 "final VPlan is invalid");
7284 if (!Exit->hasPredecessors())
7306 MDNode *LID = OrigLoop->getLoopID();
7307 unsigned OrigLoopInvocationWeight = 0;
7308 std::optional<unsigned> OrigAverageTripCount =
7320 bool DisableRuntimeUnroll = !ILV.
RTChecks.hasChecks() && !BestVF.
isScalar();
7322 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
7324 HeaderVPBB, BestVPlan, VectorizingEpilogue, LID, OrigAverageTripCount,
7325 OrigLoopInvocationWeight,
7327 DisableRuntimeUnroll);
7335 return ExpandedSCEVs;
7350 EPI.EpilogueIterationCountCheck =
7352 EPI.EpilogueIterationCountCheck->setName(
"iter.check");
7362 EPI.MainLoopIterationCountCheck =
7371 dbgs() <<
"Create Skeleton for epilogue vectorized loop (first pass)\n"
7372 <<
"Main Loop VF:" <<
EPI.MainLoopVF
7373 <<
", Main Loop UF:" <<
EPI.MainLoopUF
7374 <<
", Epilogue Loop VF:" <<
EPI.EpilogueVF
7375 <<
", Epilogue Loop UF:" <<
EPI.EpilogueUF <<
"\n";
7381 dbgs() <<
"intermediate fn:\n"
7382 << *
OrigLoop->getHeader()->getParent() <<
"\n";
7388 assert(Bypass &&
"Expected valid bypass basic block.");
7392 VectorPH, ForEpilogue ?
EPI.EpilogueVF :
EPI.MainLoopVF,
7393 ForEpilogue ?
EPI.EpilogueUF :
EPI.MainLoopUF);
7397 TCCheckBlock->
setName(
"vector.main.loop.iter.check");
7423 return TCCheckBlock;
7436 OriginalScalarPH->
setName(
"vec.epilog.iter.check");
7444 R.moveBefore(*NewEntry, NewEntry->
end());
7448 Plan.setEntry(NewEntry);
7451 return OriginalScalarPH;
7456 dbgs() <<
"Create Skeleton for epilogue vectorized loop (second pass)\n"
7457 <<
"Epilogue Loop VF:" <<
EPI.EpilogueVF
7458 <<
", Epilogue Loop UF:" <<
EPI.EpilogueUF <<
"\n";
7464 dbgs() <<
"final fn:\n" << *
OrigLoop->getHeader()->getParent() <<
"\n";
7472 "Must be called with either a load or store");
7476 CM.getWideningDecision(
I, VF);
7478 "CM decision should be taken at this point.");
7481 if (CM.isScalarAfterVectorization(
I, VF) ||
7482 CM.isProfitableToScalarize(
I, VF))
7491 if (
Legal->isMaskRequired(
I))
7492 Mask = getBlockInMask(Builder.getInsertBlock());
7497 CM.getWideningDecision(
I,
Range.Start);
7505 Ptr->getUnderlyingValue()->stripPointerCasts());
7513 CM.foldTailByMasking() || !
GEP
7515 :
GEP->getNoWrapFlags().withoutNoUnsignedWrap();
7518 -1, Flags,
I->getDebugLoc());
7521 GEP ?
GEP->getNoWrapFlags()
7525 Builder.insert(VectorPtr);
7529 return new VPWidenLoadRecipe(*Load,
Ptr, Mask, Consecutive,
Reverse,
7530 VPIRMetadata(*Load, LVer),
I->getDebugLoc());
7533 return new VPWidenStoreRecipe(*Store,
Ptr, Operands[0], Mask, Consecutive,
7534 Reverse, VPIRMetadata(*Store, LVer),
7540static VPWidenIntOrFpInductionRecipe *
7547 "step must be loop invariant");
7554 TruncI->getDebugLoc());
7558 IndDesc, Phi->getDebugLoc());
7561VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
7566 if (
auto *
II =
Legal->getIntOrFpInductionDescriptor(Phi))
7568 *PSE.
getSE(), *OrigLoop);
7571 if (
auto *
II =
Legal->getPointerInductionDescriptor(Phi)) {
7573 return new VPWidenPointerInductionRecipe(
7574 Phi, Operands[0], Step, &Plan.
getVFxUF(), *
II,
7576 [&](ElementCount VF) {
7577 return CM.isScalarAfterVectorization(Phi, VF);
7580 Phi->getDebugLoc());
7585VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
7594 auto IsOptimizableIVTruncate =
7595 [&](
Instruction *
K) -> std::function<
bool(ElementCount)> {
7596 return [=](ElementCount VF) ->
bool {
7597 return CM.isOptimizableIVTruncate(K, VF);
7602 IsOptimizableIVTruncate(
I),
Range)) {
7605 const InductionDescriptor &
II = *
Legal->getIntOrFpInductionDescriptor(Phi);
7613VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
7617 [
this, CI](ElementCount VF) {
7618 return CM.isScalarWithPredication(CI, VF);
7626 if (
ID && (
ID == Intrinsic::assume ||
ID == Intrinsic::lifetime_end ||
7627 ID == Intrinsic::lifetime_start ||
ID == Intrinsic::sideeffect ||
7628 ID == Intrinsic::pseudoprobe ||
7629 ID == Intrinsic::experimental_noalias_scope_decl))
7635 bool ShouldUseVectorIntrinsic =
7637 [&](ElementCount VF) ->
bool {
7638 return CM.getCallWideningDecision(CI, VF).Kind ==
7642 if (ShouldUseVectorIntrinsic)
7643 return new VPWidenIntrinsicRecipe(*CI,
ID,
Ops, CI->
getType(),
7647 std::optional<unsigned> MaskPos;
7651 [&](ElementCount VF) ->
bool {
7666 LoopVectorizationCostModel::CallWideningDecision Decision =
7667 CM.getCallWideningDecision(CI, VF);
7677 if (ShouldUseVectorCall) {
7678 if (MaskPos.has_value()) {
7686 VPValue *
Mask =
nullptr;
7687 if (
Legal->isMaskRequired(CI))
7688 Mask = getBlockInMask(Builder.getInsertBlock());
7693 Ops.insert(
Ops.begin() + *MaskPos, Mask);
7696 Ops.push_back(Operands.
back());
7697 return new VPWidenCallRecipe(CI, Variant,
Ops, CI->
getDebugLoc());
7703bool VPRecipeBuilder::shouldWiden(Instruction *
I, VFRange &
Range)
const {
7705 !
isa<StoreInst>(
I) &&
"Instruction should have been handled earlier");
7708 auto WillScalarize = [
this,
I](ElementCount VF) ->
bool {
7709 return CM.isScalarAfterVectorization(
I, VF) ||
7710 CM.isProfitableToScalarize(
I, VF) ||
7711 CM.isScalarWithPredication(
I, VF);
7717VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *
I,
7719 switch (
I->getOpcode()) {
7722 case Instruction::SDiv:
7723 case Instruction::UDiv:
7724 case Instruction::SRem:
7725 case Instruction::URem: {
7728 if (CM.isPredicatedInst(
I)) {
7730 VPValue *
Mask = getBlockInMask(Builder.getInsertBlock());
7733 auto *SafeRHS = Builder.createSelect(Mask,
Ops[1], One,
I->getDebugLoc());
7735 return new VPWidenRecipe(*
I,
Ops);
7739 case Instruction::Add:
7740 case Instruction::And:
7741 case Instruction::AShr:
7742 case Instruction::FAdd:
7743 case Instruction::FCmp:
7744 case Instruction::FDiv:
7745 case Instruction::FMul:
7746 case Instruction::FNeg:
7747 case Instruction::FRem:
7748 case Instruction::FSub:
7749 case Instruction::ICmp:
7750 case Instruction::LShr:
7751 case Instruction::Mul:
7752 case Instruction::Or:
7753 case Instruction::Select:
7754 case Instruction::Shl:
7755 case Instruction::Sub:
7756 case Instruction::Xor:
7757 case Instruction::Freeze: {
7763 ScalarEvolution &SE = *PSE.
getSE();
7764 auto GetConstantViaSCEV = [
this, &SE](VPValue *
Op) {
7765 if (!
Op->isLiveIn())
7767 Value *
V =
Op->getUnderlyingValue();
7776 if (
I->getOpcode() == Instruction::Mul)
7777 NewOps[0] = GetConstantViaSCEV(NewOps[0]);
7779 NewOps[1] = GetConstantViaSCEV(NewOps[1]);
7781 return new VPWidenRecipe(*
I, NewOps);
7783 case Instruction::ExtractValue: {
7785 Type *I32Ty = IntegerType::getInt32Ty(
I->getContext());
7787 assert(EVI->getNumIndices() == 1 &&
"Expected one extractvalue index");
7788 unsigned Idx = EVI->getIndices()[0];
7789 NewOps.push_back(Plan.
getOrAddLiveIn(ConstantInt::get(I32Ty, Idx,
false)));
7790 return new VPWidenRecipe(*
I, NewOps);
7796VPRecipeBuilder::tryToWidenHistogram(
const HistogramInfo *HI,
7799 unsigned Opcode =
HI->Update->getOpcode();
7800 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
7801 "Histogram update operation must be an Add or Sub");
7807 HGramOps.
push_back(getVPValueOrAddLiveIn(
HI->Update->getOperand(1)));
7811 if (
Legal->isMaskRequired(
HI->Store))
7812 HGramOps.
push_back(getBlockInMask(Builder.getInsertBlock()));
7814 return new VPHistogramRecipe(Opcode, HGramOps,
HI->Store->getDebugLoc());
7821 [&](
ElementCount VF) {
return CM.isUniformAfterVectorization(
I, VF); },
7824 bool IsPredicated = CM.isPredicatedInst(
I);
7832 case Intrinsic::assume:
7833 case Intrinsic::lifetime_start:
7834 case Intrinsic::lifetime_end:
7856 VPValue *BlockInMask =
nullptr;
7857 if (!IsPredicated) {
7861 LLVM_DEBUG(
dbgs() <<
"LV: Scalarizing and predicating:" << *
I <<
"\n");
7872 assert((
Range.Start.isScalar() || !IsUniform || !IsPredicated ||
7874 "Should not predicate a uniform recipe");
7885 PartialReductionChains;
7886 for (
const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
7887 getScaledReductions(Phi, RdxDesc.getLoopExitInstr(),
Range,
7888 PartialReductionChains);
7897 for (
const auto &[PartialRdx,
_] : PartialReductionChains)
7898 PartialReductionOps.
insert(PartialRdx.ExtendUser);
7900 auto ExtendIsOnlyUsedByPartialReductions =
7902 return all_of(Extend->users(), [&](
const User *U) {
7903 return PartialReductionOps.contains(U);
7909 for (
auto Pair : PartialReductionChains) {
7911 if (ExtendIsOnlyUsedByPartialReductions(Chain.
ExtendA) &&
7912 (!Chain.
ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.
ExtendB)))
7913 ScaledReductionMap.try_emplace(Chain.
Reduction, Pair.second);
7917bool VPRecipeBuilder::getScaledReductions(
7919 SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
7920 if (!CM.TheLoop->contains(RdxExitInstr))
7927 Value *
Op = Update->getOperand(0);
7928 Value *PhiOp = Update->getOperand(1);
7936 if (getScaledReductions(
PHI, OpInst,
Range, Chains)) {
7937 PHI = Chains.rbegin()->first.Reduction;
7939 Op = Update->getOperand(0);
7940 PhiOp = Update->getOperand(1);
7948 using namespace llvm::PatternMatch;
7955 std::optional<unsigned> BinOpc;
7956 Type *ExtOpTypes[2] = {
nullptr};
7959 auto CollectExtInfo = [
this, &Exts, &ExtOpTypes,
7960 &ExtKinds](SmallVectorImpl<Value *> &
Ops) ->
bool {
7965 ExtOpTypes[
I] = ExtOpTypes[0];
7966 ExtKinds[
I] = ExtKinds[0];
7975 if (!CM.TheLoop->contains(Exts[
I]))
7993 if (!CollectExtInfo(
Ops))
7996 BinOpc = std::make_optional(ExtendUser->
getOpcode());
8000 if (!CollectExtInfo(
Ops))
8003 ExtendUser = Update;
8004 BinOpc = std::nullopt;
8008 PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
8010 TypeSize PHISize =
PHI->getType()->getPrimitiveSizeInBits();
8017 [&](ElementCount VF) {
8019 Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
8020 PHI->getType(), VF, ExtKinds[0], ExtKinds[1], BinOpc,
8025 Chains.emplace_back(Chain, TargetScaleFactor);
8044 "Non-header phis should have been handled during predication");
8046 assert(Operands.
size() == 2 &&
"Must have 2 operands for header phis");
8047 if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands,
Range)))
8051 assert((Legal->isReductionVariable(Phi) ||
8052 Legal->isFixedOrderRecurrence(Phi)) &&
8053 "can only widen reductions and fixed-order recurrences here");
8054 VPValue *StartV = Operands[0];
8055 if (Legal->isReductionVariable(Phi)) {
8058 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
8061 unsigned ScaleFactor =
8065 CM.useOrderedReductions(RdxDesc), ScaleFactor);
8077 assert(!R->isPhi() &&
"only VPPhi nodes expected at this point");
8079 if (
isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8089 return tryToWidenCall(CI, Operands,
Range);
8092 if (
auto HistInfo = Legal->getHistogramInfo(
SI))
8093 return tryToWidenHistogram(*HistInfo, Operands);
8096 return tryToWidenMemory(Instr, Operands,
Range);
8099 if (
auto PartialRed =
8104 if (!shouldWiden(Instr,
Range))
8119 return tryToWiden(Instr, Operands);
8125 unsigned ScaleFactor) {
8127 "Unexpected number of operands for partial reduction");
8140 unsigned ReductionOpcode = Reduction->getOpcode();
8141 if (ReductionOpcode == Instruction::Sub) {
8142 auto *
const Zero = ConstantInt::get(Reduction->getType(), 0);
8144 Ops.push_back(Plan.getOrAddLiveIn(Zero));
8145 Ops.push_back(BinOp);
8148 ReductionOpcode = Instruction::Add;
8152 if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
8153 assert((ReductionOpcode == Instruction::Add ||
8154 ReductionOpcode == Instruction::Sub) &&
8155 "Expected an ADD or SUB operation for predicated partial "
8156 "reductions (because the neutral element in the mask is zero)!");
8159 Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
8160 BinOp = Builder.createSelect(
Cond, BinOp, Zero, Reduction->getDebugLoc());
8163 ScaleFactor, Reduction);
8166void LoopVectorizationPlanner::buildVPlansWithVPRecipes(
ElementCount MinVF,
8171 assert(OrigLoop->isInnermost() &&
"Inner loop expected.");
8175 OrigLoop, LI, DT, PSE.
getSE());
8180 LVer.prepareNoAliasMetadata();
8186 OrigLoop, *LI,
Legal->getWidestInductionType(),
8189 auto MaxVFTimes2 = MaxVF * 2;
8191 VFRange SubRange = {VF, MaxVFTimes2};
8192 if (
auto Plan = tryToBuildVPlanWithVPRecipes(
8193 std::unique_ptr<VPlan>(VPlan0->duplicate()), SubRange, &LVer)) {
8196 *Plan, CM.getMinimalBitwidths());
8199 if (CM.foldTailWithEVL())
8201 *Plan, CM.getMaxSafeElements());
8204 VPlans.push_back(std::move(
P));
8207 VPlans.push_back(std::move(Plan));
8213VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8216 using namespace llvm::VPlanPatternMatch;
8217 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
8224 bool RequiresScalarEpilogueCheck =
8226 [
this](ElementCount VF) {
8227 return !CM.requiresScalarEpilogue(VF.
isVector());
8232 CM.foldTailByMasking());
8240 bool IVUpdateMayOverflow =
false;
8241 for (ElementCount VF :
Range)
8249 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8250 bool HasNUW = !IVUpdateMayOverflow ||
Style == TailFoldingStyle::None;
8255 m_VPInstruction<Instruction::Add>(
8257 "Did not find the canonical IV increment");
8270 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
8271 auto ApplyIG = [IG,
this](ElementCount VF) ->
bool {
8273 CM.getWideningDecision(IG->getInsertPos(), VF) ==
8278 "Unsupported interleave factor for scalable vectors");
8281 if (!getDecisionAndClampRange(ApplyIG,
Range))
8283 InterleaveGroups.
insert(IG);
8290 *Plan, CM.foldTailByMasking());
8296 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &
TTI,
Legal, CM, PSE,
8297 Builder, BlockMaskCache, LVer);
8298 RecipeBuilder.collectScaledReductions(
Range);
8303 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
8306 auto *MiddleVPBB = Plan->getMiddleBlock();
8310 DenseMap<VPValue *, VPValue *> Old2New;
8315 auto *UnderlyingValue = SingleDef->getUnderlyingValue();
8329 UnderlyingValue &&
"unsupported recipe");
8334 Builder.setInsertPoint(SingleDef);
8341 Legal->isInvariantAddressOfReduction(
SI->getPointerOperand())) {
8343 if (
Legal->isInvariantStoreOfReduction(SI)) {
8345 new VPReplicateRecipe(SI,
R.operands(),
true ,
8346 nullptr , VPIRMetadata(*SI, LVer));
8347 Recipe->insertBefore(*MiddleVPBB, MBIP);
8349 R.eraseFromParent();
8353 VPRecipeBase *Recipe =
8354 RecipeBuilder.tryToCreateWidenRecipe(SingleDef,
Range);
8356 Recipe = RecipeBuilder.handleReplication(Instr,
R.operands(),
Range);
8358 RecipeBuilder.setRecipe(Instr, Recipe);
8364 Builder.insert(Recipe);
8371 "Unexpected multidef recipe");
8372 R.eraseFromParent();
8381 RecipeBuilder.updateBlockMaskCache(Old2New);
8382 for (VPValue *Old : Old2New.
keys())
8383 Old->getDefiningRecipe()->eraseFromParent();
8387 "entry block must be set to a VPRegionBlock having a non-empty entry "
8393 for (
const auto &[Phi,
ID] :
Legal->getInductionVars()) {
8395 Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
8398 VPWidenInductionRecipe *WideIV =
8400 VPRecipeBase *
R = RecipeBuilder.getRecipe(IVInc);
8407 DenseMap<VPValue *, VPValue *> IVEndValues;
8416 adjustRecipesForReductions(Plan, RecipeBuilder,
Range.Start);
8428 if (!CM.foldTailWithEVL()) {
8429 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind,
8435 for (ElementCount VF :
Range)
8437 Plan->setName(
"Initial VPlan");
8443 InterleaveGroups, RecipeBuilder,
8444 CM.isScalarEpilogueAllowed());
8448 Legal->getLAI()->getSymbolicStrides());
8450 auto BlockNeedsPredication = [
this](
BasicBlock *BB) {
8451 return Legal->blockNeedsPredication(BB);
8454 BlockNeedsPredication);
8466 bool WithoutRuntimeCheck =
8467 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
8469 WithoutRuntimeCheck);
8477VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &
Range) {
8482 assert(!OrigLoop->isInnermost());
8486 OrigLoop, *LI,
Legal->getWidestInductionType(),
8495 for (ElementCount VF :
Range)
8500 [
this](PHINode *
P) {
8501 return Legal->getIntOrFpInductionDescriptor(
P);
8508 DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
8509 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &
TTI,
Legal, CM, PSE,
8510 Builder, BlockMaskCache,
nullptr );
8511 for (
auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8515 RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
8517 DenseMap<VPValue *, VPValue *> IVEndValues;
8541void LoopVectorizationPlanner::adjustRecipesForReductions(
8542 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
8543 using namespace VPlanPatternMatch;
8544 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
8546 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
8549 for (VPRecipeBase &R : Header->phis()) {
8551 if (!PhiR || !PhiR->isInLoop() || (MinVF.
isScalar() && !PhiR->isOrdered()))
8558 "AnyOf and FindIV reductions are not allowed for in-loop reductions");
8561 SetVector<VPSingleDefRecipe *> Worklist;
8563 for (
unsigned I = 0;
I != Worklist.
size(); ++
I) {
8564 VPSingleDefRecipe *Cur = Worklist[
I];
8565 for (VPUser *U : Cur->
users()) {
8567 if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
8568 assert((UserRecipe->getParent() == MiddleVPBB ||
8569 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
8570 "U must be either in the loop region, the middle block or the "
8571 "scalar preheader.");
8574 Worklist.
insert(UserRecipe);
8585 VPSingleDefRecipe *PreviousLink = PhiR;
8586 for (VPSingleDefRecipe *CurrentLink :
drop_begin(Worklist)) {
8588 assert(Blend->getNumIncomingValues() == 2 &&
8589 "Blend must have 2 incoming values");
8590 if (Blend->getIncomingValue(0) == PhiR) {
8591 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
8593 assert(Blend->getIncomingValue(1) == PhiR &&
8594 "PhiR must be an operand of the blend");
8595 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
8600 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
8603 unsigned IndexOfFirstOperand;
8605 bool IsFMulAdd = (
Kind == RecurKind::FMulAdd);
8607 VPBasicBlock *LinkVPBB = CurrentLink->getParent();
8611 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
8614 CurrentLink->getOperand(2) == PreviousLink &&
8615 "expected a call where the previous link is the added operand");
8621 VPInstruction *FMulRecipe =
new VPInstruction(
8623 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
8625 LinkVPBB->
insert(FMulRecipe, CurrentLink->getIterator());
8627 }
else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
8628 CurrentLinkI->
getOpcode() == Instruction::Sub) {
8629 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8630 auto *
Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
8631 VPWidenRecipe *
Sub =
new VPWidenRecipe(
8632 Instruction::Sub, {
Zero, CurrentLink->getOperand(1)}, {},
8634 Sub->setUnderlyingValue(CurrentLinkI);
8635 LinkVPBB->
insert(
Sub, CurrentLink->getIterator());
8641 "need to have the compare of the select");
8645 "must be a select recipe");
8646 IndexOfFirstOperand = 1;
8649 "Expected to replace a VPWidenSC");
8650 IndexOfFirstOperand = 0;
8655 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
8656 ? IndexOfFirstOperand + 1
8657 : IndexOfFirstOperand;
8658 VecOp = CurrentLink->getOperand(VecOpId);
8659 assert(VecOp != PreviousLink &&
8660 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
8661 (VecOpId - IndexOfFirstOperand)) ==
8663 "PreviousLink must be the operand other than VecOp");
8666 VPValue *CondOp =
nullptr;
8667 if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->
getParent()))
8671 RecurrenceDescriptor RdxDesc =
Legal->getRecurrenceDescriptor(
8677 auto *RedRecipe =
new VPReductionRecipe(
8678 Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
8685 RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->
end())));
8689 CurrentLink->replaceAllUsesWith(RedRecipe);
8691 PreviousLink = RedRecipe;
8695 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->
end())));
8697 for (VPRecipeBase &R :
8698 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8703 const RecurrenceDescriptor &RdxDesc =
Legal->getRecurrenceDescriptor(
8714 if (!PhiR->
isInLoop() && CM.foldTailByMasking() &&
8717 std::optional<FastMathFlags> FMFs =
8722 Builder.createSelect(
Cond, OrigExitingVPV, PhiR, {},
"", FMFs);
8723 OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U,
unsigned) {
8732 if (CM.usePredicatedReductionSelect())
8743 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
8749 VPInstruction *FinalReductionResult;
8750 VPBuilder::InsertPointGuard Guard(Builder);
8751 Builder.setInsertPoint(MiddleVPBB, IP);
8756 FinalReductionResult =
8761 FinalReductionResult =
8763 {PhiR,
Start, NewExitingVPV}, ExitDL);
8769 FinalReductionResult =
8771 {PhiR, NewExitingVPV},
Flags, ExitDL);
8778 assert(!PhiR->
isInLoop() &&
"Unexpected truncated inloop reduction!");
8780 "Unexpected truncated min-max recurrence!");
8782 VPWidenCastRecipe *Trunc;
8784 RdxDesc.
isSigned() ? Instruction::SExt : Instruction::ZExt;
8785 VPWidenCastRecipe *Extnd;
8787 VPBuilder::InsertPointGuard Guard(Builder);
8788 Builder.setInsertPoint(
8789 NewExitingVPV->getDefiningRecipe()->getParent(),
8790 std::next(NewExitingVPV->getDefiningRecipe()->getIterator()));
8792 Builder.createWidenCast(Instruction::Trunc, NewExitingVPV, RdxTy);
8793 Extnd = Builder.createWidenCast(ExtendOpc, Trunc, PhiTy);
8801 FinalReductionResult =
8802 Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
8807 for (
auto *U :
to_vector(OrigExitingVPV->users())) {
8809 if (FinalReductionResult == U || Parent->getParent())
8811 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
8822 return isa<VPWidenSelectRecipe>(U) ||
8823 (isa<VPReplicateRecipe>(U) &&
8824 cast<VPReplicateRecipe>(U)->getUnderlyingInstr()->getOpcode() ==
8825 Instruction::Select);
8830 if (VPRecipeBase *CmpR =
Cmp->getDefiningRecipe())
8832 Builder.setInsertPoint(
Select);
8836 if (
Select->getOperand(1) == PhiR)
8837 Cmp = Builder.createNot(Cmp);
8838 VPValue *
Or = Builder.createOr(PhiR, Cmp);
8839 Select->getVPSingleValue()->replaceAllUsesWith(
Or);
8845 OrigLoop->getHeader()->getContext())));
8860 VPBuilder PHBuilder(Plan->getVectorPreheader());
8861 VPValue *Iden = Plan->getOrAddLiveIn(
8864 unsigned ScaleFactor =
8868 auto *ScaleFactorVPV =
8869 Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
8870 VPValue *StartV = PHBuilder.createNaryOp(
8878 for (VPRecipeBase *R : ToDelete)
8879 R->eraseFromParent();
8884void LoopVectorizationPlanner::attachRuntimeChecks(
8885 VPlan &Plan, GeneratedRTChecks &RTChecks,
bool HasBranchWeights)
const {
8886 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
8887 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
8888 assert((!CM.OptForSize ||
8890 "Cannot SCEV check stride or overflow when optimizing for size");
8894 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
8895 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
8899 "Runtime checks are not supported for outer loops yet");
8901 if (CM.OptForSize) {
8904 "Cannot emit memory checks when optimizing for size, unless forced "
8907 return OptimizationRemarkAnalysis(
DEBUG_TYPE,
"VectorizationCodeSize",
8908 OrigLoop->getStartLoc(),
8909 OrigLoop->getHeader())
8910 <<
"Code-size may be reduced by not forcing "
8911 "vectorization, or by source-code modifications "
8912 "eliminating the need for runtime checks "
8913 "(e.g., adding 'restrict').";
8927 bool IsIndvarOverflowCheckNeededForVF =
8928 VF.
isScalable() && !TTI.isVScaleKnownToBeAPowerOfTwo() &&
8930 CM.getTailFoldingStyle() !=
8937 Plan, VF, UF, MinProfitableTripCount,
8938 CM.requiresScalarEpilogue(VF.
isVector()), CM.foldTailByMasking(),
8939 IsIndvarOverflowCheckNeededForVF, OrigLoop, BranchWeigths,
8940 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
8945 assert(!State.Lane &&
"VPDerivedIVRecipe being replicated.");
8950 State.Builder.setFastMathFlags(FPBinOp->getFastMathFlags());
8958 State.set(
this, DerivedIV,
VPLane(0));
9004 if (
TTI->preferPredicateOverEpilogue(&TFI))
9023 LLVM_DEBUG(
dbgs() <<
"LV: cannot compute the outer-loop trip count\n");
9027 Function *
F = L->getHeader()->getParent();
9033 LoopVectorizationCostModel CM(
SEL, L, PSE, LI, LVL, *
TTI, TLI, DB, AC, ORE,
F,
9034 &Hints, IAI, PSI, BFI);
9038 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *
TTI, LVL, CM, IAI, PSE, Hints,
9058 GeneratedRTChecks Checks(PSE, DT, LI,
TTI,
F->getDataLayout(), CM.
CostKind);
9060 BFI, PSI, Checks, BestPlan);
9062 << L->getHeader()->getParent()->getName() <<
"\"\n");
9084 if (S->getValueOperand()->getType()->isFloatTy())
9094 while (!Worklist.
empty()) {
9096 if (!L->contains(
I))
9098 if (!Visited.
insert(
I).second)
9108 I->getDebugLoc(), L->getHeader())
9109 <<
"floating point conversion changes vector width. "
9110 <<
"Mixed floating point precision requires an up/down "
9111 <<
"cast that will negatively impact performance.";
9114 for (
Use &
Op :
I->operands())
9130 for (
auto *PredVPBB : ExitVPBB->getPredecessors()) {
9136 << PredVPBB->getName() <<
":\n");
9137 Cost += PredVPBB->cost(VF, CostCtx);
9156 std::optional<unsigned> VScale) {
9172 <<
"LV: Interleaving only is not profitable due to runtime checks\n");
9231 uint64_t MinTC = std::max(MinTC1, MinTC2);
9233 MinTC =
alignTo(MinTC, IntVF);
9237 dbgs() <<
"LV: Minimum required TC for runtime checks to be profitable:"
9244 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is not beneficial: expected "
9245 "trip count < minimum profitable VF ("
9256 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9258 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9279 if (EpiWidenedPhis.
contains(&VPIRInst->getIRPhi()))
9298 auto AddFreezeForFindLastIVReductions = [](
VPlan &Plan,
9299 bool UpdateResumePhis) {
9305 VPValue *OrigStart = VPI->getOperand(1);
9309 Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {},
"fr");
9311 if (UpdateResumePhis)
9317 AddFreezeForFindLastIVReductions(MainPlan,
true);
9318 AddFreezeForFindLastIVReductions(EpiPlan,
false);
9325 auto ResumePhiIter =
9327 return match(&R, m_VPInstruction<Instruction::PHI>(m_Specific(VectorTC),
9330 VPPhi *ResumePhi =
nullptr;
9331 if (ResumePhiIter == MainScalarPH->
phis().
end()) {
9336 {},
"vec.epilog.resume.val");
9339 if (MainScalarPH->
begin() == MainScalarPH->
end())
9341 else if (&*MainScalarPH->
begin() != ResumePhi)
9356 VPlan &Plan,
Loop *L,
const SCEV2ValueTy &ExpandedSCEVs,
9361 Header->
setName(
"vec.epilog.vector.body");
9372 PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
9377 "Must only have a single non-zero incoming value");
9388 [](
Value *Inc) { return match(Inc, m_SpecificInt(0)); }) &&
9389 "all incoming values must be 0");
9395 return isa<VPScalarIVStepsRecipe>(U) ||
9396 isa<VPDerivedIVRecipe>(U) ||
9397 cast<VPRecipeBase>(U)->isScalarCast() ||
9398 cast<VPInstruction>(U)->getOpcode() ==
9401 "the canonical IV should only be used by its increment or "
9402 "ScalarIVSteps when resetting the start value");
9403 VPBuilder Builder(Header, Header->getFirstNonPhi());
9405 IV->replaceAllUsesWith(
Add);
9406 Add->setOperand(0,
IV);
9414 Value *ResumeV =
nullptr;
9419 auto *VPI = dyn_cast<VPInstruction>(U);
9421 (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
9422 VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
9423 VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
9426 ->getIncomingValueForBlock(L->getLoopPreheader());
9427 RecurKind RK = ReductionPhi->getRecurrenceKind();
9435 ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
9440 ToFrozen[StartV] =
cast<PHINode>(ResumeV)->getIncomingValueForBlock(
9451 Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
9454 Value *
Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
9455 ResumeV = Builder.CreateSelect(Cmp,
Sentinel, ResumeV);
9463 "unexpected start value");
9464 VPI->setOperand(0, StartVal);
9476 assert(ResumeV &&
"Must have a resume value");
9490 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
9492 ToFrozen.
lookup(VPI->getOperand(0)->getLiveInIRValue())));
9507 ExpandR->eraseFromParent();
9511 unsigned MainLoopStep =
9513 unsigned EpilogueLoopStep =
9518 EPI.
EpilogueUF, MainLoopStep, EpilogueLoopStep, SE);
9529 const SCEV2ValueTy &ExpandedSCEVs,
Value *MainVectorTripCount,
9534 Value *EndValueFromAdditionalBypass = MainVectorTripCount;
9535 if (OrigPhi != OldInduction) {
9536 auto *BinOp =
II.getInductionBinOp();
9542 EndValueFromAdditionalBypass =
9544 II.getStartValue(), Step,
II.getKind(), BinOp);
9545 EndValueFromAdditionalBypass->
setName(
"ind.end");
9547 return EndValueFromAdditionalBypass;
9553 const SCEV2ValueTy &ExpandedSCEVs,
9554 Value *MainVectorTripCount) {
9559 if (Phi.getBasicBlockIndex(Pred) != -1)
9561 Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
9565 if (ScalarPH->hasPredecessors()) {
9568 for (
const auto &[R, IRPhi] :
9569 zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
9578 auto *Inc =
cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
9580 IVPhi,
II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
9583 Inc->setIncomingValueForBlock(BypassBlock, V);
9606 "expected this to be saved from the previous pass.");
9609 VecEpilogueIterationCountCheck, VecEpiloguePreHeader);
9612 VecEpilogueIterationCountCheck},
9614 VecEpiloguePreHeader}});
9619 VecEpilogueIterationCountCheck, ScalarPH);
9622 VecEpilogueIterationCountCheck},
9626 BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
9627 BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
9628 if (SCEVCheckBlock) {
9630 VecEpilogueIterationCountCheck, ScalarPH);
9632 VecEpilogueIterationCountCheck},
9635 if (MemCheckBlock) {
9637 VecEpilogueIterationCountCheck, ScalarPH);
9650 for (
PHINode *Phi : PhisInBlock) {
9652 Phi->replaceIncomingBlockWith(
9654 VecEpilogueIterationCountCheck);
9661 return EPI.EpilogueIterationCountCheck == IncB;
9666 Phi->removeIncomingValue(SCEVCheckBlock);
9668 Phi->removeIncomingValue(MemCheckBlock);
9672 for (
auto *
I : InstsToMove)
9684 "VPlan-native path is not enabled. Only process inner loops.");
9687 << L->getHeader()->getParent()->getName() <<
"' from "
9688 << L->getLocStr() <<
"\n");
9693 dbgs() <<
"LV: Loop hints:"
9704 Function *
F = L->getHeader()->getParent();
9726 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Cannot prove legality.\n");
9733 "early exit is not enabled",
9734 "UncountableEarlyExitLoopsDisabled",
ORE, L);
9740 "faulting load is not supported",
9741 "PotentiallyFaultingLoadsNotSupported",
ORE, L);
9750 if (!L->isInnermost())
9754 assert(L->isInnermost() &&
"Inner loop expected.");
9757 bool UseInterleaved =
TTI->enableInterleavedAccessVectorization();
9771 [LoopLatch](
BasicBlock *BB) { return BB != LoopLatch; })) {
9773 "requiring a scalar epilogue is unsupported",
9774 "UncountableEarlyExitUnsupported",
ORE, L);
9787 if (ExpectedTC && ExpectedTC->isFixed() &&
9789 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop with a very small trip count. "
9790 <<
"This loop is worth vectorizing only if no scalar "
9791 <<
"iteration overheads are incurred.");
9793 LLVM_DEBUG(
dbgs() <<
" But vectorizing was explicitly forced.\n");
9809 if (
F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9811 "Can't vectorize when the NoImplicitFloat attribute is used",
9812 "loop not vectorized due to NoImplicitFloat attribute",
9813 "NoImplicitFloat",
ORE, L);
9823 TTI->isFPVectorizationPotentiallyUnsafe()) {
9825 "Potentially unsafe FP op prevents vectorization",
9826 "loop not vectorized due to unsafe FP support.",
9827 "UnsafeFP",
ORE, L);
9832 bool AllowOrderedReductions;
9837 AllowOrderedReductions =
TTI->enableOrderedReductions();
9842 ExactFPMathInst->getDebugLoc(),
9843 ExactFPMathInst->getParent())
9844 <<
"loop not vectorized: cannot prove it is safe to reorder "
9845 "floating-point operations";
9847 LLVM_DEBUG(
dbgs() <<
"LV: loop not vectorized: cannot prove it is safe to "
9848 "reorder floating-point operations\n");
9854 LoopVectorizationCostModel CM(
SEL, L, PSE,
LI, &LVL, *
TTI,
TLI,
DB,
AC,
ORE,
9857 LoopVectorizationPlanner LVP(L,
LI,
DT,
TLI, *
TTI, &LVL, CM, IAI, PSE, Hints,
9865 LVP.
plan(UserVF, UserIC);
9872 GeneratedRTChecks Checks(PSE,
DT,
LI,
TTI,
F->getDataLayout(), CM.
CostKind);
9877 unsigned SelectedIC = std::max(IC, UserIC);
9886 if (Checks.getSCEVChecks().first &&
9887 match(Checks.getSCEVChecks().first,
m_One()))
9889 if (Checks.getMemRuntimeChecks().first &&
9890 match(Checks.getMemRuntimeChecks().first,
m_One()))
9895 bool ForceVectorization =
9899 if (!ForceVectorization &&
9905 DEBUG_TYPE,
"CantReorderMemOps", L->getStartLoc(),
9907 <<
"loop not vectorized: cannot prove it is safe to reorder "
9908 "memory operations";
9917 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
9918 bool VectorizeLoop =
true, InterleaveLoop =
true;
9920 LLVM_DEBUG(
dbgs() <<
"LV: Vectorization is possible but not beneficial.\n");
9922 "VectorizationNotBeneficial",
9923 "the cost-model indicates that vectorization is not beneficial"};
9924 VectorizeLoop =
false;
9930 LLVM_DEBUG(
dbgs() <<
"LV: Ignoring UserIC, because vectorization and "
9931 "interleaving should be avoided up front\n");
9932 IntDiagMsg = {
"InterleavingAvoided",
9933 "Ignoring UserIC, because interleaving was avoided up front"};
9934 InterleaveLoop =
false;
9935 }
else if (IC == 1 && UserIC <= 1) {
9939 "InterleavingNotBeneficial",
9940 "the cost-model indicates that interleaving is not beneficial"};
9941 InterleaveLoop =
false;
9943 IntDiagMsg.first =
"InterleavingNotBeneficialAndDisabled";
9944 IntDiagMsg.second +=
9945 " and is explicitly disabled or interleave count is set to 1";
9947 }
else if (IC > 1 && UserIC == 1) {
9949 LLVM_DEBUG(
dbgs() <<
"LV: Interleaving is beneficial but is explicitly "
9951 IntDiagMsg = {
"InterleavingBeneficialButDisabled",
9952 "the cost-model indicates that interleaving is beneficial "
9953 "but is explicitly disabled or interleave count is set to 1"};
9954 InterleaveLoop =
false;
9960 if (!VectorizeLoop && InterleaveLoop && LVL.
hasHistograms()) {
9961 LLVM_DEBUG(
dbgs() <<
"LV: Not interleaving without vectorization due "
9962 <<
"to histogram operations.\n");
9964 "HistogramPreventsScalarInterleaving",
9965 "Unable to interleave without vectorization due to constraints on "
9966 "the order of histogram operations"};
9967 InterleaveLoop =
false;
9971 IC = UserIC > 0 ? UserIC : IC;
9975 if (!VectorizeLoop && !InterleaveLoop) {
9979 L->getStartLoc(), L->getHeader())
9980 << VecDiagMsg.second;
9984 L->getStartLoc(), L->getHeader())
9985 << IntDiagMsg.second;
9990 if (!VectorizeLoop && InterleaveLoop) {
9994 L->getStartLoc(), L->getHeader())
9995 << VecDiagMsg.second;
9997 }
else if (VectorizeLoop && !InterleaveLoop) {
9999 <<
") in " << L->getLocStr() <<
'\n');
10002 L->getStartLoc(), L->getHeader())
10003 << IntDiagMsg.second;
10005 }
else if (VectorizeLoop && InterleaveLoop) {
10007 <<
") in " << L->getLocStr() <<
'\n');
10013 using namespace ore;
10018 <<
"interleaved loop (interleaved count: "
10019 << NV(
"InterleaveCount", IC) <<
")";
10036 std::unique_ptr<VPlan> BestMainPlan(BestPlan.
duplicate());
10048 PSI, Checks, *BestMainPlan);
10050 *BestMainPlan, MainILV,
DT,
false);
10056 BFI,
PSI, Checks, BestEpiPlan);
10058 BestEpiPlan, L, ExpandedSCEVs, EPI, CM, *PSE.
getSE());
10062 Checks, InstsToMove);
10063 ++LoopsEpilogueVectorized;
10065 InnerLoopVectorizer LB(L, PSE,
LI,
DT,
TTI,
AC, VF.
Width, IC, &CM,
BFI,
PSI,
10079 assert(
DT->verify(DominatorTree::VerificationLevel::Fast) &&
10080 "DT not preserved correctly");
10095 if (!
TTI->getNumberOfRegisters(
TTI->getRegisterClassForType(
true)) &&
10099 bool Changed =
false, CFGChanged =
false;
10106 for (
const auto &L : *
LI)
10118 LoopsAnalyzed += Worklist.
size();
10121 while (!Worklist.
empty()) {
10164 if (
PSI &&
PSI->hasProfileSummary())
10167 if (!Result.MadeAnyChange)
10181 if (Result.MadeCFGChange) {
10197 OS, MapClassName2PassName);
10200 OS << (InterleaveOnlyWhenForced ?
"" :
"no-") <<
"interleave-forced-only;";
10201 OS << (VectorizeOnlyWhenForced ?
"" :
"no-") <<
"vectorize-forced-only;";
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static unsigned getIntrinsicID(const SDNode *N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< OutputCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(OutputCostKind::RecipThroughput), cl::values(clEnumValN(OutputCostKind::RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(OutputCostKind::Latency, "latency", "Instruction latency"), clEnumValN(OutputCostKind::CodeSize, "code-size", "Code size"), clEnumValN(OutputCostKind::SizeAndLatency, "size-latency", "Code size and latency"), clEnumValN(OutputCostKind::All, "all", "Print all cost kinds")))
static cl::opt< IntrinsicCostStrategy > IntrinsicCost("intrinsic-cost-strategy", cl::desc("Costing strategy for intrinsic instructions"), cl::init(IntrinsicCostStrategy::InstructionCost), cl::values(clEnumValN(IntrinsicCostStrategy::InstructionCost, "instruction-cost", "Use TargetTransformInfo::getInstructionCost"), clEnumValN(IntrinsicCostStrategy::IntrinsicCost, "intrinsic-cost", "Use TargetTransformInfo::getIntrinsicInstrCost"), clEnumValN(IntrinsicCostStrategy::TypeBasedIntrinsicCost, "type-based-intrinsic-cost", "Calculate the intrinsic cost based only on argument types")))
static InstructionCost getCost(Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI, TargetLibraryInfo &TLI)
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This is the interface for a simple mod/ref and alias analysis over globals.
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Legalize the Machine IR a function s Machine IR
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static void legacyCSE(BasicBlock *BB)
FIXME: This legacy common-subexpression-elimination routine is scheduled for removal,...
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static Value * createInductionAdditionalBypassValues(PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, Instruction *OldInduction)
static void fixReductionScalarResumeWhenVectorizingEpilog(VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock)
static Value * getStartValueFromReductionResult(VPInstruction *RdxResult)
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool > ConsiderRegPressure("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high."))
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L, EpilogueLoopVectorizationInfo &EPI, DominatorTree *DT, LoopVectorizationLegality &LVL, DenseMap< const SCEV *, Value * > &ExpandedSCEVs, GeneratedRTChecks &Checks, ArrayRef< Instruction * > InstsToMove)
Connect the epilogue vector loop generated for EpiPlan to the main vector.
static bool planContainsAdditionalSimplifications(VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop, ElementCount VF)
Return true if the original loop \ TheLoop contains any instructions that do not have corresponding r...
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static SmallVector< Instruction * > preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel &CM, ScalarEvolution &SE)
Prepare Plan for vectorizing the epilogue loop.
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static std::optional< ElementCount > getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L, VPlan &BestEpiPlan, LoopVectorizationLegality &LVL, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
Create an analysis remark that explains why vectorization failed.
This file implements a map that provides insertion order iteration.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
static InstructionCost getScalarizationOverhead(const TargetTransformInfo &TTI, Type *ScalarTy, VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={})
This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorTyp...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
static const uint32_t IV[8]
A manager for alias analyses.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
uint64_t getZExtValue() const
Get zero extended value.
unsigned getActiveBits() const
Compute the number of active bits in the value.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & back() const
back - Get the last element.
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
size_t size() const
size - Get the array size.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BinaryOps getOpcode() const
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Represents analyses that only rely on functions' control flow.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
This is the shared class of boolean and integer constants.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A parsed version of the target data layout string in and methods for querying it.
static DebugLoc getTemporary()
static DebugLoc getUnknown()
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
void insert_range(Range &&R)
Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
static constexpr UpdateKind Delete
static constexpr UpdateKind Insert
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
void printDebugTracesAtEnd() override
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (i....
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB)
Introduces a new VPIRBasicBlock for CheckIRBB to Plan between the vector preheader and its predecesso...
BasicBlock * emitIterationCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
void printDebugTracesAtEnd() override
Value * createIterationCountCheck(BasicBlock *VectorPH, ElementCount VF, unsigned UF) const
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the main loop strategy (i....
Convenience struct for specifying and reasoning about fast-math flags.
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
Common base class shared among various IRBuilders.
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
ElementCount MinProfitableTripCount
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor)
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
Value * TripCount
Trip count of the original loop.
const TargetTransformInfo * TTI
Target Transform Info.
LoopVectorizationCostModel * Cost
The profitablity analysis.
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
friend class LoopVectorizationPlanner
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
void fixVectorizedLoop(VPTransformState &State)
Fix the vectorized code, taking care of header phi's, and more.
virtual BasicBlock * createVectorizedLoopSkeleton()
Creates a basic block for the scalar preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks, VPlan &Plan)
IRBuilder Builder
The builder that we use.
void fixNonInductionPHIs(VPTransformState &State)
Fix the non-induction PHIs in Plan.
VPBasicBlock * VectorPHVPBB
The vector preheader block of Plan, used as target for check blocks introduced during skeleton creati...
unsigned UF
The vectorization unroll factor to use.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
BasicBlock * createScalarPreheader(StringRef Prefix)
Create and return a new IR basic block for the scalar preheader whose name is prefixed with Prefix.
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getHeader() const
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
bool isEpilogueVectorizationProfitable(const ElementCount VF, const unsigned IC) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
bool hasPredStores() const
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
std::optional< unsigned > getMaxSafeElements() const
Return maximum safe number of elements to be processed per vector iteration, which do not prevent sto...
const TargetTransformInfo & TTI
Vector target information.
friend class LoopVectorizationPlanner
const Function * TheFunction
LoopVectorizationLegality * Legal
Vectorization legality.
std::optional< InstructionCost > getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy) const
Return the cost of instructions in an inloop reduction pattern, if I is part of that pattern.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
DemandedBits * DB
Demanded bits analysis.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool OptForSize
Whether this loop should be optimized for size based on function attribute or profile information.
bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind)
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
bool shouldConsiderRegPressureForVF(ElementCount VF)
Loop * TheLoop
The loop that we evaluate.
TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
std::optional< unsigned > getVScaleForTuning() const
Return the value of vscale used for tuning the cost model.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool runtimeChecksRequired()
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool usePredicatedReductionSelect() const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
FixedScalableVFPair MaxPermissibleVFWithoutMaxBW
The highest VF possible for this loop, without using MaxBandwidth.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
const SmallPtrSetImpl< const Instruction * > & getPotentiallyFaultingLoads() const
Returns potentially faulting loads.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool hasUncountableEarlyExit() const
Returns true if the loop has exactly one uncountable early exit, i.e.
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
Planner drives the vectorization process after having passed Legality checks.
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
VectorizationFactor computeBestVF()
Compute and return the most profitable vectorization factor.
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool VectorizingEpilogue)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
void printPlans(raw_ostream &O)
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Instruction * getExactFPInst()
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
enum ForceKind getForce() const
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
void emitRemarkWithHints() const
Dumps all the hint information.
bool isPotentiallyUnsafe() const
ElementCount getWidth() const
@ FK_Enabled
Forcing enabled.
@ FK_Undefined
Not selected.
@ FK_Disabled
Forcing disabled.
unsigned getPredicate() const
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
unsigned getInterleave() const
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
bool hasLoopInvariantOperands(const Instruction *I) const
Return true if all the operands of the specified instruction are loop invariant.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
This class implements a map that also provides access to all stored values in a deterministic order.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
void setIncomingValueForBlock(const BasicBlock *BB, Value *V)
Set every incoming value(s) for block BB to V.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
unsigned getNumIncomingValues() const
Return the number of incoming edges.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
LLVM_ABI SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
static LLVM_ABI bool isFloatingPointRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is a floating point kind.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Value * getSentinelValue() const
Returns the sentinel value for FindFirstIV & FindLastIV recurrences to replace the start value.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
void insert_range(Range &&R)
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
A Use represents the edge between a Value definition and its users.
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Value * getOperand(unsigned i) const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::iterator iterator
Instruction iterators...
iterator begin()
Recipe iterator methods.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
void setName(const Twine &newName)
size_t getNumSuccessors() const
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
size_t getNumPredecessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
VPlan-based builder utility analogous to IRBuilder.
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
Canonical scalar induction phi of the vector loop.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
VPValue * getStartValue() const
A special type of VPBasicBlock that wraps an existing IR basic block.
This is a concrete Recipe that models a single VPlan-level instruction.
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
@ FirstOrderRecurrenceSplice
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
unsigned getOpcode() const
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
A recipe for forming partial reductions.
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
VPBasicBlock * getParent()
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRecipeBase * tryToCreateWidenRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for R if one can be created within the given VF Range.
VPValue * getBlockInMask(VPBasicBlock *VPBB) const
Returns the entry mask for block VPBB or null if the mask is all-true.
std::optional< unsigned > getScalingForReduction(const Instruction *ExitInst)
void collectScaledReductions(VFRange &Range)
Find all possible partial reductions in the loop and track all of those that are valid so recipes can...
VPReplicateRecipe * handleReplication(Instruction *I, ArrayRef< VPValue * > Operands, VFRange &Range)
Build a VPReplicationRecipe for I using Operands.
VPRecipeBase * tryToCreatePartialReduction(Instruction *Reduction, ArrayRef< VPValue * > Operands, unsigned ScaleFactor)
Create and return a partial reduction recipe for a reduction instruction along with binary operation ...
A recipe for handling reduction phis.
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the region.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
VPValue * getOperand(unsigned N) const
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceAllUsesWith(VPValue *New)
user_iterator user_begin()
unsigned getNumUsers() const
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A recipe to compute a pointer to the last element of each part of a widened memory access for widened...
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
VPValue * getStepValue()
Returns the step value of the induction.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A common base class for widening memory operations.
A recipe for widened phis.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
bool hasVF(ElementCount VF) const
VPBasicBlock * getEntry()
VPValue & getVectorTripCount()
The vector trip count.
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
VPValue & getVF()
Returns the VF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
bool hasUF(unsigned UF) const
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isNonZero() const
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ PredicateElseScalarEpilogue
@ PredicateOrDontVectorize
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
specificloop_ty m_SpecificLoop(const Loop *L)
cst_pred_ty< is_specific_signed_cst > m_scev_SpecificSInt(int64_t V)
Match an SCEV constant with a plain signed integer (sign-extended value will be matched)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
bind_ty< const SCEVMulExpr > m_scev_Mul(const SCEVMulExpr *&V)
bool match(const SCEV *S, const Pattern &P)
class_match< const SCEV > m_SCEV()
match_combine_or< AllRecipe_match< Instruction::ZExt, Op0_t >, AllRecipe_match< Instruction::SExt, Op0_t > > m_ZExtOrSExt(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ExtractLastElement, Op0_t > m_ExtractLastElement(const Op0_t &Op0)
class_match< VPValue > m_VPValue()
Match an arbitrary VPValue and ignore it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
NodeAddr< PhiNode * > Phi
friend class Instruction
Iterator for Instructions in a `BasicBlock.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr)
Get or create a VPValue that corresponds to the expansion of Expr.
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
unsigned getVFScaleFactor(VPRecipeBase *R)
Get the VF scaling factor applied to the recipe's output, if the recipe has one.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
FunctionAddr VTableAddr Value
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
auto cast_if_present(const Y &Val)
cast_if_present<X> - Functionally identical to cast, except that a null value is accepted.
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
cl::opt< bool > VerifyEachVPlan
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate=false)
Verify invariants for general VPlans.
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &ValuesToIgnore)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI cl::opt< bool > EnableLoopVectorization
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
FunctionAddr VTableAddr Count
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
bool canConstantBeExtended(const APInt *C, Type *NarrowType, TTI::PartialReductionExtendKind ExtKind)
Check if a constant CI can be safely treated as having been extended from a narrower type with the gi...
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
cl::opt< unsigned > ForceTargetInstructionCost
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ Sub
Subtraction of integers.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
LLVM_ABI void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
DWARFExpression::Operation Op
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
auto predecessors(const MachineBasicBlock *BB)
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
cl::opt< bool > EnableVPlanNativePath
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool pred_empty(const BasicBlock *BB)
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
unsigned getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind)
A helper function that returns how much we should divide the cost of a predicated block by.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
LLVM_ABI MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
LLVM_ABI cl::opt< bool > EnableLoopInterleaving
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
A special type used by analysis passes to provide an address that identifies that particular analysis...
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
Encapsulate information regarding vectorization of a loop and its epilogue.
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF, VPlan &EpiloguePlan)
BasicBlock * MainLoopIterationCountCheck
BasicBlock * EpilogueIterationCountCheck
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
std::optional< unsigned > MaskPos
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
LoopAccessInfoManager * LAIs
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
TargetTransformInfo * TTI
Storage for information about made changes.
A chain of instructions that form a partial reduction.
Instruction * Reduction
The top-level binary operation that forms the reduction to a scalar after the loop body.
Instruction * ExtendA
The extension of each of the inner binary operation's operands.
A CRTP mix-in to automatically provide informational APIs needed for passes.
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool isLegacyUniformAfterVectorization(Instruction *I, ElementCount VF) const
Return true if I is considered uniform-after-vectorization in the legacy cost model for VF.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
A struct that represents some properties of the register usage of a loop.
A recipe for widening select instructions.
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI bool HoistRuntimeChecks