LLVM 24.0.0git
LoopVectorize.cpp
Go to the documentation of this file.
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has four parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. LoopVectorizationPlanner - A unit that builds VPlans for the candidate
23// vectorization factors, selects the best one and executes it to generate
24// the vector code.
25// 4. LoopVectorizationCostModel - A unit that checks for the profitability
26// of vectorization. It decides on the optimal vector width, which
27// can be one, if vectorization is not profitable.
28//
29// There is a development effort going on to migrate loop vectorizer to the
30// VPlan infrastructure and to introduce outer loop vectorization support (see
31// docs/VectorizationPlan.rst and
32// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
33// purpose, we temporarily introduced the VPlan-native vectorization path: an
34// alternative vectorization path that is natively implemented on top of the
35// VPlan infrastructure. See EnableVPlanNativePath for enabling.
36//
37//===----------------------------------------------------------------------===//
38//
39// The reduction-variable vectorization is based on the paper:
40// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
41//
42// Variable uniformity checks are inspired by:
43// Karrenberg, R. and Hack, S. Whole Function Vectorization.
44//
45// The interleaved access vectorization is based on the paper:
46// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
47// Data for SIMD
48//
49// Other ideas/concepts are from:
50// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
51//
52// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
53// Vectorizing Compilers.
54//
55//===----------------------------------------------------------------------===//
56
59#include "VPRecipeBuilder.h"
60#include "VPlan.h"
61#include "VPlanAnalysis.h"
62#include "VPlanCFG.h"
63#include "VPlanHelpers.h"
64#include "VPlanPatternMatch.h"
65#include "VPlanTransforms.h"
66#include "VPlanUtils.h"
67#include "VPlanVerifier.h"
68#include "VectorizeOptions.h"
69#include "llvm/ADT/APInt.h"
70#include "llvm/ADT/ArrayRef.h"
71#include "llvm/ADT/DenseMap.h"
72#include "llvm/ADT/Hashing.h"
73#include "llvm/ADT/MapVector.h"
74#include "llvm/ADT/STLExtras.h"
77#include "llvm/ADT/Statistic.h"
78#include "llvm/ADT/StringRef.h"
79#include "llvm/ADT/Twine.h"
80#include "llvm/ADT/TypeSwitch.h"
86#include "llvm/Analysis/CFG.h"
104#include "llvm/IR/Attributes.h"
105#include "llvm/IR/BasicBlock.h"
106#include "llvm/IR/CFG.h"
107#include "llvm/IR/Constant.h"
108#include "llvm/IR/Constants.h"
109#include "llvm/IR/DataLayout.h"
110#include "llvm/IR/DebugInfo.h"
111#include "llvm/IR/DebugLoc.h"
112#include "llvm/IR/DerivedTypes.h"
114#include "llvm/IR/Dominators.h"
115#include "llvm/IR/Function.h"
116#include "llvm/IR/IRBuilder.h"
117#include "llvm/IR/InstrTypes.h"
118#include "llvm/IR/Instruction.h"
119#include "llvm/IR/Instructions.h"
121#include "llvm/IR/Intrinsics.h"
122#include "llvm/IR/MDBuilder.h"
123#include "llvm/IR/Metadata.h"
124#include "llvm/IR/Module.h"
125#include "llvm/IR/Operator.h"
126#include "llvm/IR/PatternMatch.h"
128#include "llvm/IR/Type.h"
129#include "llvm/IR/Use.h"
130#include "llvm/IR/User.h"
131#include "llvm/IR/Value.h"
132#include "llvm/IR/Verifier.h"
133#include "llvm/Support/Casting.h"
135#include "llvm/Support/Debug.h"
150#include <algorithm>
151#include <cassert>
152#include <cmath>
153#include <cstdint>
154#include <functional>
155#include <iterator>
156#include <memory>
157#include <string>
158#include <tuple>
159#include <utility>
160
161using namespace llvm;
162using namespace SCEVPatternMatch;
163using namespace LoopVectorizationUtils;
164
165#define LV_NAME "loop-vectorize"
166#define DEBUG_TYPE LV_NAME
167
168#ifndef NDEBUG
169const char VerboseDebug[] = DEBUG_TYPE "-verbose";
170#endif
171
172STATISTIC(LoopsVectorized, "Number of loops vectorized");
173STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
174STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
175STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
176STATISTIC(LoopsPartialAliasVectorized,
177 "Number of partial aliasing loops vectorized");
178
180 "epilogue-vectorization-force-VF", cl::init(ElementCount::getFixed(1)),
182 cl::desc("When epilogue vectorization is enabled, and a value greater than "
183 "1 is specified, forces the given VF for all applicable epilogue "
184 "loops. Note: This allows all scalable VFs >= vscale x 1."));
185
186// Returns true if the epilogue VF has been set to a non-zero value other than
187// VF=1 (scalar).
192
193// Likelyhood of bypassing the vectorized loop because there are zero trips left
194// after prolog. See `emitIterationCountCheck`.
195static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
196
197/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
198/// ElementCount to include loops whose trip count is a function of vscale.
200 const Loop *L) {
201 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
202 return ElementCount::getFixed(ExpectedTC);
203
204 const SCEV *BTC = SE->getBackedgeTakenCount(L);
206 return ElementCount::getFixed(0);
207
208 const SCEV *ExitCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
209 if (isa<SCEVVScale>(ExitCount))
211
212 const APInt *Scale;
213 if (match(ExitCount, m_scev_Mul(m_scev_APInt(Scale), m_SCEVVScale())))
214 if (cast<SCEVMulExpr>(ExitCount)->hasNoUnsignedWrap())
215 if (Scale->getActiveBits() <= 32)
217
218 return ElementCount::getFixed(0);
219}
220
221/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
222/// zero from the range. Only valid when not folding the tail, as the minimum
223/// iteration count check guards against a zero trip count. Returns 0 if
224/// unknown.
226 Loop *L) {
227 const SCEV *BTC = PSE.getBackedgeTakenCount();
229 return 0;
230 ScalarEvolution *SE = PSE.getSE();
231 const SCEV *TripCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
232 ConstantRange TCRange =
233 SE->getUnsignedRange(SE->applyLoopGuards(TripCount, L));
234 APInt MaxTCFromRange = TCRange.getUnsignedMax();
235 if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
236 return MaxTCFromRange.getZExtValue();
237 return 0;
238}
239
240/// Returns "best known" trip count, which is either a valid positive trip count
241/// or std::nullopt when an estimate cannot be made (including when the trip
242/// count would overflow), for the specified loop \p L as defined by the
243/// following procedure:
244/// 1) Returns exact trip count if it is known.
245/// 2) Returns expected trip count according to profile data if any.
246/// 3) Returns upper bound estimate if known, if \p CanUseConstantMax, and
247/// if \p ComputeUpperBoundOnly is false.
248/// 4) Returns the maximum trip count from the SCEV range excluding zero,
249/// if \p CanUseConstantMax and \p CanExcludeZeroTrips.
250/// 5) Returns std::nullopt if all of the above failed.
251static std::optional<ElementCount> getSmallBestKnownTC(
252 const VectorizeOptions &Opts, PredicatedScalarEvolution &PSE, Loop *L,
253 bool CanUseConstantMax = true, bool CanExcludeZeroTrips = false,
254 bool ComputeUpperBoundOnly = false) {
255 // Check if exact trip count is known.
256 if (auto ExpectedTC = getSmallConstantTripCount(PSE.getSE(), L))
257 return ExpectedTC;
258
259 // Check if there is an expected trip count available from profile data.
260 // An estimate of zero means the loop is estimated not to be entered; it is
261 // not a usable trip count for the profitability decisions below (and would
262 // e.g. divide by zero when scaling runtime check cost), so treat it as
263 // unknown.
264 if (Opts.loop_vectorize_with_block_frequency && !ComputeUpperBoundOnly)
265 if (unsigned EstimatedTC = getLoopEstimatedTripCount(L).value_or(0))
266 return ElementCount::getFixed(EstimatedTC);
267
268 if (!CanUseConstantMax)
269 return std::nullopt;
270
271 // Check if upper bound estimate is known.
272 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
273 return ElementCount::getFixed(ExpectedTC);
274
275 // Get the maximum trip count from the SCEV range excluding zero. This is
276 // only safe when not folding the tail, as the minimum iteration count check
277 // prevents entering the vector loop with a zero trip count.
278 if (CanUseConstantMax && CanExcludeZeroTrips)
279 if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
280 return ElementCount::getFixed(RefinedTC);
281
282 return std::nullopt;
283}
284
285namespace {
286using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
287} // namespace
288
289namespace llvm {
290
292
293} // end namespace llvm
294
295/// Look for a meaningful debug location on the instruction or its operands.
297 if (!I)
298 return DebugLoc::getUnknown();
299
301 if (I->getDebugLoc() != Empty)
302 return I->getDebugLoc();
303
304 for (Use &Op : I->operands()) {
305 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
306 if (OpInst->getDebugLoc() != Empty)
307 return OpInst->getDebugLoc();
308 }
309
310 return I->getDebugLoc();
311}
312
313namespace llvm {
314
315/// Return the runtime value for VF.
317 return B.CreateElementCount(Ty, VF);
318}
319
320} // end namespace llvm
321
322namespace llvm {
323
324// Loop vectorization cost-model hints how the epilogue/tail loop should be
325// lowered.
327
328 // The default: allowing epilogues.
330
331 // Vectorization with OptForSize: don't allow epilogues.
333
334 // A special case for loops with a very small trip count: they are vectorized
335 // only if no scalar epilogue is required, ensuring the vectorized loop body
336 // is the dominant cost without scalar epilogue overheads.
338
339 // Loop hint indicating an epilogue is undesired, apply tail folding.
341
342 // Directive indicating we must either fold the epilogue/tail or not vectorize
344};
345
347
348/// LoopVectorizationCostModel - estimates the expected speedups due to
349/// vectorization.
350/// In many cases vectorization is not profitable. This can happen because of
351/// a number of reasons. In this class we mainly attempt to predict the
352/// expected speedup/slowdowns due to the supported instruction set. We use the
353/// TargetTransformInfo to query the different backends for the cost of
354/// different operations.
357
358public:
365 std::function<BlockFrequencyInfo &()> GetBFI,
366 const Function *F, InterleavedAccessInfo &IAI,
367 VFSelectionContext &Config)
368 : Opts(Config.getOpts()), Config(Config), EpilogueLoweringStatus(SEL),
369 TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC),
371
372 /// \return An upper bound for the vectorization factors (both fixed and
373 /// scalable). If the factors are 0, vectorization and interleaving should be
374 /// avoided up front.
375 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
376
377 /// Memory access instruction may be vectorized in more than one way.
378 /// Form of instruction after vectorization depends on cost.
379 /// This function takes cost-based decisions for Load/Store instructions
380 /// and collects them in a map. This decisions map is used for building
381 /// the lists of loop-uniform and loop-scalar instructions.
382 /// The calculated cost is saved with widening decision in order to
383 /// avoid redundant calculations.
385
386 /// Collect values we want to ignore in the cost model.
388
389 /// \returns True if it is more profitable to scalarize instruction \p I for
390 /// vectorization factor \p VF.
392 assert(VF.isVector() &&
393 "Profitable to scalarize relevant only for VF > 1.");
394 assert(
395 TheLoop->isInnermost() &&
396 "cost-model should not be used for outer loops (in VPlan-native path)");
397
398 auto Scalars = InstsToScalarize.find(VF);
399 assert(Scalars != InstsToScalarize.end() &&
400 "VF not yet analyzed for scalarization profitability");
401 return Scalars->second.contains(I);
402 }
403
404 /// Returns true if \p I is known to be uniform after vectorization.
406 assert(
407 TheLoop->isInnermost() &&
408 "cost-model should not be used for outer loops (in VPlan-native path)");
409
410 // If VF is scalar, then all instructions are trivially uniform.
411 if (VF.isScalar())
412 return true;
413
414 // Pseudo probes must be duplicated per vector lane so that the
415 // profiled loop trip count is not undercounted.
417 return false;
418
419 auto UniformsPerVF = Uniforms.find(VF);
420 assert(UniformsPerVF != Uniforms.end() &&
421 "VF not yet analyzed for uniformity");
422 return UniformsPerVF->second.count(I);
423 }
424
425 /// Returns true if \p I is known to be scalar after vectorization.
427 assert(
428 TheLoop->isInnermost() &&
429 "cost-model should not be used for outer loops (in VPlan-native path)");
430 if (VF.isScalar())
431 return true;
432
433 auto ScalarsPerVF = Scalars.find(VF);
434 assert(ScalarsPerVF != Scalars.end() &&
435 "Scalar values are not calculated for VF");
436 return ScalarsPerVF->second.count(I);
437 }
438
439 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
440 /// for vectorization factor \p VF.
442 const auto &MinBWs = Config.getMinimalBitwidths();
443 // Truncs must truncate at most to their destination type.
444 if (isa_and_nonnull<TruncInst>(I) && MinBWs.contains(I) &&
445 I->getType()->getScalarSizeInBits() < MinBWs.lookup(I))
446 return false;
447 return VF.isVector() && MinBWs.contains(I) &&
450 }
451
452 /// Decision that was taken during cost calculation for memory instruction.
455 CM_Widen, // For consecutive accesses with stride +1.
456 CM_Widen_Reverse, // For consecutive accesses with stride -1.
460 /// A widening decision that has been invalidated after replacing the
461 /// corresponding recipe during VPlan transforms.
462 /// TODO: Remove once the legacy exit cost computation is retired.
464 };
465
466#ifndef NDEBUG
468 constexpr StringLiteral WideningStr[] = {
469 "Unknown", "Widen", "Widen_Reverse", "Interleave",
470 "GatherScatter", "Scalarize", "InvalidatedDecision"};
471 return WideningStr[W];
472 }
473#endif
474
475 /// Save vectorization decision \p W and \p Cost taken by the cost model for
476 /// instruction \p I and vector width \p VF.
479 assert(VF.isVector() && "Expected VF >=2");
480 LLVM_DEBUG(dbgs() << "LV: Setting widening decision to "
481 << getInstWideningStr(W) << " for VF " << VF
482 << " and instruction: " << *I << '\n');
483 WideningDecisions[{I, VF}] = {W, Cost};
484 }
485
486 /// Save vectorization decision \p W and \p Cost taken by the cost model for
487 /// interleaving group \p Grp and vector width \p VF.
491 assert(VF.isVector() && "Expected VF >=2");
492 /// Broadcast this decicion to all instructions inside the group.
493 /// When interleaving, the cost will only be assigned one instruction, the
494 /// insert position. For other cases, add the appropriate fraction of the
495 /// total cost to each instruction. This ensures accurate costs are used,
496 /// even if the insert position instruction is not used.
497 InstructionCost InsertPosCost = Cost;
498 InstructionCost OtherMemberCost = 0;
499 if (W != CM_Interleave)
500 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
501 ;
502 for (auto *I : Grp->members()) {
503 LLVM_DEBUG(dbgs() << "LV: Setting widening decision to "
504 << getInstWideningStr(W) << " for VF " << VF
505 << " and instruction: " << *I << '\n');
506 if (Grp->getInsertPos() == I)
507 WideningDecisions[{I, VF}] = {W, InsertPosCost};
508 else
509 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
510 }
511 }
512
513 /// Return the cost model decision for the given instruction \p I and vector
514 /// width \p VF. Return CM_Unknown if this instruction did not pass
515 /// through the cost modeling.
517 assert(VF.isVector() && "Expected VF to be a vector VF");
518 assert(
519 TheLoop->isInnermost() &&
520 "cost-model should not be used for outer loops (in VPlan-native path)");
521
522 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
523 auto Itr = WideningDecisions.find(InstOnVF);
524 if (Itr == WideningDecisions.end())
525 return CM_Unknown;
526 return Itr->second.first;
527 }
528
529 /// Return the vectorization cost for the given instruction \p I and vector
530 /// width \p VF.
532 assert(VF.isVector() && "Expected VF >=2");
533 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
534 assert(WideningDecisions.contains(InstOnVF) &&
535 "The cost is not calculated");
536 return WideningDecisions[InstOnVF].second;
537 }
538
539 /// Return True if instruction \p I is an optimizable truncate whose operand
540 /// is an induction variable. Such a truncate will be removed by adding a new
541 /// induction variable with the destination type.
543 // If the instruction is not a truncate, return false.
544 auto *Trunc = dyn_cast<TruncInst>(I);
545 if (!Trunc)
546 return false;
547
548 // Get the source and destination types of the truncate.
549 Type *SrcTy = toVectorTy(Trunc->getSrcTy(), VF);
550 Type *DestTy = toVectorTy(Trunc->getDestTy(), VF);
551
552 // If the truncate is free for the given types, return false. Replacing a
553 // free truncate with an induction variable would add an induction variable
554 // update instruction to each iteration of the loop. We exclude from this
555 // check the primary induction variable since it will need an update
556 // instruction regardless.
557 Value *Op = Trunc->getOperand(0);
558 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
559 return false;
560
561 // If the truncated value is not an induction variable, return false.
562 return Legal->isInductionPhi(Op);
563 }
564
565 /// Collects the instructions to scalarize for each predicated instruction in
566 /// the loop.
568
569 /// Collect values that will not be widened, including Uniforms, Scalars, and
570 /// Instructions to Scalarize for the given \p VF.
571 /// The sets depend on CM decision for Load/Store instructions
572 /// that may be vectorized as interleave, gather-scatter or scalarized.
573 /// Also make a decision on what to do about call instructions in the loop
574 /// at that VF -- scalarize, call a known vector routine, or call a
575 /// vector intrinsic.
577 // Do the analysis once.
578 if (VF.isScalar() || Uniforms.contains(VF))
579 return;
581 collectLoopUniforms(VF);
582 collectLoopScalars(VF);
584 }
585
586 /// Given costs for both strategies, return true if the scalar predication
587 /// lowering should be used for div/rem. This incorporates an override
588 /// option so it is not simply a cost comparison.
590 InstructionCost MaskedCost) const {
591 return !valueOr(Opts.force_widen_divrem_via_masked_intrinsic,
592 !(ScalarCost < MaskedCost));
593 }
594
595 /// Returns true if \p I is an instruction which requires predication and
596 /// for which our chosen predication strategy is scalarization (i.e. we
597 /// don't have an alternate strategy such as masking available).
598 /// \p VF is the vectorization factor that will be used to vectorize \p I.
600
601 /// Wrapper function for LoopVectorizationLegality::isMaskRequired,
602 /// that passes the Instruction \p I and if we fold tail.
603 bool isMaskRequired(Instruction *I) const;
604
605 /// Returns true if \p I is an instruction that needs to be predicated
606 /// at runtime. The result is independent of the predication mechanism.
607 /// Superset of instructions that return true for isScalarWithPredication.
608 bool isPredicatedInst(Instruction *I) const;
609
610 /// A helper function that returns how much we should divide the cost of a
611 /// predicated block by. Typically this is the reciprocal of the block
612 /// probability, i.e. if we return X we are assuming the predicated block will
613 /// execute once for every X iterations of the loop header so the block should
614 /// only contribute 1/X of its cost to the total cost calculation, but when
615 /// optimizing for code size it will just be 1 as code size costs don't depend
616 /// on execution probabilities.
617 ///
618 /// Note that if a block wasn't originally predicated but was predicated due
619 /// to tail folding, the divisor will still be 1 because it will execute for
620 /// every iteration of the loop header.
621 inline uint64_t
623 const BasicBlock *BB);
624
625 /// Returns true if an artificially high cost for emulated masked memrefs
626 /// should be used.
628
629 /// Return the costs for our two available strategies for lowering a
630 /// div/rem operation which requires speculating at least one lane.
631 /// First result is for scalarization (will be invalid for scalable
632 /// vectors); second is for the masked intrinsic strategy.
633 std::pair<InstructionCost, InstructionCost>
635
636 /// If \p I is a memory instruction with a consecutive pointer that can be
637 /// widened, returns the widening kind (CM_Widen or CM_Widen_Reverse) and
638 /// std::nullopt otherwise.
639 std::optional<InstWidening> memoryInstructionCanBeWidened(Instruction *I,
640 ElementCount VF);
641
642 /// Returns true if \p I is a memory instruction in an interleaved-group
643 /// of memory accesses that can be vectorized with wide vector loads/stores
644 /// and shuffles.
646
647 /// Returns true if the target machine supports masked loads or stores
648 /// for \p I's data type and alignment. The caller must ensure the access is
649 /// consecutive or part of an interleave group.
651
652 /// Returns true if the target machine supports gather or scatter for \p I's
653 /// data type and alignment.
655
656 /// Check if \p Instr belongs to any interleaved access group.
657 bool isAccessInterleaved(Instruction *Instr) const {
658 return InterleaveInfo.isInterleaved(Instr);
659 }
660
661 /// Get the interleaved access group that \p Instr belongs to.
664 return InterleaveInfo.getInterleaveGroup(Instr);
665 }
666
667 /// Returns true if we're required to use a scalar epilogue for at least
668 /// the final iteration of the original loop.
669 bool requiresScalarEpilogue(bool IsVectorizing) const {
670 if (!isEpilogueAllowed()) {
671 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
672 return false;
673 }
674 // If we might exit from anywhere but the latch and early exit vectorization
675 // is disabled, we must run the exiting iteration in scalar form.
676 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
677 !(Opts.enable_early_exit_vectorization &&
678 Legal->hasUncountableEarlyExit())) {
679 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
680 "from latch block\n");
681 return true;
682 }
683 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
684 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
685 "interleaved group requires scalar epilogue\n");
686 return true;
687 }
688 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
689 return false;
690 }
691
692 /// Returns true if an epilogue is allowed (e.g., not prevented by
693 /// optsize or a loop hint annotation).
694 bool isEpilogueAllowed() const {
695 return EpilogueLoweringStatus == CM_EpilogueAllowed;
696 }
697
698 /// Returns the TailFoldingStyle that is best for the current loop.
700 return ChosenTailFoldingStyle;
701 }
702
703 /// Selects and saves TailFoldingStyle.
704 /// \param IsScalableVF true if scalable vector factors enabled.
705 /// \param UserIC User specific interleave count.
706 void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
707 assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
708 "Tail folding must not be selected yet.");
709 if (!Legal->canFoldTailByMasking()) {
710 ChosenTailFoldingStyle = TailFoldingStyle::None;
711 return;
712 }
713
714 // Default to TTI preference, but allow command line override.
715 ChosenTailFoldingStyle = Opts.force_tail_folding_style.value_or(
716 TTI.getPreferredTailFoldingStyle());
717
718 if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
719 return;
720 // Override EVL styles if needed.
721 // FIXME: Investigate opportunity for fixed vector factor.
722 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
723 TTI.hasActiveVectorLength() &&
724 !Opts.enable_vplan_native_path;
725 if (EVLIsLegal)
726 return;
727 // If for some reason EVL mode is unsupported, fallback to an epilogue
728 // if it's allowed, or DataWithoutLaneMask otherwise.
729 if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
730 EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
731 ChosenTailFoldingStyle = TailFoldingStyle::None;
732 else
733 ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
734
736 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
737 "not try to generate VP Intrinsics "
738 << (UserIC > 1
739 ? "since interleave count specified is greater than 1.\n"
740 : "due to non-interleaving reasons.\n"));
741 }
742
743 /// Returns true if all loop blocks should be masked to fold tail loop.
744 bool foldTailByMasking() const {
746 }
747
749 assert(foldTailByMasking() && "Expected tail folding to be enabled!");
751 "Did not expect to enable alias masking with EVL!");
752 assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
753
754 // Assume we fail to enable alias masking (in case we early exit).
755 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
756
757 // Note: FixedOrderRecurrences are not supported yet as we cannot handle
758 // the required `splice.right` with the alias-mask.
759 if (!Opts.force_partial_aliasing_vectorization ||
760 !Legal->getFixedOrderRecurrences().empty())
761 return;
762
763 const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
764 if (!Checks)
765 return;
766
767 auto DiffChecks = Checks->getDiffChecks();
768 if (!DiffChecks || DiffChecks->empty())
769 return;
770
771 [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
772 return any_of(CB->args(), [](Value const *Arg) {
773 return Arg->getType()->isPointerTy();
774 });
775 };
776
777 for (BasicBlock *BB : TheLoop->blocks()) {
778 for (Instruction &I : *BB) {
780 [[maybe_unused]] auto *Call = dyn_cast<CallInst>(&I);
781 assert(
782 (!I.mayReadOrWriteMemory() || (Call && !HasPointerArgs(Call))) &&
783 "Skipped unexpected memory access");
784 continue;
785 }
786
787 Type *ScalarTy = getLoadStoreType(&I);
789
790 // Currently, we can't handle alias masking in reverse. Reversing the
791 // alias mask is not correct (or necessary). When combined with
792 // tail-folding the active lane mask should only be reversed where the
793 // alias-mask is true.
794 if (Legal->isConsecutivePtr(ScalarTy, Ptr) == -1)
795 return;
796 }
797 }
798
799 PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
800 }
801
802 /// Returns true if all loop blocks should have partial aliases masked.
803 bool maskPartialAliasing() const {
804 return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
805 }
806
807 /// Returns true if the instructions in this block requires predication
808 /// for any reason, e.g. because tail folding now requires a predicate,
809 /// because memory accesses are masked due to an uncountable exit with side
810 /// effects, or because the block in the original loop was predicated.
812 return foldTailByMasking() || Legal->hasUncountableExitWithSideEffects() ||
813 Legal->blockNeedsPredication(BB);
814 }
815
816 /// Returns true if VP intrinsics with explicit vector length support should
817 /// be generated in the tail folded loop.
821
822 /// Returns true if the predicated reduction select should be used to set the
823 /// incoming value for the reduction phi.
825 bool HasUsesOutsideReductionChain) const {
826 // Force to use predicated reduction select since the EVL of the
827 // second-to-last iteration might not be VF*UF.
828 if (foldTailWithEVL())
829 return true;
830
831 // Force a predicated select with alias-masking to avoid propagating poison
832 // values to the header phi for lanes outside the alias-mask.
834 return true;
835
836 // Note: For FindLast recurrences and multi-use reductions we prefer a
837 // predicated select to simplify matching in handleFindLastReductions() and
838 // handleMultiUseReductions() respectively, rather than handle multiple
839 // cases.
841 HasUsesOutsideReductionChain)
842 return true;
843
844 return Opts.prefer_predicated_reduction_select ||
845 TTI.preferPredicatedReductionSelect();
846 }
847
848 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
849 /// with factor VF. Return the cost of the instruction, including
850 /// scalarization overhead if it's needed.
852
853 /// Estimate cost of a call instruction CI if it were vectorized with factor
854 /// VF. Return the cost of the instruction, including scalarization overhead
855 /// if it's needed.
857
858 /// Invalidates decisions already taken by the cost model.
860 WideningDecisions.clear();
861 Uniforms.clear();
862 Scalars.clear();
863 }
864
865 /// Returns the expected execution cost. The unit of the cost does
866 /// not matter because we use the 'cost' units to compare different
867 /// vector widths. The cost that is returned is *not* normalized by
868 /// the factor width.
870
871 /// Returns the execution time cost of an instruction for a given vector
872 /// width. Vector width of one means scalar.
874
875 /// Returns true if \p Op should be considered invariant and if it is
876 /// trivially hoistable.
878
879 /// Returns true if \p I has been forced to be scalarized at \p VF.
881 auto FS = ForcedScalars.find(VF);
882 return FS != ForcedScalars.end() && FS->second.contains(I);
883 }
884
885private:
886 const VectorizeOptions &Opts;
887
888 unsigned NumPredStores = 0;
889
890 /// VF selection state independent of cost-modeling decisions.
891 VFSelectionContext &Config;
892
893 /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
894 /// account if alias-masking is enabled. We consider the VF to be unknown when
895 /// alias masking.
896 bool isUniform(Value *V, ElementCount VF) const {
897 // With alias-masking our runtime VF is [2, VF] (and not necessarily a
898 // power-of-two). Something that is uniform for VF may not be for the full
899 // range.
900 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
901 "alias-mask status must be decided already");
902 return Legal->isUniform(V, PartialAliasMaskingStatus ==
904 ? std::optional(VF)
905 : std::nullopt);
906 }
907
908 /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
909 /// account if alias-masking is enabled. We consider the VF to be unknown when
910 /// alias masking.
911 bool isUniformMemOp(Instruction &I, ElementCount VF) const {
912 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
913 "alias-mask status must be decided already");
914 return Legal->isUniformMemOp(I, PartialAliasMaskingStatus ==
916 ? std::optional(VF)
917 : std::nullopt);
918 }
919
920 /// Calculate vectorization cost of memory instruction \p I.
921 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
922
923 /// The cost computation for scalarized memory instruction.
924 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
925
926 /// The cost computation for interleaving group of memory instructions.
927 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
928
929 /// The cost computation for Gather/Scatter instruction.
930 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
931
932 /// The cost computation for widening instruction \p I with consecutive
933 /// memory access.
934 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF,
935 InstWidening Kind);
936
937 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
938 /// Load: scalar load + broadcast.
939 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
940 /// element)
941 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
942
943 /// Estimate the overhead of scalarizing an instruction. This is a
944 /// convenience wrapper for the type-based getScalarizationOverhead API.
945 InstructionCost getScalarizationOverhead(Instruction *I,
946 ElementCount VF) const;
947
948 /// A type representing the costs for instructions if they were to be
949 /// scalarized rather than vectorized. The entries are Instruction-Cost
950 /// pairs.
951 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
952
953 /// A set containing all BasicBlocks that are known to present after
954 /// vectorization as a predicated block.
955 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
956 PredicatedBBsAfterVectorization;
957
958 /// Records whether it is allowed to have the original scalar loop execute at
959 /// least once. This may be needed as a fallback loop in case runtime
960 /// aliasing/dependence checks fail, or to handle the tail/remainder
961 /// iterations when the trip count is unknown or doesn't divide by the VF,
962 /// or as a peel-loop to handle gaps in interleave-groups.
963 /// Under optsize and when the trip count is very small we don't allow any
964 /// iterations to execute in the scalar loop.
965 EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
966
967 /// Control finally chosen tail folding style.
968 TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
969
970 /// If partial alias masking is enabled/disabled or not decided.
971 AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
972
973 /// A map holding scalar costs for different vectorization factors. The
974 /// presence of a cost for an instruction in the mapping indicates that the
975 /// instruction will be scalarized when vectorizing with the associated
976 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
977 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
978
979 /// Holds the instructions known to be uniform after vectorization.
980 /// The data is collected per VF.
981 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
982
983 /// Holds the instructions known to be scalar after vectorization.
984 /// The data is collected per VF.
985 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
986
987 /// Holds the instructions (address computations) that are forced to be
988 /// scalarized.
989 DenseMap<ElementCount, SmallSetVector<Instruction *, 4>> ForcedScalars;
990
991 /// Returns the expected difference in cost from scalarizing the expression
992 /// feeding a predicated instruction \p PredInst. The instructions to
993 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
994 /// non-negative return value implies the expression will be scalarized.
995 /// Currently, only single-use chains are considered for scalarization.
996 InstructionCost computePredInstDiscount(Instruction *PredInst,
997 ScalarCostsTy &ScalarCosts,
998 ElementCount VF);
999
1000 /// Collect the instructions that are uniform after vectorization. An
1001 /// instruction is uniform if we represent it with a single scalar value in
1002 /// the vectorized loop corresponding to each vector iteration. Examples of
1003 /// uniform instructions include pointer operands of consecutive or
1004 /// interleaved memory accesses. Note that although uniformity implies an
1005 /// instruction will be scalar, the reverse is not true. In general, a
1006 /// scalarized instruction will be represented by VF scalar values in the
1007 /// vectorized loop, each corresponding to an iteration of the original
1008 /// scalar loop.
1009 void collectLoopUniforms(ElementCount VF);
1010
1011 /// Collect the instructions that are scalar after vectorization. An
1012 /// instruction is scalar if it is known to be uniform or will be scalarized
1013 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1014 /// to the list if they are used by a load/store instruction that is marked as
1015 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1016 /// VF values in the vectorized loop, each corresponding to an iteration of
1017 /// the original scalar loop.
1018 void collectLoopScalars(ElementCount VF);
1019
1020 /// Keeps cost model vectorization decision and cost for instructions.
1021 /// Right now it is used for memory instructions only.
1022 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1023 std::pair<InstWidening, InstructionCost>>;
1024
1025 DecisionList WideningDecisions;
1026
1027 /// Returns true if \p V is expected to be vectorized and it needs to be
1028 /// extracted.
1029 bool needsExtract(Value *V, ElementCount VF) const {
1031 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1032 TheLoop->isLoopInvariant(I) ||
1034 return false;
1035
1036 // Assume we can vectorize V (and hence we need extraction) if the
1037 // scalars are not computed yet. This can happen, because it is called
1038 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1039 // the scalars are collected. That should be a safe assumption in most
1040 // cases, because we check if the operands have vectorizable types
1041 // beforehand in LoopVectorizationLegality.
1042 return !Scalars.contains(VF) || !isScalarAfterVectorization(I, VF);
1043 };
1044
1045public:
1046 /// The loop that we evaluate.
1048
1049 /// Predicated scalar evolution analysis.
1051
1052 /// Loop Info analysis.
1054
1055 /// Vectorization legality.
1057
1058 /// Vector target information.
1060
1061 /// Target Library Info.
1063
1064 /// Assumption cache.
1066
1067 /// Interface to emit optimization remarks.
1069
1070 /// A function to lazily fetch BlockFrequencyInfo. This avoids computing it
1071 /// unless necessary, e.g. when the loop isn't legal to vectorize or when
1072 /// there is no predication.
1073 std::function<BlockFrequencyInfo &()> GetBFI;
1074 /// The BlockFrequencyInfo returned from GetBFI.
1076 /// Returns the BlockFrequencyInfo for the function if cached, otherwise
1077 /// fetches it via GetBFI. Avoids an indirect call to the std::function.
1079 if (!BFI)
1080 BFI = &GetBFI();
1081 return *BFI;
1082 }
1083
1085
1086 /// The interleave access information contains groups of interleaved accesses
1087 /// with the same stride and close to each other.
1089
1090 /// Values to ignore in the cost model.
1092
1093 /// Values to ignore in the cost model when VF > 1.
1095};
1096} // end namespace llvm
1097
1098namespace {
1099/// Helper struct to manage generating runtime checks for vectorization.
1100///
1101/// The runtime checks are created up-front in temporary blocks to allow better
1102/// estimating the cost and un-linked from the existing IR. After deciding to
1103/// vectorize, the checks are attached to VPlan as IR or recipes. If deciding
1104/// not to vectorize, the temporary blocks are completely removed.
1105class GeneratedRTChecks {
1106 const VectorizeOptions &Opts;
1107
1108 /// Basic block which contains the generated SCEV checks, if any.
1109 BasicBlock *SCEVCheckBlock = nullptr;
1110
1111 /// The value representing the result of the generated SCEV checks. If it is
1112 /// nullptr no SCEV checks have been generated.
1113 Value *SCEVCheckCond = nullptr;
1114
1115 /// Basic block which contains the generated memory runtime checks, if any.
1116 BasicBlock *MemCheckBlock = nullptr;
1117
1118 /// The value representing the result of the generated memory runtime checks.
1119 /// If it is nullptr no memory runtime checks have been generated.
1120 Value *MemRuntimeCheckCond = nullptr;
1121
1122 /// Whether checks were generated, retained after their IR is replaced or
1123 /// removed during VPlan execution.
1124 bool HasChecks = false;
1125
1126 DominatorTree *DT;
1127 LoopInfo *LI;
1129
1130 SCEVExpander SCEVExp;
1131 SCEVExpander MemCheckExp;
1132
1133 bool CostTooHigh = false;
1134
1135 Loop *OuterLoop = nullptr;
1136
1138
1139 /// The kind of cost that we are calculating
1140 TTI::TargetCostKind CostKind;
1141
1142 /// True if the loop is alias-masked (which allows us to omit diff checks).
1143 bool LoopUsesPartialAliasMasking = false;
1144
1145public:
1146 GeneratedRTChecks(const VectorizeOptions &Opts,
1149 TTI::TargetCostKind CostKind,
1150 bool LoopUsesPartialAliasMasking)
1151 : Opts(Opts), DT(DT), LI(LI), TTI(TTI),
1152 SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1153 MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1154 PSE(PSE), CostKind(CostKind),
1155 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1156
1157 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1158 /// accurately estimate the cost of the runtime checks. The blocks are
1159 /// un-linked from the IR and attached to VPlan as IR or recipes if
1160 /// profitable. Otherwise, the check blocks are removed completely. If \p LAI
1161 /// is nullptr (outer loops have no LoopAccessInfo), no memory runtime checks
1162 /// are generated.
1163 void create(Loop *L, const LoopAccessInfo *LAI,
1164 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC,
1165 OptimizationRemarkEmitter &ORE) {
1166
1167 // Hard cutoff to limit compile-time increase in case a very large number of
1168 // runtime checks needs to be generated.
1169 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1170 // profile info.
1171 CostTooHigh = LAI && LAI->getNumRuntimePointerChecks() >
1173 if (CostTooHigh) {
1174 // Mark runtime checks as never succeeding when they exceed the threshold.
1175 MemRuntimeCheckCond = ConstantInt::getTrue(L->getHeader()->getContext());
1176 SCEVCheckCond = ConstantInt::getTrue(L->getHeader()->getContext());
1177 ORE.emit([&]() {
1178 return OptimizationRemarkAnalysisAliasing(
1179 DEBUG_TYPE, "TooManyMemoryRuntimeChecks", L->getStartLoc(),
1180 L->getHeader())
1181 << "loop not vectorized: too many memory checks needed";
1182 });
1183 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
1184 return;
1185 }
1186
1187 BasicBlock *LoopHeader = L->getHeader();
1188 BasicBlock *Preheader = L->getLoopPreheader();
1189
1190 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1191 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1192 // may be used by SCEVExpander. The blocks will be un-linked from their
1193 // predecessors and removed from LI & DT at the end of the function.
1194 if (!UnionPred.isAlwaysTrue()) {
1195 SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1196 nullptr, "vector.scevcheck");
1197
1198 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1199 &UnionPred, SCEVCheckBlock->getTerminator());
1200 if (isa<Constant>(SCEVCheckCond)) {
1201 // Clean up directly after expanding the predicate to a constant, to
1202 // avoid further expansions re-using anything left over from SCEVExp.
1203 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1204 SCEVCleaner.cleanup();
1205 }
1206 }
1207
1208 // TODO: We need to estimate the cost of alias-masking in
1209 // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
1210 // alias-mask is generated later in VPlan.
1211 if (LAI && LAI->getRuntimePointerChecking()->Need &&
1212 !LoopUsesPartialAliasMasking) {
1213 const auto &RtPtrChecking = *LAI->getRuntimePointerChecking();
1214 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1215 MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
1216 "vector.memcheck");
1217
1218 auto DiffChecks = RtPtrChecking.getDiffChecks();
1219 if (DiffChecks) {
1220 MemRuntimeCheckCond = addDiffRuntimeChecks(
1221 MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp, VF, IC);
1222 } else {
1223 MemRuntimeCheckCond = addRuntimeChecks(
1224 MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
1226 }
1227 assert(MemRuntimeCheckCond &&
1228 "no RT checks generated although RtPtrChecking "
1229 "claimed checks are required");
1230 }
1231
1232 SCEVExp.eraseDeadInstructions(SCEVCheckCond);
1233 HasChecks = getSCEVChecks().first || getMemRuntimeChecks().first;
1234
1235 if (!MemCheckBlock && !SCEVCheckBlock)
1236 return;
1237
1238 // Unhook the temporary block with the checks, update various places
1239 // accordingly.
1240 if (SCEVCheckBlock)
1241 SCEVCheckBlock->replaceAllUsesWith(Preheader);
1242 if (MemCheckBlock)
1243 MemCheckBlock->replaceAllUsesWith(Preheader);
1244
1245 if (SCEVCheckBlock) {
1246 SCEVCheckBlock->getTerminator()->moveBefore(
1247 Preheader->getTerminator()->getIterator());
1248 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1249 UI->setDebugLoc(DebugLoc::getTemporary());
1250 Preheader->getTerminator()->eraseFromParent();
1251 }
1252 if (MemCheckBlock) {
1253 MemCheckBlock->getTerminator()->moveBefore(
1254 Preheader->getTerminator()->getIterator());
1255 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1256 UI->setDebugLoc(DebugLoc::getTemporary());
1257 Preheader->getTerminator()->eraseFromParent();
1258 }
1259
1260 DT->changeImmediateDominator(LoopHeader, Preheader);
1261 if (MemCheckBlock) {
1262 DT->eraseNode(MemCheckBlock);
1263 LI->removeBlock(MemCheckBlock);
1264 }
1265 if (SCEVCheckBlock) {
1266 DT->eraseNode(SCEVCheckBlock);
1267 LI->removeBlock(SCEVCheckBlock);
1268 }
1269
1270 // Clean up directly if the memory checks folded to false, to avoid further
1271 // expansions re-using anything left over from MemCheckExp.
1272 if (MemCheckBlock && !getMemRuntimeChecks().first)
1273 eraseMemCheckBlock();
1274
1275 // Outer loop is used as part of the later cost calculations.
1276 OuterLoop = L->getParentLoop();
1277 }
1278
1280 if (SCEVCheckBlock || MemCheckBlock)
1281 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1282
1283 if (CostTooHigh) {
1285 Cost.setInvalid();
1286 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1287 return Cost;
1288 }
1289
1290 InstructionCost RTCheckCost = 0;
1291 if (SCEVCheckBlock)
1292 for (Instruction &I : *SCEVCheckBlock) {
1293 if (SCEVCheckBlock->getTerminator() == &I)
1294 continue;
1295 InstructionCost C = TTI->getInstructionCost(&I, CostKind);
1296 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1297 RTCheckCost += C;
1298 }
1299 if (MemCheckBlock) {
1300 InstructionCost MemCheckCost = 0;
1301 for (Instruction &I : *MemCheckBlock) {
1302 if (MemCheckBlock->getTerminator() == &I)
1303 continue;
1304 InstructionCost C = TTI->getInstructionCost(&I, CostKind);
1305 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1306 MemCheckCost += C;
1307 }
1308
1309 // If the runtime memory checks are being created inside an outer loop
1310 // we should find out if these checks are outer loop invariant. If so,
1311 // the checks will likely be hoisted out and so the effective cost will
1312 // reduce according to the outer loop trip count.
1313 if (OuterLoop) {
1314 ScalarEvolution *SE = MemCheckExp.getSE();
1315 // TODO: If profitable, we could refine this further by analysing every
1316 // individual memory check, since there could be a mixture of loop
1317 // variant and invariant checks that mean the final condition is
1318 // variant.
1319 const SCEV *Cond = SE->getSCEV(MemRuntimeCheckCond);
1320 if (SE->isLoopInvariant(Cond, OuterLoop)) {
1321 // It seems reasonable to assume that we can reduce the effective
1322 // cost of the checks even when we know nothing about the trip
1323 // count. Assume that the outer loop executes at least twice.
1324 unsigned BestTripCount = 2;
1325
1326 // Get the best known TC estimate.
1327 if (auto EstimatedTC = getSmallBestKnownTC(
1328 Opts, PSE, OuterLoop, /* CanUseConstantMax = */ false))
1329 if (EstimatedTC->isFixed())
1330 BestTripCount = EstimatedTC->getFixedValue();
1331
1332 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1333
1334 // Let's ensure the cost is always at least 1.
1335 NewMemCheckCost = std::max(NewMemCheckCost.getValue(),
1337
1338 if (BestTripCount > 1)
1340 << "We expect runtime memory checks to be hoisted "
1341 << "out of the outer loop. Cost reduced from "
1342 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1343
1344 MemCheckCost = NewMemCheckCost;
1345 }
1346 }
1347
1348 RTCheckCost += MemCheckCost;
1349 }
1350
1351 if (SCEVCheckBlock || MemCheckBlock)
1352 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1353 << "\n");
1354
1355 return RTCheckCost;
1356 }
1357
1358 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1359 /// unused.
1360 ~GeneratedRTChecks() {
1361 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1362 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(SCEVCheckBlock);
1363 if (SCEVChecksUsed)
1364 SCEVCleaner.markResultUsed();
1365
1366 if (MemCheckBlock && pred_empty(MemCheckBlock))
1367 eraseMemCheckBlock();
1368
1369 SCEVCleaner.cleanup();
1370
1371 if (!SCEVChecksUsed)
1372 SCEVCheckBlock->eraseFromParent();
1373 }
1374
1375 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1376 /// outside VPlan.
1377 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1378 using namespace llvm::PatternMatch;
1379 if (!SCEVCheckCond || match(SCEVCheckCond, m_ZeroInt()))
1380 return {nullptr, nullptr};
1381
1382 return {SCEVCheckCond, SCEVCheckBlock};
1383 }
1384
1385 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1386 /// outside VPlan.
1387 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
1388 using namespace llvm::PatternMatch;
1389 if (MemRuntimeCheckCond && match(MemRuntimeCheckCond, m_ZeroInt()))
1390 return {nullptr, nullptr};
1391 return {MemRuntimeCheckCond, MemCheckBlock};
1392 }
1393
1394 /// Return true if any runtime checks have been added
1395 bool hasChecks() const { return HasChecks; }
1396
1397 /// Erase the memory check block, its instructions and their SCEV expansions.
1398 void eraseMemCheckBlock() {
1399 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1400 auto &SE = *MemCheckExp.getSE();
1401 // Memory runtime check generation creates compares that use expanded
1402 // values. Remove them before running the SCEVExpanderCleaner.
1403 for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
1404 if (MemCheckExp.isInsertedInstruction(&I))
1405 continue;
1406 SE.forgetValue(&I);
1407 I.eraseFromParent();
1408 }
1409 MemCheckCleaner.cleanup();
1410 MemCheckBlock->eraseFromParent();
1411 MemCheckBlock = nullptr;
1412 MemRuntimeCheckCond = nullptr;
1413 }
1414};
1415} // namespace
1416
1418 return Style == TailFoldingStyle::Data ||
1420}
1421
1425
1426// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1427// vectorization. The loop needs to be annotated with #pragma omp simd
1428// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1429// vector length information is not provided, vectorization is not considered
1430// explicit. Interleave hints are not allowed either. These limitations will be
1431// relaxed in the future.
1432// Please, note that we are currently forced to abuse the pragma 'clang
1433// vectorize' semantics. This pragma provides *auto-vectorization hints*
1434// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1435// provides *explicit vectorization hints* (LV can bypass legal checks and
1436// assume that vectorization is legal). However, both hints are implemented
1437// using the same metadata (llvm.loop.vectorize, processed by
1438// LoopVectorizeHints). This will be fixed in the future when the native IR
1439// representation for pragma 'omp simd' is introduced.
1440static bool isExplicitVecOuterLoop(const VectorizeOptions &Opts, Loop *OuterLp,
1442 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1443 LoopVectorizeHints Hints(Opts, OuterLp, true /*DisableInterleaving*/, *ORE);
1444
1445 // Only outer loops with an explicit vectorization hint are supported.
1446 // Unannotated outer loops are ignored.
1448 return false;
1449
1450 if (!Hints.allowVectorization(OuterLp, true /*VectorizeOnlyWhenForced*/)) {
1451 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1452 return false;
1453 }
1454
1455 if (Hints.getInterleave() > 1) {
1456 // TODO: Interleave support is future work.
1457 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1458 "outer loops.\n");
1459 Hints.emitRemarkWithHints();
1460 return false;
1461 }
1462
1463 return true;
1464}
1465
1466static void collectSupportedLoops(const VectorizeOptions &Opts, Loop &L,
1469 // Collect inner loops and outer loops without irreducible control flow. For
1470 // now, only collect outer loops that have explicit vectorization hints. If we
1471 // are stress testing the VPlan H-CFG construction, we collect the outermost
1472 // loop of every loop nest.
1473 if (L.isInnermost() || Opts.vplan_build_outerloop_stress_test ||
1474 (Opts.enable_vplan_native_path &&
1475 isExplicitVecOuterLoop(Opts, &L, ORE))) {
1476 LoopBlocksRPO RPOT(&L);
1477 RPOT.perform(LI);
1479 V.push_back(&L);
1480 // TODO: Collect inner loops inside marked outer loops in case
1481 // vectorization fails for the outer loop. Do not invoke
1482 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1483 // already known to be reducible. We can use an inherited attribute for
1484 // that.
1485 return;
1486 }
1487 }
1488 for (Loop *InnerL : L)
1489 collectSupportedLoops(Opts, *InnerL, LI, ORE, V);
1490}
1491
1492//===----------------------------------------------------------------------===//
1493// Implementation of LoopVectorizationCostModel and LoopVectorizationPlanner.
1494//===----------------------------------------------------------------------===//
1495
1496/// For the given VF and UF and maximum trip count computed for the loop, return
1497/// whether the induction variable might overflow in the vectorized loop. If not,
1498/// then we know a runtime overflow check always evaluates to false and can be
1499/// removed.
1501 const VectorizeOptions &Opts, const LoopVectorizationCostModel *Cost,
1502 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1503 // Always be conservative if we don't know the exact unroll factor.
1504 uint64_t MaxUF = UF ? *UF
1505 : std::max(Cost->TTI.getMaxInterleaveFactor(VF, false),
1506 Cost->TTI.getMaxInterleaveFactor(VF, true));
1507
1508 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1509 APInt MaxUIntTripCount = IdxTy->getMask();
1510
1511 // We know the runtime overflow check is known false iff the (max) trip-count
1512 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
1513 // the vector loop induction variable.
1514 if (std::optional<ElementCount> TC = getSmallBestKnownTC(
1515 Opts, Cost->PSE, Cost->TheLoop,
1516 /*CanUseConstantMax=*/true, /*CanExcludeZeroTrips=*/false,
1517 /*ComputeUpperBoundOnly=*/true)) {
1518 // Compute the maximum runtime values of VF and the trip count.
1519 std::optional<uint64_t> MaxStep =
1520 getMaxRuntimeElementCount(VF * MaxUF, *Cost->TheFunction);
1521 std::optional<uint64_t> MaxTC =
1522 getMaxRuntimeElementCount(*TC, *Cost->TheFunction);
1523 if (!MaxStep || !MaxTC)
1524 return false;
1525
1526 // Bail out if the maximum trip count is not representable in the induction
1527 // variable's type.
1528 if (MaxUIntTripCount.ult(*MaxTC))
1529 return false;
1530
1531 return (MaxUIntTripCount - *MaxTC).ugt(*MaxStep);
1532 }
1533
1534 return false;
1535}
1536
1537// Return whether we allow using masked interleave-groups (for dealing with
1538// strided loads/stores that reside in predicated blocks, or for dealing
1539// with gaps).
1540static bool useMaskedInterleavedAccesses(const VectorizeOptions &Opts,
1541 const TargetTransformInfo &TTI) {
1542 // If an override option has been passed in for interleaved accesses, use it.
1543 return valueOr(Opts.enable_masked_interleaved_mem_accesses,
1544 TTI.enableMaskedInterleavedAccessVectorization());
1545}
1546
1547/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
1548/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
1549/// predecessors and successors of VPBB, if any, are rewired to the new
1550/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
1552 BasicBlock *IRBB,
1553 VPlan *Plan = nullptr) {
1554 if (!Plan)
1555 Plan = VPBB->getPlan();
1556 VPIRBasicBlock *IRVPBB = Plan->createEmptyVPIRBasicBlock(IRBB);
1557 auto IP = IRVPBB->begin();
1558 for (auto &R : make_early_inc_range(VPBB->phis()))
1559 R.moveBefore(IP);
1560
1561 for (auto &R :
1563 R.moveBefore(IRVPBB->end());
1564
1565 VPBlockUtils::reassociateBlocks(VPBB, IRVPBB);
1566 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
1567 return IRVPBB;
1568}
1569
1570/// Knowing that loop \p L executes a single vector iteration, add instructions
1571/// that will get simplified and thus should not have any cost to \p
1572/// InstsToIgnore.
1575 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
1576 auto *Cmp = L->getLatchCmpInst();
1577 if (Cmp)
1578 InstsToIgnore.insert(Cmp);
1579 for (PHINode *IV : IL.keys()) {
1580 // The induction is free: a widened induction generates a vector phi with
1581 // its start value and an increment that is dead without a backedge.
1582 InstsToIgnore.insert(IV);
1583
1584 // Get next iteration value of the induction variable.
1585 Instruction *IVInst =
1586 cast<Instruction>(IV->getIncomingValueForBlock(L->getLoopLatch()));
1587 if (all_of(IVInst->users(),
1588 [&](const User *U) { return U == IV || U == Cmp; }))
1589 InstsToIgnore.insert(IVInst);
1590 }
1591}
1592
1593namespace {
1594
1595struct CSEDenseMapInfo {
1596 static bool canHandle(const Instruction *I) {
1599 }
1600
1601 static unsigned getHashValue(const Instruction *I) {
1602 assert(canHandle(I) && "Unknown instruction!");
1603 return hash_combine(I->getOpcode(),
1604 hash_combine_range(I->operand_values()));
1605 }
1606
1607 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
1608 return LHS->isIdenticalTo(RHS);
1609 }
1610};
1611
1612} // end anonymous namespace
1613
1614/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
1615/// removal, in favor of the VPlan-based one.
1616static void legacyCSE(BasicBlock *BB) {
1617 // Perform simple cse.
1619 for (Instruction &In : llvm::make_early_inc_range(*BB)) {
1620 if (!CSEDenseMapInfo::canHandle(&In))
1621 continue;
1622
1623 // Check if we can replace this instruction with any of the
1624 // visited instructions.
1625 if (Instruction *V = CSEMap.lookup(&In)) {
1626 In.replaceAllUsesWith(V);
1627 In.eraseFromParent();
1628 continue;
1629 }
1630
1631 CSEMap[&In] = &In;
1632 }
1633}
1634
1635/// This function attempts to return a value that represents the ElementCount
1636/// at runtime. For fixed-width VFs we know this precisely at compile
1637/// time, but for scalable VFs we calculate it based on an estimate of the
1638/// vscale value.
1640 std::optional<unsigned> VScale) {
1641 unsigned EstimatedVF = VF.getKnownMinValue();
1642 if (VF.isScalable())
1643 if (VScale)
1644 EstimatedVF *= *VScale;
1645 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
1646 return EstimatedVF;
1647}
1648
1649/// Returns the vector library variant function of \p CI usable at \p VF,
1650/// respecting \p MaskRequired, or nullptr if none is found: a mapping with
1651/// matching VF, masked if required, whose vector function is declared in the
1652/// module.
1654 bool MaskRequired,
1655 const TargetLibraryInfo *TLI) {
1656 if (!TLI || CI.isNoBuiltin())
1657 return nullptr;
1658 for (const VFInfo &Info : VFDatabase::getMappings(CI))
1659 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
1660 if (Function *F = CI.getModule()->getFunction(Info.VectorName))
1661 return F;
1662 return nullptr;
1663}
1664
1665/// Returns true iff \p CI has a library vector variant usable at \p VF.
1667 bool MaskRequired,
1668 const TargetLibraryInfo *TLI) {
1669 return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
1670}
1671
1674 ElementCount VF) const {
1675 Type *RetTy = CI->getType();
1677 for (auto &ArgOp : CI->args())
1678 Tys.push_back(ArgOp->getType());
1679
1680 InstructionCost ScalarCallCost = TTI.getCallInstrCost(
1681 CI->getCalledFunction(), RetTy, Tys, Config.CostKind);
1682
1683 // Cost of the scalar call (scalar VF) or its scalarization (vector VF). The
1684 // scalarization cost is only meaningful for fixed VFs.
1687 : ScalarCallCost * VF.getKnownMinValue() +
1688 getScalarizationOverhead(CI, VF);
1689
1690 // The call may be vectorized at this VF, via a vector intrinsic or a vector
1691 // library variant.
1693 Cost = std::min(Cost, getVectorIntrinsicCost(CI, VF));
1694
1695 if (Function *Variant =
1697 Cost = std::min(Cost,
1698 TTI.getCallInstrCost(
1699 /*F=*/nullptr, Variant->getReturnType(),
1700 Variant->getFunctionType()->params(), Config.CostKind));
1701
1702 return Cost;
1703}
1704
1706 if (VF.isScalar() || !canVectorizeTy(Ty))
1707 return Ty;
1708 return toVectorizedTy(Ty, VF);
1709}
1710
1713 ElementCount VF) const {
1715 assert(ID && "Expected intrinsic call!");
1716 Type *RetTy = maybeVectorizeType(CI->getType(), VF);
1717 FastMathFlags FMF;
1718 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
1719 FMF = FPMO->getFastMathFlags();
1720
1723 SmallVector<Type *> ParamTys;
1724 std::transform(FTy->param_begin(), FTy->param_end(),
1725 std::back_inserter(ParamTys),
1726 [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
1727
1728 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
1731 return TTI.getIntrinsicInstrCost(CostAttrs, Config.CostKind);
1732}
1733
1734void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
1735 // We should not collect Scalars more than once per VF. Right now, this
1736 // function is called from collectUniformsAndScalars(), which already does
1737 // this check. Collecting Scalars for VF=1 does not make any sense.
1738 assert(VF.isVector() && !Scalars.contains(VF) &&
1739 "This function should not be visited twice for the same VF");
1740
1741 // This avoids any chances of creating a REPLICATE recipe during planning
1742 // since that would result in generation of scalarized code during execution,
1743 // which is not supported for scalable vectors.
1744 if (VF.isScalable()) {
1745 Scalars[VF].insert_range(Uniforms[VF]);
1746 return;
1747 }
1748
1750
1751 // These sets are used to seed the analysis with pointers used by memory
1752 // accesses that will remain scalar.
1754 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
1755 auto *Latch = TheLoop->getLoopLatch();
1756
1757 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
1758 // The pointer operands of loads and stores will be scalar as long as the
1759 // memory access is not a gather/scatter or histogram operation. The value
1760 // operand of a store will remain scalar if the store is scalarized.
1761 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
1762 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
1763 assert(WideningDecision != CM_Unknown &&
1764 "Widening decision should be ready at this moment");
1765 auto *Store = dyn_cast<StoreInst>(MemAccess);
1766 if (Store && Ptr == Store->getValueOperand())
1767 return WideningDecision == CM_Scalarize;
1768 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
1769 "Ptr is neither a value or pointer operand");
1770 return WideningDecision != CM_GatherScatter &&
1772 };
1773
1774 // A helper that returns true if the given value is a getelementptr
1775 // instruction contained in the loop.
1776 auto IsLoopVaryingGEP = [&](Value *V) {
1777 return isa<GetElementPtrInst>(V) && !TheLoop->isLoopInvariant(V);
1778 };
1779
1780 // A helper that evaluates a memory access's use of a pointer. If the use will
1781 // be a scalar use and the pointer is only used by memory accesses, we place
1782 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
1783 // PossibleNonScalarPtrs.
1784 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
1785 // We only care about bitcast and getelementptr instructions contained in
1786 // the loop.
1787 if (!IsLoopVaryingGEP(Ptr))
1788 return;
1789
1790 // If the pointer has already been identified as scalar (e.g., if it was
1791 // also identified as uniform), there's nothing to do.
1792 auto *I = cast<Instruction>(Ptr);
1793 if (Worklist.count(I))
1794 return;
1795
1796 // If the use of the pointer will be a scalar use, and all users of the
1797 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
1798 // place the pointer in PossibleNonScalarPtrs.
1799 if (IsScalarUse(MemAccess, Ptr) &&
1801 ScalarPtrs.insert(I);
1802 else
1803 PossibleNonScalarPtrs.insert(I);
1804 };
1805
1806 // We seed the scalars analysis with three classes of instructions: (1)
1807 // instructions marked uniform-after-vectorization and (2) bitcast,
1808 // getelementptr and (pointer) phi instructions used by memory accesses
1809 // requiring a scalar use.
1810 //
1811 // (1) Add to the worklist all instructions that have been identified as
1812 // uniform-after-vectorization.
1813 Worklist.insert_range(Uniforms[VF]);
1814
1815 // (2) Add to the worklist all bitcast and getelementptr instructions used by
1816 // memory accesses requiring a scalar use. The pointer operands of loads and
1817 // stores will be scalar unless the operation is a gather or scatter.
1818 // The value operand of a store will remain scalar if the store is scalarized.
1819 for (auto *BB : TheLoop->blocks())
1820 for (auto &I : *BB) {
1821 if (auto *Load = dyn_cast<LoadInst>(&I)) {
1822 EvaluatePtrUse(Load, Load->getPointerOperand());
1823 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
1824 EvaluatePtrUse(Store, Store->getPointerOperand());
1825 EvaluatePtrUse(Store, Store->getValueOperand());
1826 }
1827 }
1828 for (auto *I : ScalarPtrs)
1829 if (!PossibleNonScalarPtrs.count(I)) {
1830 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
1831 Worklist.insert(I);
1832 }
1833
1834 // Insert the forced scalars.
1835 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
1836 // induction variable when the PHI user is scalarized.
1837 auto ForcedScalar = ForcedScalars.find(VF);
1838 if (ForcedScalar != ForcedScalars.end())
1839 for (auto *I : ForcedScalar->second) {
1840 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
1841 Worklist.insert(I);
1842 }
1843
1844 // Expand the worklist by looking through any bitcasts and getelementptr
1845 // instructions we've already identified as scalar. This is similar to the
1846 // expansion step in collectLoopUniforms(); however, here we're only
1847 // expanding to include additional bitcasts and getelementptr instructions.
1848 unsigned Idx = 0;
1849 while (Idx != Worklist.size()) {
1850 Instruction *Dst = Worklist[Idx++];
1851 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
1852 continue;
1853 auto *Src = cast<Instruction>(Dst->getOperand(0));
1854 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
1855 auto *J = cast<Instruction>(U);
1856 return !TheLoop->contains(J) || Worklist.count(J) ||
1857 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
1858 IsScalarUse(J, Src));
1859 })) {
1860 Worklist.insert(Src);
1861 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
1862 }
1863 }
1864
1865 // An induction variable will remain scalar if all users of the induction
1866 // variable and induction variable update remain scalar.
1867 for (const auto &Induction : Legal->getInductionVars()) {
1868 auto *Ind = Induction.first;
1869 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
1870
1871 // If tail-folding is applied, the primary induction variable will be used
1872 // to feed a vector compare.
1873 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
1874 continue;
1875
1876 // Returns true if \p Indvar is a pointer induction that is used directly by
1877 // load/store instruction \p I.
1878 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
1879 Instruction *I) {
1880 return Induction.second.getKind() ==
1883 Indvar == getLoadStorePointerOperand(I) && IsScalarUse(I, Indvar);
1884 };
1885
1886 // Determine if all users of the induction variable are scalar after
1887 // vectorization.
1888 bool ScalarInd = all_of(Ind->users(), [&](User *U) -> bool {
1889 auto *I = cast<Instruction>(U);
1890 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
1891 IsDirectLoadStoreFromPtrIndvar(Ind, I);
1892 });
1893 if (!ScalarInd)
1894 continue;
1895
1896 // If the induction variable update is a fixed-order recurrence, neither the
1897 // induction variable or its update should be marked scalar after
1898 // vectorization.
1899 auto *IndUpdatePhi = dyn_cast<PHINode>(IndUpdate);
1900 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(IndUpdatePhi))
1901 continue;
1902
1903 // Determine if all users of the induction variable update instruction are
1904 // scalar after vectorization.
1905 bool ScalarIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
1906 auto *I = cast<Instruction>(U);
1907 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
1908 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
1909 });
1910 if (!ScalarIndUpdate)
1911 continue;
1912
1913 // The induction variable and its update instruction will remain scalar.
1914 Worklist.insert(Ind);
1915 Worklist.insert(IndUpdate);
1916 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
1917 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
1918 << "\n");
1919 }
1920
1921 Scalars[VF].insert_range(Worklist);
1922}
1923
1931
1933 ElementCount VF) const {
1935 return Config.isLegalGatherOrScatter(isa<LoadInst>(I), getLoadStoreType(I),
1937}
1938
1940 ElementCount VF) {
1941 if (!isPredicatedInst(I))
1942 return false;
1943
1944 // Do we have a non-scalar lowering for this predicated
1945 // instruction? No - it is scalar with predication.
1946 switch(I->getOpcode()) {
1947 default:
1948 return true;
1949 case Instruction::Call: {
1950 if (VF.isScalar())
1951 return true;
1952 auto *CI = cast<CallInst>(I);
1953 // A vector intrinsic or library variant lowering avoids scalarization.
1954 return !getVectorIntrinsicIDForCall(CI, TLI) &&
1956 }
1957 case Instruction::Load:
1958 case Instruction::Store: {
1959 Type *ScalarTy = getLoadStoreType(I);
1961 if (Legal->isCompressedLoadOrStore(I))
1962 return false; // We can't scalarize compressed loads/stores.
1963 bool IsConsecutive = Legal->isConsecutivePtr(ScalarTy, Ptr);
1964 return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
1966 }
1967 case Instruction::UDiv:
1968 case Instruction::SDiv:
1969 case Instruction::SRem:
1970 case Instruction::URem: {
1971 // We have the option to use the llvm.masked.udiv intrinsics to avoid
1972 // predication. The cost based decision here will always select the masked
1973 // intrinsics for scalable vectors as scalarization isn't legal.
1974 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
1975 return isDivRemScalarWithPredication(ScalarCost, MaskedCost);
1976 }
1977 }
1978}
1979
1981 return Legal->isMaskRequired(I, foldTailByMasking());
1982}
1983
1984// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
1986 // TODO: We can use the loop-preheader as context point here and get
1987 // context sensitive reasoning for isSafeToSpeculativelyExecute.
1991 return false;
1992
1993 // If the instruction was executed conditionally in the original scalar loop,
1994 // predication is needed with a mask whose lanes are all possibly inactive.
1995 if (Legal->blockNeedsPredication(I->getParent()))
1996 return true;
1997
1998 // If we're not folding the tail by masking and not vectorizing a loop with
1999 // uncountable exits and side effects, predication is unnecessary.
2000 if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
2001 return false;
2002
2003 // All that remain are instructions with side-effects originally executed in
2004 // the loop unconditionally, but now execute under a tail-fold mask (only)
2005 // having at least one active lane (the first). If the side-effects of the
2006 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2007 // - it will cause the same side-effects as when masked.
2008 switch(I->getOpcode()) {
2009 default:
2011 "instruction should have been considered by earlier checks");
2012 case Instruction::Call:
2013 // Side-effects of a Call are assumed to be non-invariant, needing a
2014 // (fold-tail) mask.
2016 "should have returned earlier for calls not needing a mask");
2017 return true;
2018 case Instruction::Load:
2019 // If the address is loop invariant no predication is needed.
2020 return !Legal->isInvariant(getLoadStorePointerOperand(I));
2021 case Instruction::Store: {
2022 // For stores, we need to prove both speculation safety (which follows from
2023 // the same argument as loads), but also must prove the value being stored
2024 // is correct. The easiest form of the later is to require that all values
2025 // stored are the same.
2026 return !(Legal->isInvariant(getLoadStorePointerOperand(I)) &&
2027 TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand()));
2028 }
2029 case Instruction::UDiv:
2030 case Instruction::URem:
2031 // If the divisor is loop-invariant no predication is needed.
2032 return !Legal->isInvariant(I->getOperand(1));
2033 case Instruction::SDiv:
2034 case Instruction::SRem:
2035 // Conservative for now, since masked-off lanes may be poison and could
2036 // trigger signed overflow.
2037 return true;
2038 }
2039}
2040
2042 TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
2043 if (CostKind == TTI::TCK_CodeSize)
2044 return 1;
2045 // If the block wasn't originally predicated then return early to avoid
2046 // computing BlockFrequencyInfo unnecessarily.
2047 if (!Legal->blockNeedsPredication(BB))
2048 return 1;
2049
2050 uint64_t HeaderFreq =
2051 getBFI().getBlockFreq(TheLoop->getHeader()).getFrequency();
2052 uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
2053 assert(HeaderFreq >= BBFreq &&
2054 "Header has smaller block freq than dominated BB?");
2055 return std::round((double)HeaderFreq / BBFreq);
2056}
2057
2059 switch (Opcode) {
2060 case Instruction::UDiv:
2061 return Intrinsic::masked_udiv;
2062 case Instruction::SDiv:
2063 return Intrinsic::masked_sdiv;
2064 case Instruction::URem:
2065 return Intrinsic::masked_urem;
2066 case Instruction::SRem:
2067 return Intrinsic::masked_srem;
2068 default:
2069 llvm_unreachable("Unexpected opcode");
2070 }
2071}
2072
2073std::pair<InstructionCost, InstructionCost>
2075 ElementCount VF) {
2076 assert(I->getOpcode() == Instruction::UDiv ||
2077 I->getOpcode() == Instruction::SDiv ||
2078 I->getOpcode() == Instruction::SRem ||
2079 I->getOpcode() == Instruction::URem);
2081
2082 // Scalarization isn't legal for scalable vector types
2083 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2084 if (!VF.isScalable()) {
2085 // Get the scalarization cost and scale this amount by the probability of
2086 // executing the predicated block. If the instruction is not predicated,
2087 // we fall through to the next case.
2088 ScalarizationCost = 0;
2089
2090 // These instructions have a non-void type, so account for the phi nodes
2091 // that we will create. This cost is likely to be zero. The phi node
2092 // cost, if any, should be scaled by the block probability because it
2093 // models a copy at the end of each predicated block.
2094 ScalarizationCost += VF.getFixedValue() *
2095 TTI.getCFInstrCost(Instruction::PHI, Config.CostKind);
2096
2097 // The cost of the non-predicated instruction.
2098 ScalarizationCost +=
2099 VF.getFixedValue() * TTI.getArithmeticInstrCost(
2100 I->getOpcode(), I->getType(), Config.CostKind);
2101
2102 // The cost of insertelement and extractelement instructions needed for
2103 // scalarization.
2104 ScalarizationCost += getScalarizationOverhead(I, VF);
2105
2106 // Scale the cost by the probability of executing the predicated blocks.
2107 // This assumes the predicated block for each vector lane is equally
2108 // likely.
2109 ScalarizationCost =
2110 ScalarizationCost /
2111 getPredBlockCostDivisor(Config.CostKind, I->getParent());
2112 }
2113
2114 auto *VecTy = toVectorTy(I->getType(), VF);
2115 auto *MaskTy = toVectorTy(Type::getInt1Ty(I->getContext()), VF);
2116 IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(I->getOpcode()), VecTy,
2117 {VecTy, VecTy, MaskTy});
2118 InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, Config.CostKind);
2119 return {ScalarizationCost, MaskedCost};
2120}
2121
2123 Instruction *I, ElementCount VF) const {
2124 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2126 "Decision should not be set yet.");
2127 auto *Group = getInterleavedAccessGroup(I);
2128 assert(Group && "Must have a group.");
2129 unsigned InterleaveFactor = Group->getFactor();
2130
2131 // If the instruction's allocated size doesn't equal its type size, it
2132 // requires padding and will be scalarized.
2133 auto &DL = I->getDataLayout();
2134 auto *ScalarTy = getLoadStoreType(I);
2135 if (hasIrregularType(ScalarTy, DL))
2136 return false;
2137
2138 // For scalable vectors, the interleave factors must be <= 8 since we require
2139 // the (de)interleaveN intrinsics instead of shufflevectors.
2140 if (VF.isScalable() && InterleaveFactor > 8)
2141 return false;
2142
2143 // If the group involves a non-integral pointer, we may not be able to
2144 // losslessly cast all values to a common type.
2145 bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy);
2146 for (Instruction *Member : Group->members()) {
2147 auto *MemberTy = getLoadStoreType(Member);
2148 bool MemberNI = DL.isNonIntegralPointerType(MemberTy);
2149 // Don't coerce non-integral pointers to integers or vice versa.
2150 if (MemberNI != ScalarNI)
2151 // TODO: Consider adding special nullptr value case here
2152 return false;
2153 if (MemberNI && ScalarNI &&
2154 ScalarTy->getPointerAddressSpace() !=
2155 MemberTy->getPointerAddressSpace())
2156 return false;
2157 }
2158
2159 // Check if masking is required.
2160 // A Group may need masking for one of two reasons: it resides in a block that
2161 // needs predication, or it was decided to use masking to deal with gaps
2162 // (either a gap at the end of a load-access that may result in a speculative
2163 // load, or any gaps in a store-access).
2164 bool PredicatedAccessRequiresMasking =
2166 bool LoadAccessWithGapsRequiresEpilogMasking =
2167 isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
2169 bool StoreAccessWithGapsRequiresMasking =
2170 isa<StoreInst>(I) && !Group->isFull();
2171 if (!PredicatedAccessRequiresMasking &&
2172 !LoadAccessWithGapsRequiresEpilogMasking &&
2173 !StoreAccessWithGapsRequiresMasking)
2174 return true;
2175
2176 // If masked interleaving is required, we expect that the user/target had
2177 // enabled it, because otherwise it either wouldn't have been created or
2178 // it should have been invalidated by the CostModel.
2180 "Masked interleave-groups for predicated accesses are not enabled.");
2181
2182 if (Group->isReverse())
2183 return false;
2184
2185 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2186 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2187 StoreAccessWithGapsRequiresMasking;
2188 if (VF.isScalable() && NeedsMaskForGaps)
2189 return false;
2190
2191 return isLegalMaskedLoadOrStore(I, VF);
2192}
2193
2194std::optional<LoopVectorizationCostModel::InstWidening>
2196 ElementCount VF) {
2197 // Get and ensure we have a valid memory instruction.
2198 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2199
2200 auto *Ptr = getLoadStorePointerOperand(I);
2201 auto *ScalarTy = getLoadStoreType(I);
2202
2203 // In order to be widened, the pointer should be consecutive or compressed.
2204 int Stride = Legal->isConsecutivePtr(ScalarTy, Ptr);
2205
2206 // Check if the pointer is derived from a conditional induction PHI, which
2207 // can be widened as a compressstore/expandload.
2208 if (std::optional<CompressedPtrInfo> PtrInfo =
2209 Legal->getCompressedPtrInfo(Ptr)) {
2210 Stride = getStrideFromAddRec(PtrInfo->PtrSCEV, TheLoop, ScalarTy, Ptr, PSE)
2211 .value_or(0);
2212 assert(Stride == 1 && "Compressed memory ops must be consecutive");
2213 }
2214
2215 if (!Stride)
2216 return std::nullopt;
2217
2218 // If the instruction is a store located in a predicated block, it will be
2219 // scalarized.
2220 if (isScalarWithPredication(I, VF))
2221 return std::nullopt;
2222
2223 // If the instruction's allocated size doesn't equal it's type size, it
2224 // requires padding and will be scalarized.
2225 auto &DL = I->getDataLayout();
2226 if (hasIrregularType(ScalarTy, DL))
2227 return std::nullopt;
2228
2229 return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
2230}
2231
2232void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
2233 // We should not collect Uniforms more than once per VF. Right now,
2234 // this function is called from collectUniformsAndScalars(), which
2235 // already does this check. Collecting Uniforms for VF=1 does not make any
2236 // sense.
2237
2238 assert(VF.isVector() && !Uniforms.contains(VF) &&
2239 "This function should not be visited twice for the same VF");
2240
2241 // Visit the list of Uniforms. If we find no uniform value, we won't
2242 // analyze again. Uniforms.count(VF) will return 1.
2243 Uniforms[VF].clear();
2244
2245 // Now we know that the loop is vectorizable!
2246 // Collect instructions inside the loop that will remain uniform after
2247 // vectorization.
2248
2249 // Global values, params and instructions outside of current loop are out of
2250 // scope.
2251 auto IsOutOfScope = [&](Value *V) -> bool {
2253 return (!I || !TheLoop->contains(I));
2254 };
2255
2256 // Worklist containing uniform instructions demanding lane 0.
2257 SetVector<Instruction *> Worklist;
2258
2259 // Add uniform instructions demanding lane 0 to the worklist. Instructions
2260 // that require predication must not be considered uniform after
2261 // vectorization, because that would create an erroneous replicating region
2262 // where only a single instance out of VF should be formed.
2263 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
2264 if (IsOutOfScope(I)) {
2265 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
2266 << *I << "\n");
2267 return;
2268 }
2269 if (isPredicatedInst(I)) {
2270 LLVM_DEBUG(
2271 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
2272 << "\n");
2273 return;
2274 }
2275 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
2276 Worklist.insert(I);
2277 };
2278
2279 // Start with the conditional branches exiting the loop. If the branch
2280 // condition is an instruction contained in the loop that is only used by the
2281 // branch, it is uniform. Note conditions from uncountable early exits are not
2282 // uniform.
2284 TheLoop->getExitingBlocks(Exiting);
2285 for (BasicBlock *E : Exiting) {
2286 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
2287 continue;
2288 auto *Cmp = dyn_cast<Instruction>(E->getTerminator()->getOperand(0));
2289 if (!Cmp || !TheLoop->contains(Cmp) || !Cmp->hasOneUse())
2290 continue;
2291
2292 // If we have an exit condition that is actually two conditions (one
2293 // countable and the other uncountable) combined via an or, only add the
2294 // countable comparison as a uniform value.
2295 if (Legal->hasUncountableExitWithSideEffects() &&
2296 TheLoop->getLoopLatch() == E) {
2297 if (Instruction *Countable =
2298 Legal->findCountableComparisonInCombinedCondition(Cmp)) {
2299 if (Countable->hasOneUse())
2300 AddToWorklistIfAllowed(Countable);
2301 continue;
2302 }
2303 }
2304
2305 // Normal exit comparisons are uniform.
2306 AddToWorklistIfAllowed(Cmp);
2307 }
2308
2309 auto PrevVF = VF.divideCoefficientBy(2);
2310 // Return true if all lanes perform the same memory operation, and we can
2311 // thus choose to execute only one.
2312 auto IsUniformMemOpUse = [&](Instruction *I) {
2313 // If the value was already known to not be uniform for the previous
2314 // (smaller VF), it cannot be uniform for the larger VF.
2315 if (PrevVF.isVector()) {
2316 auto Iter = Uniforms.find(PrevVF);
2317 if (Iter != Uniforms.end() && !Iter->second.contains(I))
2318 return false;
2319 }
2320 if (!isUniformMemOp(*I, VF))
2321 return false;
2322 if (isa<LoadInst>(I))
2323 // Loading the same address always produces the same result - at least
2324 // assuming aliasing and ordering which have already been checked.
2325 return true;
2326 // Storing the same value on every iteration.
2327 return TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand());
2328 };
2329
2330 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
2331 InstWidening WideningDecision = getWideningDecision(I, VF);
2332 assert(WideningDecision != CM_Unknown &&
2333 "Widening decision should be ready at this moment");
2334
2335 if (IsUniformMemOpUse(I))
2336 return true;
2337
2338 return (WideningDecision == CM_Widen ||
2339 WideningDecision == CM_Widen_Reverse ||
2340 WideningDecision == CM_Interleave);
2341 };
2342
2343 // Returns true if Ptr is the pointer operand of a memory access instruction
2344 // I, I is known to not require scalarization, and the pointer is not also
2345 // stored.
2346 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
2347 if (isa<StoreInst>(I) && I->getOperand(0) == Ptr)
2348 return false;
2349 return getLoadStorePointerOperand(I) == Ptr &&
2350 (IsUniformDecision(I, VF) || Legal->isInvariant(Ptr));
2351 };
2352
2353 // Holds a list of values which are known to have at least one uniform use.
2354 // Note that there may be other uses which aren't uniform. A "uniform use"
2355 // here is something which only demands lane 0 of the unrolled iterations;
2356 // it does not imply that all lanes produce the same value (e.g. this is not
2357 // the usual meaning of uniform)
2358 SetVector<Value *> HasUniformUse;
2359
2360 // Scan the loop for instructions which are either a) known to have only
2361 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
2362 for (auto *BB : TheLoop->blocks())
2363 for (auto &I : *BB) {
2364 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I)) {
2365 switch (II->getIntrinsicID()) {
2366 case Intrinsic::sideeffect:
2367 case Intrinsic::experimental_noalias_scope_decl:
2368 case Intrinsic::assume:
2369 case Intrinsic::lifetime_start:
2370 case Intrinsic::lifetime_end:
2371 if (TheLoop->hasLoopInvariantOperands(&I))
2372 AddToWorklistIfAllowed(&I);
2373 break;
2374 default:
2375 break;
2376 }
2377 }
2378
2379 if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
2380 if (IsOutOfScope(EVI->getAggregateOperand())) {
2381 AddToWorklistIfAllowed(EVI);
2382 continue;
2383 }
2384 // Only ExtractValue instructions where the aggregate value comes from a
2385 // call are allowed to be non-uniform.
2386 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
2387 "Expected aggregate value to be call return value");
2388 }
2389
2390 // If there's no pointer operand, there's nothing to do.
2391 auto *Ptr = getLoadStorePointerOperand(&I);
2392 if (!Ptr)
2393 continue;
2394
2395 // If the pointer can be proven to be uniform, always add it to the
2396 // worklist.
2397 if (isa<Instruction>(Ptr) && isUniform(Ptr, VF))
2398 AddToWorklistIfAllowed(cast<Instruction>(Ptr));
2399
2400 if (IsUniformMemOpUse(&I))
2401 AddToWorklistIfAllowed(&I);
2402
2403 if (IsVectorizedMemAccessUse(&I, Ptr))
2404 HasUniformUse.insert(Ptr);
2405 }
2406
2407 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
2408 // demanding) users. Since loops are assumed to be in LCSSA form, this
2409 // disallows uses outside the loop as well.
2410 for (auto *V : HasUniformUse) {
2411 if (IsOutOfScope(V))
2412 continue;
2413 auto *I = cast<Instruction>(V);
2414 bool UsersAreMemAccesses = all_of(I->users(), [&](User *U) -> bool {
2415 auto *UI = cast<Instruction>(U);
2416 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
2417 });
2418 if (UsersAreMemAccesses)
2419 AddToWorklistIfAllowed(I);
2420 }
2421
2422 // Expand Worklist in topological order: whenever a new instruction
2423 // is added , its users should be already inside Worklist. It ensures
2424 // a uniform instruction will only be used by uniform instructions.
2425 unsigned Idx = 0;
2426 while (Idx != Worklist.size()) {
2427 Instruction *I = Worklist[Idx++];
2428
2429 for (auto *OV : I->operand_values()) {
2430 // isOutOfScope operands cannot be uniform instructions.
2431 if (IsOutOfScope(OV))
2432 continue;
2433 // First order recurrence Phi's should typically be considered
2434 // non-uniform.
2435 auto *OP = dyn_cast<PHINode>(OV);
2436 if (OP && Legal->isFixedOrderRecurrence(OP))
2437 continue;
2438 // If all the users of the operand are uniform, then add the
2439 // operand into the uniform worklist.
2440 auto *OI = cast<Instruction>(OV);
2441 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
2442 auto *J = cast<Instruction>(U);
2443 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
2444 }))
2445 AddToWorklistIfAllowed(OI);
2446 }
2447 }
2448
2449 // For an instruction to be added into Worklist above, all its users inside
2450 // the loop should also be in Worklist. However, this condition cannot be
2451 // true for phi nodes that form a cyclic dependence. We must process phi
2452 // nodes separately. An induction variable will remain uniform if all users
2453 // of the induction variable and induction variable update remain uniform.
2454 // The code below handles both pointer and non-pointer induction variables.
2455 BasicBlock *Latch = TheLoop->getLoopLatch();
2456 for (PHINode *Ind : Legal->getInductionVars().keys()) {
2457 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
2458
2459 // Determine if all users of the induction variable are uniform after
2460 // vectorization.
2461 bool UniformInd = all_of(Ind->users(), [&](User *U) -> bool {
2462 auto *I = cast<Instruction>(U);
2463 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2464 IsVectorizedMemAccessUse(I, Ind);
2465 });
2466 if (!UniformInd)
2467 continue;
2468
2469 // Determine if all users of the induction variable update instruction are
2470 // uniform after vectorization.
2471 bool UniformIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
2472 auto *I = cast<Instruction>(U);
2473 return I == Ind || Worklist.count(I) ||
2474 IsVectorizedMemAccessUse(I, IndUpdate);
2475 });
2476 if (!UniformIndUpdate)
2477 continue;
2478
2479 // The induction variable and its update instruction will remain uniform.
2480 AddToWorklistIfAllowed(Ind);
2481 AddToWorklistIfAllowed(IndUpdate);
2482 }
2483
2484 Uniforms[VF].insert_range(Worklist);
2485}
2486
2489 // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
2490 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
2491 if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
2492 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
2493 });
2494
2495 // For outer loops, use simple type-based heuristic VF. No cost model or
2496 // memory dependence analysis is available.
2497 if (!TheLoop->isInnermost()) {
2498 return Config.computeVPlanOuterloopVF(UserVF);
2499 }
2500
2501 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
2502 // TODO: It may be useful to do since it's still likely to be dynamically
2503 // uniform if the target can skip.
2505 "Not inserting runtime ptr check for divergent target",
2506 "runtime pointer checks needed. Not enabled for divergent target",
2507 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
2509 }
2510
2511 ScalarEvolution *SE = PSE.getSE();
2513 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
2514 if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
2516 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
2517 if (TC != ElementCount::getFixed(MaxTC))
2518 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
2519 if (TC.isScalar()) {
2521 "Single iteration (non) loop",
2522 "loop trip count is one, irrelevant for vectorization",
2523 "SingleIterationLoop", ORE, TheLoop);
2525 }
2526
2527 // If BTC matches the widest induction type and is -1 then the trip count
2528 // computation will wrap to 0 and the vector trip count will be 0. Do not try
2529 // to vectorize.
2530 const SCEV *BTC = SE->getBackedgeTakenCount(TheLoop);
2531 if (!isa<SCEVCouldNotCompute>(BTC) &&
2532 BTC->getType()->getScalarSizeInBits() >=
2533 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2535 SE->getMinusOne(BTC->getType()))) {
2537 "Trip count computation wrapped",
2538 "backedge-taken count is -1, loop trip count wrapped to 0",
2539 "TripCountWrapped", ORE, TheLoop);
2541 }
2542
2543 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2544 "No cost-modeling decisions should have been taken at this point");
2545
2546 switch (EpilogueLoweringStatus) {
2547 case CM_EpilogueAllowed:
2548 return Config.computeFeasibleMaxVF(MaxTC, UserVF, UserIC, false,
2551 [[fallthrough]];
2553 LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
2554 << "LV: Not allowing epilogue, creating tail-folded "
2555 << "vector loop.\n");
2556 break;
2558 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
2559 << "count.\n");
2560 break;
2562 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
2563
2564 // Bail if runtime checks are required, which are not good when optimising
2565 // for size.
2566 if (Config.runtimeChecksRequired())
2568
2569 break;
2570 }
2571
2572 // Now try the tail folding
2573
2574 // Invalidate interleave groups that require an epilogue if we can't mask
2575 // the interleave-group.
2576 if (!useMaskedInterleavedAccesses(Opts, TTI)) {
2577 // Note: There is no need to invalidate any cost modeling decisions here, as
2578 // none were taken so far (see assertion above).
2579 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
2580 }
2581
2582 FixedScalableVFPair MaxFactors = Config.computeFeasibleMaxVF(
2583 MaxTC, UserVF, UserIC, true, requiresScalarEpilogue(true));
2584
2585 // Avoid tail folding if the trip count is known to be a multiple of any VF
2586 // we choose.
2587 std::optional<uint64_t> MaxPowerOf2RuntimeVF =
2588 MaxFactors.FixedVF.getFixedValue();
2589 if (MaxFactors.ScalableVF) {
2590 if (std::optional<uint64_t> MaxRuntimeScalableVF =
2592 MaxPowerOf2RuntimeVF =
2593 std::max(*MaxPowerOf2RuntimeVF, *MaxRuntimeScalableVF);
2594 else
2595 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
2596 }
2597
2598 // TODO: Make NoScalarEpilogueNeeded lambda a separate function to be used
2599 // only for main loop VF not also epilogueVF. Using it for epilogueVF against
2600 // full TC is inaccurate.
2601 auto NoScalarEpilogueNeeded = [this, &UserIC](uint64_t MaxRuntimeVF) {
2602 // Return false if the loop is neither a single-latch-exit loop nor an
2603 // early-exit loop as tail-folding is not supported in that case.
2604 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
2605 !Legal->hasUncountableEarlyExit())
2606 return false;
2607 uint64_t MaxVFtimesIC = MaxRuntimeVF * std::max<uint64_t>(UserIC, 1);
2608 ScalarEvolution *SE = PSE.getSE();
2609 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
2610 // with uncountable exits. For countable loops, the symbolic maximum must
2611 // remain identical to the known back-edge taken count.
2612 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
2613 assert((Legal->hasUncountableEarlyExit() ||
2614 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
2615 "Invalid loop count");
2616 const SCEV *ExitCount = SE->getAddExpr(
2617 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
2618 const SCEV *Rem = SE->getURemExpr(
2619 SE->applyLoopGuards(ExitCount, TheLoop),
2620 SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
2621 return Rem->isZero();
2622 };
2623
2624 if (MaxPowerOf2RuntimeVF > 0u) {
2625 assert((UserVF.isNonZero() || isPowerOf2_64(*MaxPowerOf2RuntimeVF)) &&
2626 "MaxFixedVF must be a power of 2");
2627 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
2628 // Accept MaxFixedVF if we do not have a tail.
2629 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
2630 return MaxFactors;
2631 }
2632 }
2633
2634 auto ExpectedTC = getSmallBestKnownTC(Opts, PSE, TheLoop);
2635 if (ExpectedTC && ExpectedTC->isFixed() &&
2636 ExpectedTC->getFixedValue() <=
2637 TTI.getMinTripCountTailFoldingThreshold()) {
2638 // If we have a low-trip-count, or optimize for code size, and the
2639 // fixed-width VF is known to divide the trip count the fixed-width factor
2640 // in preference to allow the generation of a non-predicated loop.
2641 if ((EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop ||
2642 EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize) &&
2643 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
2644 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
2645 "remain for any chosen VF.\n");
2646 MaxFactors.ScalableVF = ElementCount::getScalable(0);
2647 return MaxFactors;
2648 }
2649
2650 // Allow cases where the ExactTC == (VF * IC) + 1.
2651 //
2652 // This produces 1 vector iteration, and 1 scalar iteration with no
2653 // remainder. Later passes will eliminate the loop and leave straight-line
2654 // code as the both iteration counts are statically known.
2655 //
2656 // If a function is marked as minsize/optsize or OptForSize is set, do not
2657 // allow this form of transformation as this will increase CodeSize.
2658 //
2659 // For loops with small bodies, the cost model is not currently reliable
2660 // enough to accurately determine if vectorization is beneficial.
2661 unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
2662 unsigned MaxVFForTC = llvm::bit_floor(TC.getFixedValue());
2663 if (TC.getFixedValue() - MaxVFForTC == 1 && MaxVFForTC / EffectiveIC > 1 &&
2664 MaxVFForTC <= (MaxFactors.FixedVF.getFixedValue() * EffectiveIC) &&
2665 !Config.OptForSize) {
2666 unsigned NumOfInstructions = llvm::sum_of(
2667 llvm::map_range(TheLoop->blocks(),
2668 [](BasicBlock *BB) { return BB->size(); }),
2669 unsigned(0));
2670 if (NumOfInstructions > Opts.low_trip_count_loop_body_size_limit) {
2671 unsigned VF = MaxVFForTC / EffectiveIC;
2672 LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
2673 << " with 1 scalar iteration remaining.\n");
2674 MaxFactors.FixedVF = ElementCount::getFixed(VF);
2675 MaxFactors.ScalableVF = ElementCount::getScalable(0);
2676 return MaxFactors;
2677 }
2678 }
2679
2681 "The trip count is below the minial threshold value.",
2682 "loop trip count is too low, avoiding vectorization", "LowTripCount",
2683 ORE, TheLoop);
2685 }
2686
2687 // If we don't know the precise trip count, or if the trip count that we
2688 // found modulo the vectorization factor is not zero, try to fold the tail
2689 // by masking.
2690 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
2691 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
2692 setTailFoldingStyle(ContainsScalableVF, UserIC);
2693 if (foldTailByMasking()) {
2694 if (foldTailWithEVL()) {
2695 LLVM_DEBUG(
2696 dbgs()
2697 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
2698 "try to generate VP Intrinsics with scalable vector "
2699 "factors only.\n");
2700 // Tail folded loop using VP intrinsics restricts the VF to be scalable
2701 // for now.
2702 // TODO: extend it for fixed vectors, if required.
2703 assert(ContainsScalableVF && "Expected scalable vector factor.");
2704
2705 MaxFactors.FixedVF = ElementCount::getFixed(1);
2706 } else {
2708 }
2709 return MaxFactors;
2710 }
2711
2712 // If there was a tail-folding hint/switch, but we can't fold the tail by
2713 // masking, fallback to a vectorization with an epilogue.
2714 if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
2715 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
2716 "epilogue instead.\n");
2717 EpilogueLoweringStatus = CM_EpilogueAllowed;
2718 return MaxFactors;
2719 }
2720
2721 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
2722 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
2724 }
2725
2726 if (TC.isZero()) {
2728 "unable to calculate the loop count due to complex control flow",
2729 "UnknownLoopCountComplexCFG", ORE, TheLoop);
2731 }
2732
2734 "Cannot optimize for size and vectorize at the same time.",
2735 "cannot optimize for size and vectorize at the same time. "
2736 "Enable vectorization of this loop with '#pragma clang loop "
2737 "vectorize(enable)' when compiling with -Os/-Oz",
2738 "NoTailLoopWithOptForSize", ORE, TheLoop);
2740}
2741
2744 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
2745 SmallVector<RecipeVFPair> InvalidCosts;
2746 for (const auto &Plan : VPlans) {
2747 // Skip cost remarks when Plan is not compatible with the CM.
2748 // Specifically for the case of epilogue tail-folded Plans.
2749 if (Plan->hasTailFolded() != CM->foldTailByMasking())
2750 continue;
2751 for (ElementCount VF : Plan->vectorFactors()) {
2752 // The VPlan-based cost model is designed for computing vector cost.
2753 // Querying VPlan-based cost model with a scalar VF will cause some
2754 // errors because we expect the VF is vector for most of the widen
2755 // recipes.
2756 if (VF.isScalar())
2757 continue;
2758
2759 VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
2760 /*ReusePrintingSlotTracker=*/true);
2761 precomputeCosts(*Plan, VF, CostCtx);
2762 auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
2764 for (auto &R : *VPBB) {
2765 if (!R.cost(VF, CostCtx).isValid())
2766 InvalidCosts.emplace_back(&R, VF);
2767 }
2768 }
2769 }
2770 }
2771 if (InvalidCosts.empty())
2772 return;
2773
2774 // Emit a report of VFs with invalid costs in the loop.
2775
2776 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
2778 unsigned I = 0;
2779 for (auto &Pair : InvalidCosts)
2780 if (Numbering.try_emplace(Pair.first, I).second)
2781 ++I;
2782
2783 // Sort the list, first on recipe(number) then on VF.
2784 sort(InvalidCosts, [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
2785 unsigned NA = Numbering[A.first];
2786 unsigned NB = Numbering[B.first];
2787 if (NA != NB)
2788 return NA < NB;
2789 return ElementCount::isKnownLT(A.second, B.second);
2790 });
2791
2792 // For a list of ordered recipe-VF pairs:
2793 // [(load, VF1), (load, VF2), (store, VF1)]
2794 // group the recipes together to emit separate remarks for:
2795 // load (VF1, VF2)
2796 // store (VF1)
2797 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
2798 auto Subset = ArrayRef<RecipeVFPair>();
2799 do {
2800 if (Subset.empty())
2801 Subset = Tail.take_front(1);
2802
2803 VPRecipeBase *R = Subset.front().first;
2804
2805 unsigned Opcode =
2807 .Case([](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
2808 .Case(
2809 [](const VPWidenStoreRecipe *R) { return Instruction::Store; })
2810 .Case([](const VPWidenLoadRecipe *R) { return Instruction::Load; })
2811 .Case([](const VPWidenStoreEVLRecipe *R) {
2812 return Instruction::Store;
2813 })
2814 .Case(
2815 [](const VPWidenLoadEVLRecipe *R) { return Instruction::Load; })
2816 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
2817 [](const auto *R) { return Instruction::Call; })
2820 [](const auto *R) { return R->getOpcode(); })
2821 .Case([](const VPInterleaveRecipe *R) {
2822 return R->getStoredValues().empty() ? Instruction::Load
2823 : Instruction::Store;
2824 })
2825 .Case([](const VPReductionRecipe *R) {
2826 return RecurrenceDescriptor::getOpcode(R->getRecurrenceKind());
2827 })
2828 .Case([](const VPBranchOnMaskRecipe *) {
2829 return Instruction::CondBr;
2830 });
2831
2832 // If the next recipe is different, or if there are no other pairs,
2833 // emit a remark for the collated subset. e.g.
2834 // [(load, VF1), (load, VF2))]
2835 // to emit:
2836 // remark: invalid costs for 'load' at VF=(VF1, VF2)
2837 if (Subset == Tail || Tail[Subset.size()].first != R) {
2838 std::string OutString;
2839 raw_string_ostream OS(OutString);
2840 assert(!Subset.empty() && "Unexpected empty range");
2841 OS << "Recipe with invalid costs prevented vectorization at VF=(";
2842 for (const auto &Pair : Subset)
2843 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
2844 OS << "):";
2845 if (Opcode == Instruction::Call) {
2846 StringRef Name = "";
2847 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) {
2848 Name = Int->getIntrinsicName();
2849 } else {
2850 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
2851 Function *CalledFn =
2852 WidenCall
2853 ? WidenCall->getCalledScalarFunction()
2854 : cast<Function>(R->getLastOperand()->getLiveInIRValue());
2855 Name = CalledFn->getName();
2856 }
2857 OS << " call to " << Name;
2858 } else
2859 OS << " " << Instruction::getOpcodeName(Opcode);
2860 reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
2861 R->getDebugLoc());
2862 Tail = Tail.drop_front(Subset.size());
2863 Subset = {};
2864 } else
2865 // Grow the subset by one element
2866 Subset = Tail.take_front(Subset.size() + 1);
2867 } while (!Tail.empty());
2868}
2869
2870/// Check if any recipe of \p Plan will generate a vector value, which will be
2871/// assigned a vector register.
2873 const TargetTransformInfo &TTI) {
2874 assert(VF.isVector() && "Checking a scalar VF?");
2875 DenseSet<VPRecipeBase *> EphemeralRecipes;
2876 collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
2877 // Set of already visited types.
2878 DenseSet<Type *> Visited;
2881 for (VPRecipeBase &R : *VPBB) {
2882 if (EphemeralRecipes.contains(&R))
2883 continue;
2884 // Continue early if the recipe is considered to not produce a vector
2885 // result. Note that this includes VPInstruction where some opcodes may
2886 // produce a vector, to preserve existing behavior as VPInstructions model
2887 // aspects not directly mapped to existing IR instructions.
2888 switch (R.getVPRecipeID()) {
2889 case VPRecipeBase::VPDerivedIVSC:
2890 case VPRecipeBase::VPScalarIVStepsSC:
2891 case VPRecipeBase::VPReplicateSC:
2892 case VPRecipeBase::VPInstructionSC:
2893 case VPRecipeBase::VPCurrentIterationPHISC:
2894 case VPRecipeBase::VPVectorPointerSC:
2895 case VPRecipeBase::VPVectorEndPointerSC:
2896 case VPRecipeBase::VPExpandSCEVSC:
2897 case VPRecipeBase::VPPredInstPHISC:
2898 case VPRecipeBase::VPBranchOnMaskSC:
2899 case VPRecipeBase::VPConditionalInductionPHISC:
2900 continue;
2901 case VPRecipeBase::VPReductionSC:
2902 case VPRecipeBase::VPActiveLaneMaskPHISC:
2903 case VPRecipeBase::VPWidenCallSC:
2904 case VPRecipeBase::VPWidenCanonicalIVSC:
2905 case VPRecipeBase::VPWidenCastSC:
2906 case VPRecipeBase::VPWidenGEPSC:
2907 case VPRecipeBase::VPWidenIntrinsicSC:
2908 case VPRecipeBase::VPWidenMemIntrinsicSC:
2909 case VPRecipeBase::VPWidenSC:
2910 case VPRecipeBase::VPBlendSC:
2911 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
2912 case VPRecipeBase::VPHistogramSC:
2913 case VPRecipeBase::VPWidenPHISC:
2914 case VPRecipeBase::VPWidenIntOrFpInductionSC:
2915 case VPRecipeBase::VPWidenPointerInductionSC:
2916 case VPRecipeBase::VPReductionPHISC:
2917 case VPRecipeBase::VPInterleaveEVLSC:
2918 case VPRecipeBase::VPInterleaveSC:
2919 case VPRecipeBase::VPWidenLoadEVLSC:
2920 case VPRecipeBase::VPWidenLoadSC:
2921 case VPRecipeBase::VPWidenStoreEVLSC:
2922 case VPRecipeBase::VPWidenStoreSC:
2923 break;
2924 default:
2925 llvm_unreachable("unhandled recipe");
2926 }
2927
2928 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
2929 unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
2930 if (!NumLegalParts)
2931 return false;
2932 if (VF.isScalable()) {
2933 // <vscale x 1 x iN> is assumed to be profitable over iN because
2934 // scalable registers are a distinct register class from scalar
2935 // ones. If we ever find a target which wants to lower scalable
2936 // vectors back to scalars, we'll need to update this code to
2937 // explicitly ask TTI about the register class uses for each part.
2938 return NumLegalParts <= VF.getKnownMinValue();
2939 }
2940 // Two or more elements that share a register - are vectorized.
2941 return NumLegalParts < VF.getFixedValue();
2942 };
2943
2944 // If no def nor is a store, e.g., branches, continue - no value to check.
2945 if (R.getNumDefinedValues() == 0 &&
2947 continue;
2948 // For multi-def recipes, currently only interleaved loads, suffice to
2949 // check first def only.
2950 // For stores check their stored value; for interleaved stores suffice
2951 // the check first stored value only. In all cases this is the second
2952 // operand.
2953 VPValue *ToCheck =
2954 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
2955 Type *ScalarTy = ToCheck->getScalarType();
2956 if (!Visited.insert({ScalarTy}).second)
2957 continue;
2958 Type *WideTy = toVectorizedTy(ScalarTy, VF);
2959 if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
2960 return true;
2961 }
2962 }
2963
2964 return false;
2965}
2966
2967static bool hasReplicatorRegion(VPlan &Plan) {
2969 Plan.getVectorLoopRegion()->getEntry())),
2970 [](auto *VPRB) { return VPRB->isReplicator(); });
2971}
2972
2973/// Returns true if the VPlan contains a VPReductionPHIRecipe with
2974/// FindLast recurrence kind.
2975static bool hasFindLastReductionPhi(VPlan &Plan) {
2978 [](VPReductionPHIRecipe &RedPhi) {
2979 return RecurrenceDescriptor::isFindLastRecurrenceKind(
2980 RedPhi.getRecurrenceKind());
2981 });
2982}
2983
2984/// Returns true if the VPlan contains a VPConditionalInductionPHIRecipe.
2989
2990/// Determine how to lower the epilogue for the vector epilogue loop.
2991/// Check if there are any conflicts that prevent tail-folding the epilogue.
2992/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
2993/// otherwise CM_EpilogueAllowed.
2995 const VectorizeOptions &Opts, const LoopVectorizationCostModel &MainCM,
2996 const Loop *L, OptimizationRemarkEmitter *ORE,
2998 const TargetTransformInfo *TTI) {
2999 // Epilogue TF is only enabled when explicitly requested via command line.
3000 if (Opts.epilogue_tail_folding_policy != TailFoldingPolicyTy::PreferFoldTail)
3001 return CM_EpilogueAllowed;
3002
3003 if (!Opts.enable_epilogue_vectorization) {
3005 "Options conflict, epilogue vectorization is disallowed while "
3006 "epilogue tail-folding allowed!",
3007 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3008 return CM_EpilogueAllowed;
3009 }
3010
3011 if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
3012 reportVectorizationInfo("For now, epilogue tail-folding can't be "
3013 "applied without forced main/epilogue loop VF",
3014 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3015 return CM_EpilogueAllowed;
3016 }
3017
3019 reportVectorizationInfo("For now, epilogue tail-folding can't be applied "
3020 "when VF of the main loop <= VF of the epilogue",
3021 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3022 return CM_EpilogueAllowed;
3023 }
3024
3025 if (!L->isInnermost()) {
3027 "Epilogue tail-folding is not supported for outer loop",
3028 "InvalidTailFoldedEpilogue", ORE, L);
3029 return CM_EpilogueAllowed;
3030 }
3031
3032 // If scalar epilogue is explicitly required, we can't apply TF.
3033 if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
3035 "Epilogue tail-folding can't be applied because scalar epilogue is "
3036 "required. Fall back to a normal epilogue",
3037 "InvalidTailFoldedEpilogue", ORE, L);
3038 return CM_EpilogueAllowed;
3039 }
3040
3041 // If having epilogue is NOT allowed, then no epilogue to apply TF for.
3042 if (!MainCM.isEpilogueAllowed()) {
3043 reportVectorizationInfo("Not applying tail-folding to the epilogue, since "
3044 "no epilogue is allowed.",
3045 "InvalidTailFoldedEpilogue", ORE, L);
3046 return CM_EpilogueAllowed;
3047 }
3048
3049 if (L->getExitingBlock() != L->getLoopLatch() ||
3052 "Epilogue tail-folding is not supported yet for early-exit loops",
3053 "InvalidTailFoldedEpilogue", ORE, L);
3054 return CM_EpilogueAllowed;
3055 }
3056
3057 // The epilogue reuses the main loop's interleave groups, so it can't be
3058 // tail-folded if the target can't mask interleaved accesses.
3059 // TODO: Add support once the epilogue has its own IAI, separate from the main
3060 // loop's.
3061 if (MainCM.InterleaveInfo.hasGroups() &&
3064 "Epilogue tail-folding is not supported with interleaved accesses "
3065 "when masking them isn't supported",
3066 "InvalidTailFoldedEpilogue", ORE, L);
3067 return CM_EpilogueAllowed;
3068 }
3069
3070 if (Opts.force_partial_aliasing_vectorization) {
3072 "Epilogue tail-folding is not supported with alias masking",
3073 "InvalidTailFoldedEpilogue", ORE, L);
3074 return CM_EpilogueAllowed;
3075 }
3076
3077 if (!LVL.getReductionVars().empty()) {
3079 "Epilogue tail-folding is not supported with reductions",
3080 "InvalidTailFoldedEpilogue", ORE, L);
3081 return CM_EpilogueAllowed;
3082 }
3083
3084 if (!LVL.getFixedOrderRecurrences().empty()) {
3086 "Epilogue tail-folding is not supported with fixed-order recurrence",
3087 "InvalidTailFoldedEpilogue", ORE, L);
3088 return CM_EpilogueAllowed;
3089 }
3090
3091 // TODO: This is conservative: it rejects any target that prefers EVL, even
3092 // for fixed-width epilogue VFs where EVL won't be chosen. Move this check to
3093 // where the epilogue's TF style is known once epilogue TF is supported.
3094 TailFoldingStyle TFStyle = Opts.force_tail_folding_style.value_or(
3095 TTI->getPreferredTailFoldingStyle());
3096 // TODO: Remove once EVL recipes support cloning.
3097 if (TFStyle == TailFoldingStyle::DataWithEVL) {
3098 reportVectorizationInfo("Epilogue tail-folding is not supported yet with "
3099 "EVL-based tail-folding",
3100 "UnsupportedEpilogueTailFoldingPolicy", ORE, L);
3101 return CM_EpilogueAllowed;
3102 }
3103
3104 // We can apply tail-folding on the vectorized epilogue loop.
3106}
3107
3109 const ElementCount VF, const unsigned IC) const {
3110 // FIXME: We need a much better cost-model to take different parameters such
3111 // as register pressure, code size increase and cost of extra branches into
3112 // account. For now we apply a very crude heuristic and only consider loops
3113 // with vectorization factors larger than a certain value.
3114
3115 // Allow the target to opt out.
3116 if (!TTI.preferEpilogueVectorization(VF * IC))
3117 return false;
3118
3119 unsigned MinVFThreshold = Opts.epilogue_vectorization_minimum_VF.value_or(
3120 TTI.getEpilogueVectorizationMinVF());
3121 return estimateElementCount(VF * IC, getVScaleForTuning()) >= MinVFThreshold;
3122}
3123
3125 VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
3126 bool ScalarEpilogueAllowed) {
3127 if (!Opts.enable_epilogue_vectorization) {
3128 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
3129 return nullptr;
3130 }
3131
3132 if (!ScalarEpilogueAllowed) {
3133 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
3134 "epilogue is allowed.\n");
3135 return nullptr;
3136 }
3137
3138 if (vputils::findIncomingAliasMask(MainPlan)) {
3139 LLVM_DEBUG(
3140 dbgs()
3141 << "LEV: Epilogue vectorization not supported with alias masking.\n");
3142 return nullptr;
3143 }
3144
3145 // Not really a cost consideration, but check for unsupported cases here to
3146 // simplify the logic.
3147 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3148 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
3149 "is not a supported candidate.\n");
3150 return nullptr;
3151 }
3152
3153 if (hasForcedEpilogueVF()) {
3155 Config.getVScaleForTuning()) >=
3156 IC * estimateElementCount(MainLoopVF, Config.getVScaleForTuning())) {
3157 // Note that the main loop leaves IC * MainLoopVF iterations iff a scalar
3158 // epilogue is required, but then the epilogue loop also requires a scalar
3159 // epilogue.
3160 LLVM_DEBUG(dbgs() << "LEV: Forced epilogue VF results in dead epilogue "
3161 "vector loop, skipping vectorizing epilogue.\n");
3162 return nullptr;
3163 }
3164
3165 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
3167 std::unique_ptr<VPlan> Clone(
3169 Clone->setVF(EpilogueVectorizationForceVF);
3170 return Clone;
3171 }
3172
3173 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
3174 "viable.\n");
3175 return nullptr;
3176 }
3177
3178 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3179 LLVM_DEBUG(
3180 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
3181 return nullptr;
3182 }
3183
3184 if (!Config.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
3185 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
3186 "this loop\n");
3187 return nullptr;
3188 }
3189
3190 // Check if a plan's vector loop processes fewer iterations than VF (e.g. when
3191 // interleave groups have been narrowed) narrowInterleaveGroups) and return
3192 // the adjusted, effective VF.
3193 using namespace VPlanPatternMatch;
3194 auto GetEffectiveVF = [](VPlan &Plan, ElementCount VF) -> ElementCount {
3195 auto *Exiting = Plan.getVectorLoopRegion()->getExitingBasicBlock();
3196 if (match(&Exiting->back(),
3197 m_BranchOnCount(m_Add(m_CanonicalIV(), m_Specific(&Plan.getUF())),
3198 m_VPValue())))
3199 return ElementCount::get(1, VF.isScalable());
3200 return VF;
3201 };
3202
3203 // Check if the main loop processes fewer than MainLoopVF elements per
3204 // iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
3205 // as needed.
3206 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3207
3208 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
3209 // the main loop handles 8 lanes per iteration. We could still benefit from
3210 // vectorizing the epilogue loop with VF=4.
3211 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
3212 estimateElementCount(MainLoopVF, Config.getVScaleForTuning()));
3213
3214 Type *TCType = Legal->getWidestInductionType();
3215 const SCEV *RemainingIterations = nullptr;
3216 unsigned MaxTripCount = 0;
3217 const SCEV *TC = vputils::getSCEVExprForVPValue(MainPlan.getTripCount(), PSE);
3218 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
3219 const SCEV *KnownMinTC;
3220 bool ScalableTC = match(TC, m_scev_c_Mul(m_SCEV(KnownMinTC), m_SCEVVScale()));
3221 bool ScalableRemIter = false;
3222 ScalarEvolution &SE = *PSE.getSE();
3223 // Use versions of TC and VF in which both are either scalable or fixed.
3224 if (ScalableTC == MainLoopVF.isScalable()) {
3225 ScalableRemIter = ScalableTC;
3226 RemainingIterations =
3227 SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
3228 } else if (ScalableTC) {
3229 const SCEV *EstimatedTC = SE.getMulExpr(
3230 KnownMinTC,
3231 SE.getConstant(TCType, Config.getVScaleForTuning().value_or(1)));
3232 RemainingIterations = SE.getURemExpr(
3233 EstimatedTC, SE.getElementCount(TCType, MainLoopVF * IC));
3234 } else
3235 RemainingIterations =
3236 SE.getURemExpr(TC, SE.getElementCount(TCType, EstimatedRuntimeVF * IC));
3237
3238 // No iterations left to process in the epilogue.
3239 if (RemainingIterations->isZero())
3240 return nullptr;
3241
3242 if (MainLoopVF.isFixed()) {
3243 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
3244 if (SE.isKnownPredicate(CmpInst::ICMP_ULT, RemainingIterations,
3245 SE.getConstant(TCType, MaxTripCount))) {
3246 MaxTripCount = SE.getUnsignedRangeMax(RemainingIterations).getZExtValue();
3247 }
3248 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
3249 << MaxTripCount << "\n");
3250 }
3251
3252 auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
3253 return SE.isKnownPredicate(CmpInst::ICMP_UGT, VF, RemIter);
3254 };
3256 VPlan *BestPlan = nullptr;
3257 for (auto &NextVF : ProfitableVFs) {
3258 // Skip candidate VFs without a corresponding VPlan.
3259 if (!hasPlanWithVF(NextVF.Width))
3260 continue;
3261
3262 VPlan &CurrentPlan = getPlanFor(NextVF.Width);
3263 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3264 // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
3265 // the VF of the main loop.
3266 if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
3267 ElementCount::isKnownGT(EffectiveVF, EstimatedRuntimeVF)) ||
3268 ElementCount::isKnownGT(EffectiveVF, MainLoopVF))
3269 continue;
3270
3271 // If EffectiveVF is greater than the number of remaining iterations, the
3272 // epilogue loop would be dead. Skip such factors. If the epilogue plan
3273 // also has narrowed interleave groups, use the effective VF since
3274 // the epilogue step will be reduced to its IC.
3275 // TODO: We should also consider comparing against a scalable
3276 // RemainingIterations when SCEV be able to evaluate non-canonical
3277 // vscale-based expressions.
3278 if (!ScalableRemIter) {
3279 // Handle the case where EffectiveVF and RemainingIterations are in
3280 // different numerical spaces.
3281 if (EffectiveVF.isScalable())
3282 EffectiveVF = ElementCount::getFixed(
3283 estimateElementCount(EffectiveVF, Config.getVScaleForTuning()));
3284 if (SkipVF(SE.getElementCount(TCType, EffectiveVF), RemainingIterations))
3285 continue;
3286 }
3287
3288 if (Result.Width.isScalar() ||
3289 isMoreProfitable(NextVF, Result, MaxTripCount,
3290 !MainPlan.hasTailFolded(),
3291 /*IsEpilogue*/ true)) {
3292 Result = NextVF;
3293 BestPlan = &CurrentPlan;
3294 }
3295 }
3296
3297 if (!BestPlan)
3298 return nullptr;
3299
3300 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
3301 << Result.Width << "\n");
3302 std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
3303 Clone->setVF(Result.Width);
3304 return Clone;
3305}
3306
3307unsigned
3309 InstructionCost LoopCost) {
3310 // -- The interleave heuristics --
3311 // We interleave the loop in order to expose ILP and reduce the loop overhead.
3312 // There are many micro-architectural considerations that we can't predict
3313 // at this level. For example, frontend pressure (on decode or fetch) due to
3314 // code size, or the number and capabilities of the execution ports.
3315 //
3316 // We use the following heuristics to select the interleave count:
3317 // 1. If the code has reductions, then we interleave to break the cross
3318 // iteration dependency.
3319 // 2. If the loop is really small, then we interleave to reduce the loop
3320 // overhead.
3321 // 3. We don't interleave if we think that we will spill registers to memory
3322 // due to the increased register pressure.
3323
3324 // Do not interleave tail-folded loops, as the overhead of multiple
3325 // instructions to calculate the predicate is likely not beneficial.
3326 // If an epilogue is not allowed for any other reason, do not interleave.
3327 if (!CM->isEpilogueAllowed())
3328 return 1;
3329
3332 LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
3333 "Unroll factor forced to be 1.\n");
3334 return 1;
3335 }
3336
3337 // We used the distance for the interleave count.
3338 if (!Legal->isSafeForAnyVectorWidth())
3339 return 1;
3340
3341 // We don't attempt to perform interleaving for loops with uncountable early
3342 // exits because the VPInstruction::AnyOf code cannot currently handle
3343 // multiple parts.
3344 if (Plan.hasEarlyExit())
3345 return 1;
3346
3347 // TODO: Conditional inductions don't support interleaving yet.
3349 return 1;
3350
3351 const bool HasReductions =
3354
3355 // FIXME: implement interleaving for FindLast transform correctly.
3356 if (hasFindLastReductionPhi(Plan))
3357 return 1;
3358
3359 VPRegisterUsage R = calculateRegisterUsageForPlan(Plan, {VF}, TTI)[0];
3360
3361 // If we did not calculate the cost for VF (because the user selected the VF)
3362 // then we calculate the cost of VF here.
3363 if (LoopCost == 0) {
3364 if (VF.isScalar())
3365 LoopCost = CM->expectedCost(VF);
3366 else
3367 LoopCost = cost(Plan, VF, &R, *CM);
3368 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
3369
3370 // Loop body is free and there is no need for interleaving.
3371 if (LoopCost == 0)
3372 return 1;
3373 }
3374
3375 // We divide by these constants so assume that we have at least one
3376 // instruction that uses at least one register.
3377 for (auto &Pair : R.MaxLocalUsers) {
3378 Pair.second = std::max(Pair.second, 1U);
3379 }
3380
3381 // We calculate the interleave count using the following formula.
3382 // Subtract the number of loop invariants from the number of available
3383 // registers. These registers are used by all of the interleaved instances.
3384 // Next, divide the remaining registers by the number of registers that is
3385 // required by the loop, in order to estimate how many parallel instances
3386 // fit without causing spills. All of this is rounded down if necessary to be
3387 // a power of two. We want power of two interleave count to simplify any
3388 // addressing operations or alignment considerations.
3389 // We also want power of two interleave counts to ensure that the induction
3390 // variable of the vector loop wraps to zero, when tail is folded by masking;
3391 // this currently happens when OptForSize, in which case IC is set to 1 above.
3392 unsigned IC = UINT_MAX;
3393
3394 for (const auto &Pair : R.MaxLocalUsers) {
3395 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
3396 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
3397 << " registers of "
3398 << TTI.getRegisterClassName(Pair.first)
3399 << " register class\n");
3400 TargetNumRegisters = (VF.isScalar() ? Opts.force_target_num_scalar_regs
3401 : Opts.force_target_num_vector_regs)
3402 .value_or(TargetNumRegisters);
3403 unsigned MaxLocalUsers = Pair.second;
3404 unsigned LoopInvariantRegs = 0;
3405 if (R.LoopInvariantRegs.contains(Pair.first))
3406 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3407
3408 unsigned TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
3409 MaxLocalUsers);
3410 // Don't count the induction variable as interleaved.
3411 if (Opts.enable_ind_var_reg_heur) {
3412 TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
3413 std::max(1U, (MaxLocalUsers - 1)));
3414 }
3415
3416 IC = std::min(IC, TmpIC);
3417 }
3418
3419 // Clamp the interleave ranges to reasonable counts.
3420 bool HasUnorderedReductions =
3421 HasReductions &&
3424 [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3425 unsigned MaxInterleaveCount =
3426 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3427 LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
3428 << MaxInterleaveCount << "\n");
3429
3430 // Check if the user has overridden the max.
3431 MaxInterleaveCount = (VF.isScalar() ? Opts.force_target_max_scalar_interleave
3432 : Opts.force_target_max_vector_interleave)
3433 .value_or(MaxInterleaveCount);
3434
3435 // Try to get the exact trip count, or an estimate based on profiling data or
3436 // ConstantMax from PSE, failing that.
3437 auto BestKnownTC =
3438 getSmallBestKnownTC(Opts, PSE, OrigLoop,
3439 /*CanUseConstantMax=*/true,
3440 /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
3441
3442 // For fixed length VFs treat a scalable trip count as unknown.
3443 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
3444 // Re-evaluate trip counts and VFs to be in the same numerical space.
3445 unsigned AvailableTC =
3446 estimateElementCount(*BestKnownTC, Config.getVScaleForTuning());
3447 unsigned EstimatedVF =
3448 estimateElementCount(VF, Config.getVScaleForTuning());
3449
3450 // At least one iteration must be scalar when this constraint holds. So the
3451 // maximum available iterations for interleaving is one less.
3452 if (Plan.requiresScalarEpilogue())
3453 --AvailableTC;
3454
3455 unsigned InterleaveCountLB = bit_floor(std::max(
3456 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
3457
3458 if (getSmallConstantTripCount(PSE.getSE(), OrigLoop).isNonZero()) {
3459 // If the best known trip count is exact, we select between two
3460 // prospective ICs, where
3461 //
3462 // 1) the aggressive IC is capped by the trip count divided by VF
3463 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
3464 //
3465 // The final IC is selected in a way that the epilogue loop trip count is
3466 // minimized while maximizing the IC itself, so that we either run the
3467 // vector loop at least once if it generates a small epilogue loop, or
3468 // else we run the vector loop at least twice.
3469
3470 unsigned InterleaveCountUB = bit_floor(std::max(
3471 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
3472 MaxInterleaveCount = InterleaveCountLB;
3473
3474 if (InterleaveCountUB != InterleaveCountLB) {
3475 unsigned TailTripCountUB =
3476 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3477 unsigned TailTripCountLB =
3478 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3479 // If both produce same scalar tail, maximize the IC to do the same work
3480 // in fewer vector loop iterations
3481 if (TailTripCountUB == TailTripCountLB)
3482 MaxInterleaveCount = InterleaveCountUB;
3483 }
3484 } else {
3485 // If trip count is an estimated compile time constant, limit the
3486 // IC to be capped by the trip count divided by VF * 2, such that the
3487 // vector loop runs at least twice to make interleaving seem profitable
3488 // when there is an epilogue loop present. Since exact Trip count is not
3489 // known we choose to be conservative in our IC estimate.
3490 MaxInterleaveCount = InterleaveCountLB;
3491 }
3492 }
3493
3494 assert(MaxInterleaveCount > 0 &&
3495 "Maximum interleave count must be greater than 0");
3496
3497 // Clamp the calculated IC to be between the 1 and the max interleave count
3498 // that the target and trip count allows.
3499 if (IC > MaxInterleaveCount)
3500 IC = MaxInterleaveCount;
3501 else
3502 // Make sure IC is greater than 0.
3503 IC = std::max(1u, IC);
3504
3505 assert(IC > 0 && "Interleave count must be greater than 0.");
3506
3507 // FIXME: VF is required to be a vector only because there are several recipes
3508 // whose computeCost function asserts that the VF is a vector.
3509 if (Opts.enable_interleave_to_reduce_stalls && VF.isVector()) {
3510 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce stall cycles due to "
3511 "instruction latency.\n");
3512 // Interleaving a replicator region means we get extra copies of the blocks
3513 // in the region. Vector values used in those blocks are expected to later
3514 // be sunk to just before them, negating any reduction in stalls. Therefore
3515 // there's no gain to be had from interleaving.
3516 if (hasReplicatorRegion(Plan)) {
3517 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to replicator region.\n");
3518 return 1;
3519 }
3520 VPCostContext LatencyCtx(*TLI, Plan, *CM, Config,
3521 /*ReusePrintingSlotTracker=*/true);
3522 LatencyCtx.CostKind = TTI::TCK_Latency;
3523 VPCostContext ThroughputCtx(*TLI, Plan, *CM, Config,
3524 /*ReusePrintingSlotTracker=*/true);
3525 ThroughputCtx.CostKind = TTI::TCK_RecipThroughput;
3526 // Find the largest interleave count that will help with reducing stalls.
3527 unsigned StallsIC = 1;
3530 for (VPRecipeBase &R : *VPBB) {
3531 // Assuming that each value will be needed as soon as it's generated the
3532 // number of stall cycles is one less than the latency.
3533 InstructionCost StallCycles = R.cost(VF, LatencyCtx) - 1;
3534 if (Opts.force_target_load_latency > 0 && R.mayReadFromMemory())
3535 StallCycles = Opts.force_target_load_latency - 1;
3536 // Each interleaving above 1 will reduce the stalls by RecipThroughput,
3537 // so pick the interleaving that will reduce stalls to zero.
3538 InstructionCost RecipThroughput = R.cost(VF, ThroughputCtx);
3539 if (StallCycles.isValid() && RecipThroughput.isValid() &&
3540 StallCycles > 0 && RecipThroughput > 0) {
3541 unsigned ThisIC = bit_floor<uint64_t>(
3542 1 + (StallCycles / RecipThroughput).getValue());
3543 StallsIC = std::max(StallsIC, ThisIC);
3544 }
3545 }
3546 }
3547 if (StallsIC <= 1) {
3548 LLVM_DEBUG(
3549 dbgs() << "LV: Not interleaving as it wouldn't reduce stalls.\n");
3550 return 1;
3551 }
3552 IC = std::min(IC, StallsIC);
3553 return IC;
3554 }
3555
3556 // Interleave if we vectorized this loop and there is a reduction that could
3557 // benefit from interleaving.
3558 if (VF.isVector() && HasReductions) {
3559 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
3560 return IC;
3561 }
3562
3563 // For any scalar loop that either requires runtime checks or tail-folding we
3564 // are better off leaving this to the unroller. Note that if we've already
3565 // vectorized the loop we will have done the runtime check and so interleaving
3566 // won't require further checks.
3567 bool ScalarInterleavingRequiresPredication =
3568 (VF.isScalar() && any_of(OrigLoop->blocks(), [this](BasicBlock *BB) {
3569 return Legal->blockNeedsPredication(BB);
3570 }));
3571 bool ScalarInterleavingRequiresRuntimePointerCheck =
3572 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
3573
3574 // We want to interleave small loops in order to reduce the loop overhead and
3575 // potentially expose ILP opportunities.
3576 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
3577 << "LV: IC is " << IC << '\n'
3578 << "LV: VF is " << VF << '\n');
3579 const bool AggressivelyInterleave =
3580 TTI.enableAggressiveInterleaving(HasReductions);
3581 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3582 !ScalarInterleavingRequiresPredication &&
3583 LoopCost < Opts.small_loop_cost) {
3584 // We assume that the cost overhead is 1 and we use the cost model
3585 // to estimate the cost of the loop and interleave until the cost of the
3586 // loop overhead is about 5% of the cost of the loop.
3587 unsigned SmallIC =
3588 std::min(IC, (unsigned)llvm::bit_floor<uint64_t>(Opts.small_loop_cost /
3589 LoopCost.getValue()));
3590
3591 // Interleave until store/load ports (estimated by max interleave count) are
3592 // saturated.
3593 unsigned NumStores = 0;
3594 unsigned NumLoads = 0;
3597 for (VPRecipeBase &R : *VPBB) {
3599 NumLoads++;
3600 continue;
3601 }
3603 NumStores++;
3604 continue;
3605 }
3606
3607 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(&R)) {
3608 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
3609 NumStores += StoreOps;
3610 else
3611 NumLoads += InterleaveR->getNumDefinedValues();
3612 continue;
3613 }
3614 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
3615 NumLoads += isa<LoadInst>(RepR->getUnderlyingInstr());
3616 NumStores += isa<StoreInst>(RepR->getUnderlyingInstr());
3617 continue;
3618 }
3619 if (isa<VPHistogramRecipe>(&R)) {
3620 NumLoads++;
3621 NumStores++;
3622 continue;
3623 }
3624 }
3625 }
3626 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3627 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3628
3629 // There is little point in interleaving for reductions containing selects
3630 // and compares when VF=1 since it may just create more overhead than it's
3631 // worth for loops with small trip counts. This is because we still have to
3632 // do the final reduction after the loop.
3633 bool HasSelectCmpReductions =
3634 HasReductions &&
3637 [](VPReductionPHIRecipe &RedR) {
3638 return RecurrenceDescriptor::isAnyOfRecurrenceKind(
3639 RedR.getRecurrenceKind()) ||
3640 RecurrenceDescriptor::isFindIVRecurrenceKind(
3641 RedR.getRecurrenceKind());
3642 });
3643 if (HasSelectCmpReductions) {
3644 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
3645 return 1;
3646 }
3647
3648 // If we have a scalar reduction (vector reductions are already dealt with
3649 // by this point), we can increase the critical path length if the loop
3650 // we're interleaving is inside another loop. For tree-wise reductions
3651 // set the limit to 2, and for ordered reductions it's best to disable
3652 // interleaving entirely.
3653 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3654 bool HasOrderedReductions =
3657 [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3658 if (HasOrderedReductions) {
3659 LLVM_DEBUG(
3660 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
3661 return 1;
3662 }
3663
3664 unsigned F = Opts.max_nested_scalar_reduction_interleave;
3665 SmallIC = std::min(SmallIC, F);
3666 StoresIC = std::min(StoresIC, F);
3667 LoadsIC = std::min(LoadsIC, F);
3668 }
3669
3670 if (Opts.enable_loadstore_runtime_interleave &&
3671 std::max(StoresIC, LoadsIC) > SmallIC) {
3672 LLVM_DEBUG(
3673 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
3674 return std::max(StoresIC, LoadsIC);
3675 }
3676
3677 // If there are scalar reductions and TTI has enabled aggressive
3678 // interleaving for reductions, we will interleave to expose ILP.
3679 if (VF.isScalar() && AggressivelyInterleave) {
3680 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3681 // Interleave no less than SmallIC but not as aggressive as the normal IC
3682 // to satisfy the rare situation when resources are too limited.
3683 return std::max(IC / 2, SmallIC);
3684 }
3685
3686 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
3687 return SmallIC;
3688 }
3689
3690 // Interleave if this is a large loop (small loops are already dealt with by
3691 // this point) that could benefit from interleaving.
3692 if (AggressivelyInterleave) {
3693 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3694 return IC;
3695 }
3696
3697 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
3698 return 1;
3699}
3700
3702 Instruction *I, ElementCount VF) const {
3703 // TODO: Cost model for emulated masked load/store is completely
3704 // broken. This hack guides the cost model to use an artificially
3705 // high enough value to practically disable vectorization with such
3706 // operations, except where previously deployed legality hack allowed
3707 // using very low cost values. This is to avoid regressions coming simply
3708 // from moving "masked load/store" check from legality to cost model.
3709 // Masked Load/Gather emulation was previously never allowed.
3710 // Limited number of Masked Store/Scatter emulation was allowed.
3712 "Expecting a scalar emulated instruction");
3713 return isa<LoadInst>(I) ||
3714 (isa<StoreInst>(I) && NumPredStores > Opts.vectorize_num_stores_pred);
3715}
3716
3718 assert(VF.isVector() && "Expected VF >= 2");
3719 assert(!InstsToScalarize.contains(VF) &&
3720 !PredicatedBBsAfterVectorization.contains(VF) &&
3721 "Routine expected to be called once per VF");
3722
3723 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
3724 // not profitable to scalarize any instructions, the presence of VF in the
3725 // map will indicate that we've analyzed it already.
3726 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
3727
3728 // Find all the instructions that are scalar with predication in the loop and
3729 // determine if it would be better to not if-convert the blocks they are in.
3730 // If so, we also record the instructions to scalarize.
3731 for (BasicBlock *BB : TheLoop->blocks()) {
3733 continue;
3734 for (Instruction &I : *BB)
3735 if (isScalarWithPredication(&I, VF)) {
3736 ScalarCostsTy ScalarCosts;
3737 // Do not apply discount logic for:
3738 // 1. Scalars after vectorization, as there will only be a single copy
3739 // of the instruction.
3740 // 2. Scalable VF, as that would lead to invalid scalarization costs.
3741 // 3. Emulated masked memrefs, if a hacked cost is needed.
3742 // 4. Compressed loads/stores (which do not support scalarization)
3743 if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
3745 !Legal->isCompressedLoadOrStore(&I) &&
3746 computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
3747 for (const auto &[I, IC] : ScalarCosts)
3748 ScalarCostsVF.insert({I, IC});
3749 }
3750 // Remember that BB will remain after vectorization.
3751 PredicatedBBsAfterVectorization[VF].insert(BB);
3752 for (auto *Pred : predecessors(BB)) {
3753 if (Pred->getSingleSuccessor() == BB)
3754 PredicatedBBsAfterVectorization[VF].insert(Pred);
3755 }
3756 }
3757 }
3758}
3759
3760InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
3761 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
3762 assert(!isUniformAfterVectorization(PredInst, VF) &&
3763 "Instruction marked uniform-after-vectorization will be predicated");
3764
3765 // Initialize the discount to zero, meaning that the scalar version and the
3766 // vector version cost the same.
3767 InstructionCost Discount = 0;
3768
3769 // Holds instructions to analyze. The instructions we visit are mapped in
3770 // ScalarCosts. Those instructions are the ones that would be scalarized if
3771 // we find that the scalar version costs less.
3773
3774 // Returns true if the given instruction can be scalarized.
3775 auto CanBeScalarized = [&](Instruction *I) -> bool {
3776 // We only attempt to scalarize instructions forming a single-use chain
3777 // from the original predicated block that would otherwise be vectorized.
3778 // Although not strictly necessary, we give up on instructions we know will
3779 // already be scalar to avoid traversing chains that are unlikely to be
3780 // beneficial.
3781 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
3783 return false;
3784
3785 // If the instruction is scalar with predication, it will be analyzed
3786 // separately. We ignore it within the context of PredInst.
3787 if (isScalarWithPredication(I, VF))
3788 return false;
3789
3790 // If any of the instruction's operands are uniform after vectorization,
3791 // the instruction cannot be scalarized. This prevents, for example, a
3792 // masked load from being scalarized.
3793 //
3794 // We assume we will only emit a value for lane zero of an instruction
3795 // marked uniform after vectorization, rather than VF identical values.
3796 // Thus, if we scalarize an instruction that uses a uniform, we would
3797 // create uses of values corresponding to the lanes we aren't emitting code
3798 // for. This behavior can be changed by allowing getScalarValue to clone
3799 // the lane zero values for uniforms rather than asserting.
3800 for (Use &U : I->operands())
3801 if (auto *J = dyn_cast<Instruction>(U.get()))
3802 if (isUniformAfterVectorization(J, VF))
3803 return false;
3804
3805 // Otherwise, we can scalarize the instruction.
3806 return true;
3807 };
3808
3809 // Compute the expected cost discount from scalarizing the entire expression
3810 // feeding the predicated instruction. We currently only consider expressions
3811 // that are single-use instruction chains.
3812 Worklist.push_back(PredInst);
3813 while (!Worklist.empty()) {
3814 Instruction *I = Worklist.pop_back_val();
3815
3816 // If we've already analyzed the instruction, there's nothing to do.
3817 if (ScalarCosts.contains(I))
3818 continue;
3819
3820 // Cannot scalarize fixed-order recurrence phis at the moment.
3822 continue;
3823
3824 // Compute the cost of the vector instruction. Note that this cost already
3825 // includes the scalarization overhead of the predicated instruction.
3826 InstructionCost VectorCost = getInstructionCost(I, VF);
3827
3828 // Compute the cost of the scalarized instruction. This cost is the cost of
3829 // the instruction as if it wasn't if-converted and instead remained in the
3830 // predicated block. We will scale this cost by block probability after
3831 // computing the scalarization overhead.
3832 InstructionCost ScalarCost =
3834
3835 // Compute the scalarization overhead of needed insertelement instructions
3836 // and phi nodes.
3837 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
3838 Type *WideTy = toVectorizedTy(I->getType(), VF);
3839 for (Type *VectorTy : getContainedTypes(WideTy)) {
3840 ScalarCost += TTI.getScalarizationOverhead(
3842 /*Insert=*/true,
3843 /*Extract=*/false, Config.CostKind);
3844 }
3845 ScalarCost += VF.getFixedValue() *
3846 TTI.getCFInstrCost(Instruction::PHI, Config.CostKind);
3847 }
3848
3849 // Compute the scalarization overhead of needed extractelement
3850 // instructions. For each of the instruction's operands, if the operand can
3851 // be scalarized, add it to the worklist; otherwise, account for the
3852 // overhead.
3853 for (Use &U : I->operands())
3854 if (auto *J = dyn_cast<Instruction>(U.get())) {
3855 assert(canVectorizeTy(J->getType()) &&
3856 "Instruction has non-scalar type");
3857 if (CanBeScalarized(J))
3858 Worklist.push_back(J);
3859 else if (needsExtract(J, VF)) {
3860 Type *WideTy = toVectorizedTy(J->getType(), VF);
3861 for (Type *VectorTy : getContainedTypes(WideTy)) {
3862 ScalarCost += TTI.getScalarizationOverhead(
3863 cast<VectorType>(VectorTy),
3864 APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
3865 /*Extract*/ true, Config.CostKind);
3866 }
3867 }
3868 }
3869
3870 // Scale the total scalar cost by block probability.
3871 ScalarCost /= getPredBlockCostDivisor(Config.CostKind, I->getParent());
3872
3873 // Compute the discount. A non-negative discount means the vector version
3874 // of the instruction costs more, and scalarizing would be beneficial.
3875 Discount += VectorCost - ScalarCost;
3876 ScalarCosts[I] = ScalarCost;
3877 }
3878
3879 return Discount;
3880}
3881
3884 assert(VF.isScalar() && "must only be called for scalar VFs");
3885
3886 // For each block.
3887 for (BasicBlock *BB : TheLoop->blocks()) {
3888 InstructionCost BlockCost;
3889
3890 // For each instruction in the old loop.
3891 for (Instruction &I : *BB) {
3892 // Skip ignored values.
3893 if (ValuesToIgnore.count(&I) ||
3894 (VF.isVector() && VecValuesToIgnore.count(&I)))
3895 continue;
3896
3898
3899 // Check if we should override the cost.
3900 if (C.isValid() && Opts.force_target_instruction_cost)
3901 C = InstructionCost(*Opts.force_target_instruction_cost);
3902
3903 BlockCost += C;
3904 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
3905 << VF << " For instruction: " << I << '\n');
3906 }
3907
3908 // In the scalar loop, we may not always execute the predicated block, if it
3909 // is an if-else block. Thus, scale the block's cost by the probability of
3910 // executing it. getPredBlockCostDivisor will return 1 for blocks that are
3911 // only predicated by the header mask when folding the tail.
3912 Cost += BlockCost / getPredBlockCostDivisor(Config.CostKind, BB);
3913 }
3914
3915 return Cost;
3916}
3917
3918/// Gets the address access SCEV for Ptr, if it should be used for cost modeling
3919/// according to isAddressSCEVForCost.
3920///
3921/// This SCEV can be sent to the Target in order to estimate the address
3922/// calculation cost.
3924 Value *Ptr,
3926 const Loop *TheLoop) {
3927 const SCEV *Addr = PSE.getSCEV(Ptr);
3928 return vputils::isAddressSCEVForCost(Addr, *PSE.getSE(), TheLoop) ? Addr
3929 : nullptr;
3930}
3931
3933LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
3934 ElementCount VF) {
3935 assert(VF.isVector() &&
3936 "Scalarization cost of instruction implies vectorization.");
3937 if (VF.isScalable())
3939
3940 Type *ValTy = getLoadStoreType(I);
3941 auto *SE = PSE.getSE();
3942
3943 unsigned AS = getLoadStoreAddressSpace(I);
3945 Type *PtrTy = toVectorTy(Ptr->getType(), VF);
3946 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
3947 // that it is being called from this specific place.
3948
3949 // Figure out whether the access is strided and get the stride value
3950 // if it's known in compile time
3951 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, PSE, TheLoop);
3952
3953 // Get the cost of the scalar memory instruction and address computation.
3955 VF.getFixedValue() *
3956 TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV, Config.CostKind);
3957
3958 // Don't pass *I here, since it is scalar but will actually be part of a
3959 // vectorized loop where the user of it is a vectorized instruction.
3961 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
3962 Cost += VF.getFixedValue() *
3963 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
3964 AS, Config.CostKind, OpInfo);
3965
3966 // Get the overhead of the extractelement and insertelement instructions
3967 // we might create due to scalarization.
3968 Cost += getScalarizationOverhead(I, VF);
3969
3970 // If we have a predicated load/store, it will need extra i1 extracts and
3971 // conditional branches, but may not be executed for each vector lane. Scale
3972 // the cost by the probability of executing the predicated block.
3973 if (isPredicatedInst(I)) {
3974 Cost /= getPredBlockCostDivisor(Config.CostKind, I->getParent());
3975
3976 // Add the cost of an i1 extract and a branch
3977 auto *VecI1Ty =
3979 Cost += TTI.getScalarizationOverhead(
3980 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
3981 /*Insert=*/false, /*Extract=*/true, Config.CostKind);
3982 Cost += TTI.getCFInstrCost(Instruction::CondBr, Config.CostKind);
3983
3985 // Artificially setting to a high enough value to practically disable
3986 // vectorization with such operations.
3987 Cost = 3000000;
3988 }
3989
3990 return Cost;
3991}
3992
3993InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
3994 Instruction *I, ElementCount VF, InstWidening Kind) {
3995 assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
3996 "Expected a consecutive widening decision");
3997 Type *ValTy = getLoadStoreType(I);
3998 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
3999 unsigned AS = getLoadStoreAddressSpace(I);
4000
4003 if (isMaskRequired(I)) {
4004 Intrinsic::ID LoadIID = Intrinsic::masked_load;
4005 Intrinsic::ID StoreIID = Intrinsic::masked_store;
4006 if (Legal->isCompressedLoadOrStore(I)) {
4007 LoadIID = Intrinsic::masked_expandload;
4008 StoreIID = Intrinsic::masked_compressstore;
4009 }
4010
4011 unsigned IID = I->getOpcode() == Instruction::Load ? LoadIID : StoreIID;
4012 Cost += TTI.getMemIntrinsicInstrCost(
4013 MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4014 Config.CostKind);
4015 } else {
4016 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
4017 Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
4018 Config.CostKind, OpInfo, I);
4019 }
4020
4021 if (Kind == CM_Widen_Reverse)
4022 Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy,
4023 VectorTy, Config.CostKind, {}, 0);
4024 return Cost;
4025}
4026
4028LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
4029 ElementCount VF) const {
4030 assert(isUniformMemOp(*I, VF));
4031
4032 Type *ValTy = getLoadStoreType(I);
4034 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
4036 unsigned AS = getLoadStoreAddressSpace(I);
4037 if (isa<LoadInst>(I)) {
4038 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,
4039 Config.CostKind) +
4040 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
4041 Config.CostKind) +
4042 TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
4043 VectorTy, Config.CostKind);
4044 }
4045 StoreInst *SI = cast<StoreInst>(I);
4046
4047 bool IsLoopInvariantStoreValue = Legal->isInvariant(SI->getValueOperand());
4048 // TODO: We have existing tests that request the cost of extracting element
4049 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
4050 // the actual generated code, which involves extracting the last element of
4051 // a scalable vector where the lane to extract is unknown at compile time.
4053 TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, Config.CostKind) +
4054 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
4055 Config.CostKind);
4056 if (!IsLoopInvariantStoreValue)
4057 Cost += TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
4058 VectorTy, Config.CostKind, 0);
4059 return Cost;
4060}
4061
4063LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
4064 ElementCount VF) const {
4065 Type *ValTy = getLoadStoreType(I);
4066 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
4069 Type *PtrTy = Ptr->getType();
4070
4071 if (!isUniform(Ptr, VF))
4072 PtrTy = toVectorTy(PtrTy, VF);
4073
4074 unsigned IID = I->getOpcode() == Instruction::Load
4075 ? Intrinsic::masked_gather
4076 : Intrinsic::masked_scatter;
4077 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,
4078 Config.CostKind) +
4079 TTI.getMemIntrinsicInstrCost(
4080 MemIntrinsicCostAttributes(IID, VectorTy, Ptr, isMaskRequired(I),
4081 Alignment, I),
4082 Config.CostKind);
4083}
4084
4086LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
4087 ElementCount VF) const {
4088 const auto *Group = getInterleavedAccessGroup(I);
4089 assert(Group && "Fail to get an interleaved access group.");
4090
4091 Instruction *InsertPos = Group->getInsertPos();
4092 Type *ValTy = getLoadStoreType(InsertPos);
4093 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
4094 unsigned AS = getLoadStoreAddressSpace(InsertPos);
4095
4096 unsigned InterleaveFactor = Group->getFactor();
4097 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
4098
4099 // Holds the indices of existing members in the interleaved group.
4100 SmallVector<unsigned, 4> Indices;
4101 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
4102 if (Group->getMember(IF))
4103 Indices.push_back(IF);
4104
4105 // Calculate the cost of the whole interleaved group.
4106 bool UseMaskForGaps =
4107 (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
4108 (isa<StoreInst>(I) && !Group->isFull());
4109 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
4110 InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
4111 Group->getAlign(), AS, Config.CostKind, isMaskRequired(I),
4112 UseMaskForGaps);
4113
4114 if (Group->isReverse()) {
4115 // TODO: Add support for reversed masked interleaved access.
4117 "Reverse masked interleaved access not supported.");
4118 Cost += Group->getNumMembers() *
4119 TTI.getShuffleCost(TargetTransformInfo::SK_Reverse, VectorTy,
4120 VectorTy, Config.CostKind, {}, 0);
4121 }
4122 return Cost;
4123}
4124
4126LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
4127 ElementCount VF) {
4128 // Calculate scalar cost only. Vectorization cost should be ready at this
4129 // moment.
4130 if (VF.isScalar()) {
4131 Type *ValTy = getLoadStoreType(I);
4134 unsigned AS = getLoadStoreAddressSpace(I);
4135
4136 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
4137 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,
4138 Config.CostKind) +
4139 TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS,
4140 Config.CostKind, OpInfo, I);
4141 }
4142 return getWideningCost(I, VF);
4143}
4144
4146LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
4147 ElementCount VF) const {
4148
4149 // There is no mechanism yet to create a scalable scalarization loop,
4150 // so this is currently Invalid.
4151 if (VF.isScalable())
4153
4154 if (VF.isScalar())
4155 return 0;
4156
4158 Type *RetTy = toVectorizedTy(I->getType(), VF);
4159 if (!RetTy->isVoidTy() &&
4160 (!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore())) {
4161
4163 if (isa<LoadInst>(I))
4164 VIC = TTI::VectorInstrContext::Load;
4165 else if (isa<StoreInst>(I))
4166 VIC = TTI::VectorInstrContext::Store;
4167
4168 for (Type *VectorTy : getContainedTypes(RetTy)) {
4169 Cost += TTI.getScalarizationOverhead(
4171 /*Insert=*/true, /*Extract=*/false, Config.CostKind,
4172 /*ForPoisonSrc=*/true, {}, VIC);
4173 }
4174 }
4175
4176 // Some targets keep addresses scalar.
4177 if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
4178 return Cost;
4179
4180 // Some targets support efficient element stores.
4181 if (isa<StoreInst>(I) && TTI.supportsEfficientVectorElementLoadStore())
4182 return Cost;
4183
4184 // Collect operands to consider.
4185 CallInst *CI = dyn_cast<CallInst>(I);
4186 Instruction::op_range Ops = CI ? CI->args() : I->operands();
4187
4188 // Skip operands that do not require extraction/scalarization and do not incur
4189 // any overhead.
4191 SmallPtrSet<Value *, 4> SeenOps;
4192 for (Value *Op : Ops)
4193 if (needsExtract(Op, VF) && SeenOps.insert(Op).second)
4194 Tys.push_back(maybeVectorizeType(Op->getType(), VF));
4195
4197 ? TTI::VectorInstrContext::Store
4199 return Cost +
4200 TTI.getOperandsScalarizationOverhead(Tys, Config.CostKind, OperandVIC);
4201}
4202
4204 if (VF.isScalar())
4205 return;
4206
4207 // TODO: We should generate better code and update the cost model for
4208 // predicated uniform stores. Today they are treated as any other
4209 // predicated store (see added test cases in
4210 // invariant-store-vectorization.ll).
4211 NumPredStores = 0;
4212 for (BasicBlock *BB : TheLoop->blocks())
4213 for (Instruction &I : *BB)
4215 ++NumPredStores;
4216
4217 for (BasicBlock *BB : TheLoop->blocks()) {
4218 // For each instruction in the old loop.
4219 for (Instruction &I : *BB) {
4221
4222 // Ignore if no pointer found or a widening decision has already been
4223 // made.
4224 if (!Ptr || (getWideningDecision(&I, VF) != CM_Unknown))
4225 continue;
4226
4227 LLVM_DEBUG(dbgs() << "LV: Memory widening: calculating best strategy for "
4228 << I << '\n');
4229 if (isUniformMemOp(I, VF)) {
4230 auto IsLegalToScalarize = [&]() {
4231 if (!VF.isScalable())
4232 // Scalarization of fixed length vectors "just works".
4233 return true;
4234
4235 // We have dedicated lowering for unpredicated uniform loads and
4236 // stores. Note that even with tail folding we know that at least
4237 // one lane is active (i.e. generalized predication is not possible
4238 // here), and the logic below depends on this fact.
4239 if (!foldTailByMasking())
4240 return true;
4241
4242 // For scalable vectors, a uniform memop load is always
4243 // uniform-by-parts and we know how to scalarize that.
4244 if (isa<LoadInst>(I))
4245 return true;
4246
4247 // A uniform store isn't neccessarily uniform-by-part
4248 // and we can't assume scalarization.
4249 auto &SI = cast<StoreInst>(I);
4250 return TheLoop->isLoopInvariant(SI.getValueOperand());
4251 };
4252
4253 const InstructionCost GatherScatterCost =
4254 isLegalGatherOrScatter(&I, VF) ? getGatherScatterCost(&I, VF)
4256
4257 // Load: Scalar load + broadcast
4258 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
4259 // FIXME: This cost is a significant under-estimate for tail folded
4260 // memory ops.
4261 const InstructionCost ScalarizationCost =
4262 IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
4264
4265 // Choose better solution for the current VF, Note that Invalid
4266 // costs compare as maximumal large. If both are invalid, we get
4267 // scalable invalid which signals a failure and a vectorization abort.
4268 LLVM_DEBUG(dbgs() << "LV: Memory widening: uniform memory op has "
4269 "GatherScatterCost = "
4270 << GatherScatterCost << ", ScalarizationCost = "
4271 << ScalarizationCost << '\n');
4272 if (GatherScatterCost < ScalarizationCost)
4273 setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
4274 else
4275 setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
4276 continue;
4277 }
4278
4279 // We assume that widening is the best solution when possible.
4280 if (std::optional<InstWidening> Decision =
4282 InstructionCost WidenCost = getConsecutiveMemOpCost(&I, VF, *Decision);
4283 LLVM_DEBUG(
4284 dbgs() << "LV: Memory widening: can be widened normally with cost "
4285 << WidenCost << '\n');
4286 setWideningDecision(&I, VF, *Decision, WidenCost);
4287 continue;
4288 }
4289
4290 assert(!Legal->isCompressedLoadOrStore(&I) &&
4291 "compressed loads/stores should be handled in "
4292 "memoryInstructionCanBeWidened");
4293
4294 // Choose between Interleaving, Gather/Scatter or Scalarization.
4296 unsigned NumAccesses = 1;
4297 if (isAccessInterleaved(&I)) {
4298 const auto *Group = getInterleavedAccessGroup(&I);
4299 assert(Group && "Fail to get an interleaved access group.");
4300
4301 NumAccesses = Group->getNumMembers();
4303 InterleaveCost = getInterleaveGroupCost(&I, VF);
4304 }
4305
4306 InstructionCost GatherScatterCost =
4308 ? getGatherScatterCost(&I, VF) * NumAccesses
4310
4311 InstructionCost ScalarizationCost =
4312 getMemInstScalarizationCost(&I, VF) * NumAccesses;
4313
4314 // Choose better solution for the current VF,
4315 // write down this decision and use it during vectorization.
4317 InstWidening Decision;
4318 if (InterleaveCost <= GatherScatterCost &&
4319 InterleaveCost < ScalarizationCost) {
4320 Decision = CM_Interleave;
4321 Cost = InterleaveCost;
4322 } else if (GatherScatterCost < ScalarizationCost) {
4323 Decision = CM_GatherScatter;
4324 Cost = GatherScatterCost;
4325 } else {
4326 Decision = CM_Scalarize;
4327 Cost = ScalarizationCost;
4328 }
4329 LLVM_DEBUG(
4330 dbgs() << "LV: Memory widening: InterleaveCost = " << InterleaveCost
4331 << ", GatherScatterCost = " << GatherScatterCost
4332 << ", ScalarizationCost = " << ScalarizationCost << '\n');
4333
4334 // If the instructions belongs to an interleave group, the whole group
4335 // receives the same decision. The whole group receives the cost, but
4336 // the cost will actually be assigned to one instruction.
4337 if (const auto *Group = getInterleavedAccessGroup(&I)) {
4338 if (Decision == CM_Scalarize) {
4339 for (Instruction *I : Group->members())
4340 setWideningDecision(I, VF, Decision,
4341 getMemInstScalarizationCost(I, VF));
4342 } else {
4343 setWideningDecision(Group, VF, Decision, Cost);
4344 }
4345 } else
4346 setWideningDecision(&I, VF, Decision, Cost);
4347 }
4348 }
4349
4350 // Make sure that any load of address and any other address computation
4351 // remains scalar unless there is gather/scatter support. This avoids
4352 // inevitable extracts into address registers, and also has the benefit of
4353 // activating LSR more, since that pass can't optimize vectorized
4354 // addresses.
4355 if (TTI.prefersVectorizedAddressing())
4356 return;
4357
4358 // Start with all scalar pointer uses.
4360 for (BasicBlock *BB : TheLoop->blocks())
4361 for (Instruction &I : *BB) {
4362 Instruction *PtrDef =
4364 if (PtrDef && TheLoop->contains(PtrDef) &&
4366 AddrDefs.insert(PtrDef);
4367 }
4368
4369 // Add all instructions used to generate the addresses.
4371 append_range(Worklist, AddrDefs);
4372 while (!Worklist.empty()) {
4373 Instruction *I = Worklist.pop_back_val();
4374 for (auto &Op : I->operands())
4375 if (auto *InstOp = dyn_cast<Instruction>(Op))
4376 if (TheLoop->contains(InstOp) && !isa<PHINode>(InstOp) &&
4377 AddrDefs.insert(InstOp))
4378 Worklist.push_back(InstOp);
4379 }
4380
4381 auto UpdateMemOpUserCost = [this, VF](LoadInst *LI) {
4382 // If there are direct memory op users of the newly scalarized load,
4383 // their cost may have changed because there's no scalarization
4384 // overhead for the operand. Update it.
4385 for (User *U : LI->users()) {
4387 continue;
4389 continue;
4390 auto UI = cast<Instruction>(U);
4391 LLVM_DEBUG(
4392 dbgs() << "LV: Memory widening: updating decision for load user "
4393 << *UI << '\n');
4395 UI, VF, CM_Scalarize,
4396 getMemInstScalarizationCost(cast<Instruction>(U), VF));
4397 }
4398 };
4399 for (auto *I : AddrDefs) {
4400 if (isa<LoadInst>(I)) {
4401 // Setting the desired widening decision should ideally be handled in
4402 // by cost functions, but since this involves the task of finding out
4403 // if the loaded register is involved in an address computation, it is
4404 // instead changed here when we know this is the case.
4405 InstWidening Decision = getWideningDecision(I, VF);
4406 if (!isPredicatedInst(I) &&
4407 (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
4408 (!isUniformMemOp(*I, VF) && Decision == CM_Scalarize))) {
4409 // Scalarize a widened load of address or update the cost of a scalar
4410 // load of an address.
4411 LLVM_DEBUG(dbgs() << "LV: Memory widening: updating decision for load "
4412 << *I << '\n');
4414 I, VF, CM_Scalarize,
4415 (VF.getKnownMinValue() *
4416 getMemoryInstructionCost(I, ElementCount::getFixed(1))));
4417 UpdateMemOpUserCost(cast<LoadInst>(I));
4418 } else if (const auto *Group = getInterleavedAccessGroup(I)) {
4419 // Scalarize all members of this interleaved group when any member
4420 // is used as an address. The address-used load skips scalarization
4421 // overhead, other members include it.
4422 for (Instruction *Member : Group->members()) {
4423 InstructionCost Cost = AddrDefs.contains(Member)
4424 ? (VF.getKnownMinValue() *
4425 getMemoryInstructionCost(
4426 Member, ElementCount::getFixed(1)))
4427 : getMemInstScalarizationCost(Member, VF);
4428 LLVM_DEBUG(
4429 dbgs()
4430 << "LV: Memory widening: updating decision for interleave member "
4431 << *Member << '\n');
4433 UpdateMemOpUserCost(cast<LoadInst>(Member));
4434 }
4435 }
4436 } else {
4437 // Cannot scalarize fixed-order recurrence phis at the moment.
4438 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
4439 continue;
4440
4441 // Make sure I gets scalarized and a cost estimate without
4442 // scalarization overhead.
4443 ForcedScalars[VF].insert(I);
4444 }
4445 }
4446}
4447
4449 if (!Legal->isInvariant(Op))
4450 return false;
4451 // Consider Op invariant, if it or its operands aren't predicated
4452 // instruction in the loop. In that case, it is not trivially hoistable.
4453 auto *OpI = dyn_cast<Instruction>(Op);
4454 return !OpI || !TheLoop->contains(OpI) ||
4455 (!isPredicatedInst(OpI) &&
4456 (!isa<PHINode>(OpI) || OpI->getParent() != TheLoop->getHeader()) &&
4457 all_of(OpI->operands(),
4458 [this](Value *Op) { return shouldConsiderInvariant(Op); }));
4459}
4460
4463 ElementCount VF) {
4464 // If we know that this instruction will remain uniform, check the cost of
4465 // the scalar version.
4467 VF = ElementCount::getFixed(1);
4468
4469 if (VF.isVector() && isProfitableToScalarize(I, VF))
4470 return InstsToScalarize[VF][I];
4471
4472 // Forced scalars do not have any scalarization overhead.
4473 auto ForcedScalar = ForcedScalars.find(VF);
4474 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
4475 auto InstSet = ForcedScalar->second;
4476 if (InstSet.count(I))
4478 VF.getKnownMinValue();
4479 }
4480
4481 const auto &MinBWs = Config.getMinimalBitwidths();
4482 uint64_t InstrMinBWs = MinBWs.lookup(I);
4483 Type *RetTy = I->getType();
4485 RetTy = IntegerType::get(RetTy->getContext(), InstrMinBWs);
4486 auto *SE = PSE.getSE();
4487
4488 Type *VectorTy;
4489 if (isScalarAfterVectorization(I, VF)) {
4490 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
4491 [this](Instruction *I, ElementCount VF) -> bool {
4492 if (VF.isScalar())
4493 return true;
4494
4495 auto Scalarized = InstsToScalarize.find(VF);
4496 assert(Scalarized != InstsToScalarize.end() &&
4497 "VF not yet analyzed for scalarization profitability");
4498 return !Scalarized->second.count(I) &&
4499 llvm::all_of(I->users(), [&](User *U) {
4500 auto *UI = cast<Instruction>(U);
4501 return !Scalarized->second.count(UI);
4502 });
4503 };
4504
4505 // With the exception of GEPs and PHIs, after scalarization there should
4506 // only be one copy of the instruction generated in the loop. This is
4507 // because the VF is either 1, or any instructions that need scalarizing
4508 // have already been dealt with by the time we get here. As a result,
4509 // it means we don't have to multiply the instruction cost by VF.
4510 assert(I->getOpcode() == Instruction::GetElementPtr ||
4511 I->getOpcode() == Instruction::PHI ||
4512 (I->getOpcode() == Instruction::BitCast &&
4513 I->getType()->isPointerTy()) ||
4514 HasSingleCopyAfterVectorization(I, VF));
4515 VectorTy = RetTy;
4516 } else
4517 VectorTy = toVectorizedTy(RetTy, VF);
4518
4519 if (VF.isVector() && VectorTy->isVectorTy() &&
4520 !TTI.getNumberOfParts(VectorTy))
4522
4523 // TODO: We need to estimate the cost of intrinsic calls.
4524 switch (I->getOpcode()) {
4525 case Instruction::GetElementPtr:
4526 // We mark this instruction as zero-cost because the cost of GEPs in
4527 // vectorized code depends on whether the corresponding memory instruction
4528 // is scalarized or not. Therefore, we handle GEPs with the memory
4529 // instruction cost.
4530 return 0;
4531 case Instruction::UncondBr:
4532 case Instruction::CondBr: {
4533 // In cases of scalarized and predicated instructions, there will be VF
4534 // predicated blocks in the vectorized loop. Each branch around these
4535 // blocks requires also an extract of its vector compare i1 element.
4536 // Note that the conditional branch from the loop latch will be replaced by
4537 // a single branch controlling the loop, so there is no extra overhead from
4538 // scalarization.
4539 bool ScalarPredicatedBB = false;
4541 if (VF.isVector() && BI &&
4542 (PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(0)) ||
4543 PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(1))) &&
4544 BI->getParent() != TheLoop->getLoopLatch())
4545 ScalarPredicatedBB = true;
4546
4547 if (ScalarPredicatedBB) {
4548 // Not possible to scalarize scalable vector with predicated instructions.
4549 if (VF.isScalable())
4551 // Return cost for branches around scalarized and predicated blocks.
4552 auto *VecI1Ty =
4554 return (TTI.getScalarizationOverhead(
4555 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
4556 /*Insert*/ false, /*Extract*/ true, Config.CostKind) +
4557 (TTI.getCFInstrCost(Instruction::CondBr, Config.CostKind) *
4558 VF.getFixedValue()));
4559 }
4560
4561 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
4562 // The back-edge branch will remain, as will all scalar branches.
4563 return TTI.getCFInstrCost(Instruction::UncondBr, Config.CostKind);
4564
4565 // This branch will be eliminated by if-conversion.
4566 return 0;
4567 // Note: We currently assume zero cost for an unconditional branch inside
4568 // a predicated block since it will become a fall-through, although we
4569 // may decide in the future to call TTI for all branches.
4570 }
4571 case Instruction::Switch: {
4572 if (VF.isScalar())
4573 return TTI.getCFInstrCost(Instruction::Switch, Config.CostKind);
4574 auto *Switch = cast<SwitchInst>(I);
4575 return Switch->getNumCases() *
4576 TTI.getCmpSelInstrCost(
4577 Instruction::ICmp,
4578 toVectorTy(Switch->getCondition()->getType(), VF),
4579 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
4580 CmpInst::ICMP_EQ, Config.CostKind);
4581 }
4582 case Instruction::PHI: {
4583 auto *Phi = cast<PHINode>(I);
4584
4585 // First-order recurrences are replaced by vector shuffles inside the loop.
4586 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
4587 return TTI.getShuffleCost(
4589 cast<VectorType>(VectorTy), Config.CostKind, {}, -1);
4590 }
4591
4592 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
4593 // converted into select instructions. We require N - 1 selects per phi
4594 // node, where N is the number of incoming values.
4595 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
4596 Type *ResultTy = Phi->getType();
4597
4598 // All instructions in an Any-of reduction chain are narrowed to bool.
4599 // Check if that is the case for this phi node.
4600 auto *HeaderUser = find_singleton<PHINode>(
4601 Phi->users(), [this](User *U, bool) -> PHINode * {
4602 auto *Phi = dyn_cast<PHINode>(U);
4603 if (Phi && Phi->getParent() == TheLoop->getHeader())
4604 return Phi;
4605 return nullptr;
4606 });
4607 if (HeaderUser) {
4608 auto &ReductionVars = Legal->getReductionVars();
4609 auto Iter = ReductionVars.find(HeaderUser);
4610 if (Iter != ReductionVars.end() &&
4612 Iter->second.getRecurrenceKind()))
4613 ResultTy = Type::getInt1Ty(Phi->getContext());
4614 }
4615 return (Phi->getNumIncomingValues() - 1) *
4616 TTI.getCmpSelInstrCost(
4617 Instruction::Select, toVectorTy(ResultTy, VF),
4618 toVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
4619 CmpInst::BAD_ICMP_PREDICATE, Config.CostKind);
4620 }
4621
4622 // When tail folding with EVL, if the phi is part of an out of loop
4623 // reduction then it will be transformed into a wide vp_merge.
4624 if (VF.isVector() && foldTailWithEVL() &&
4625 Legal->getReductionVars().contains(Phi) &&
4626 !Config.isInLoopReduction(Phi)) {
4628 Intrinsic::vp_merge, toVectorTy(Phi->getType(), VF),
4629 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
4630 return TTI.getIntrinsicInstrCost(ICA, Config.CostKind);
4631 }
4632
4633 return TTI.getCFInstrCost(Instruction::PHI, Config.CostKind);
4634 }
4635 case Instruction::UDiv:
4636 case Instruction::SDiv:
4637 case Instruction::URem:
4638 case Instruction::SRem:
4639 if (VF.isVector() && isPredicatedInst(I)) {
4640 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
4641 return isDivRemScalarWithPredication(ScalarCost, MaskedCost) ? ScalarCost
4642 : MaskedCost;
4643 }
4644 // We've proven all lanes safe to speculate, fall through.
4645 [[fallthrough]];
4646 case Instruction::Add:
4647 case Instruction::Sub: {
4648 const HistogramInfo *HGram = Legal->getHistogramInfo(I);
4649 if (HGram && VF.isVector()) {
4650 // Assume that a non-constant update value (or a constant != 1) requires
4651 // a multiply, and add that into the cost.
4653 ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
4654 if (!RHS || RHS->getZExtValue() != 1)
4655 MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
4656 Config.CostKind);
4657
4658 // Find the cost of the histogram operation itself.
4659 Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
4660 Type *ScalarTy = I->getType();
4661 Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
4662 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
4663 Type::getVoidTy(I->getContext()),
4664 {PtrTy, ScalarTy, MaskTy});
4665
4666 // Add the costs together with the add/sub operation.
4667 return TTI.getIntrinsicInstrCost(ICA, Config.CostKind) + MulCost +
4668 TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy,
4669 Config.CostKind);
4670 }
4671 [[fallthrough]];
4672 }
4673 case Instruction::FAdd:
4674 case Instruction::FSub:
4675 case Instruction::Mul:
4676 case Instruction::FMul:
4677 case Instruction::FDiv:
4678 case Instruction::FRem:
4679 case Instruction::Shl:
4680 case Instruction::LShr:
4681 case Instruction::AShr:
4682 case Instruction::And:
4683 case Instruction::Or:
4684 case Instruction::Xor: {
4685 // If we're speculating on the stride being 1, the multiplication may
4686 // fold away. We can generalize this for all operations using the notion
4687 // of neutral elements. (TODO)
4688 if (I->getOpcode() == Instruction::Mul &&
4689 ((TheLoop->isLoopInvariant(I->getOperand(0)) &&
4690 PSE.getSCEV(I->getOperand(0))->isOne()) ||
4691 (TheLoop->isLoopInvariant(I->getOperand(1)) &&
4692 PSE.getSCEV(I->getOperand(1))->isOne())))
4693 return 0;
4694
4695 // Certain instructions can be cheaper to vectorize if they have a constant
4696 // second vector operand. One example of this are shifts on x86.
4697 Value *Op2 = I->getOperand(1);
4698 if (!isa<Constant>(Op2) && TheLoop->isLoopInvariant(Op2) &&
4699 PSE.getSE()->isSCEVable(Op2->getType()) &&
4700 isa<SCEVConstant>(PSE.getSCEV(Op2))) {
4701 Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
4702 }
4703 auto Op2Info = TTI.getOperandInfo(Op2);
4704 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
4707
4708 SmallVector<const Value *, 4> Operands(I->operand_values());
4709 return TTI.getArithmeticInstrCost(
4710 I->getOpcode(), VectorTy, Config.CostKind,
4711 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4712 Op2Info, Operands, I, TLI);
4713 }
4714 case Instruction::FNeg: {
4715 return TTI.getArithmeticInstrCost(
4716 I->getOpcode(), VectorTy, Config.CostKind,
4717 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4718 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
4719 I->getOperand(0), I);
4720 }
4721 case Instruction::Select: {
4723 const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
4724 bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
4725
4726 const Value *Op0, *Op1;
4727 using namespace llvm::PatternMatch;
4728 if (!ScalarCond && (match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1))) ||
4729 match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))) {
4730 // select x, y, false --> x & y
4731 // select x, true, y --> x | y
4732 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(Op0);
4733 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(Op1);
4734 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
4735 Op1->getType()->getScalarSizeInBits() == 1);
4736
4737 return TTI.getArithmeticInstrCost(
4738 match(I, m_LogicalOr()) ? Instruction::Or : Instruction::And,
4739 VectorTy, Config.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1},
4740 I);
4741 }
4742
4743 Type *CondTy = SI->getCondition()->getType();
4744 if (!ScalarCond)
4745 CondTy = VectorType::get(CondTy, VF);
4746
4748 if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
4749 Pred = Cmp->getPredicate();
4750 return TTI.getCmpSelInstrCost(
4751 I->getOpcode(), VectorTy, CondTy, Pred, Config.CostKind,
4752 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, I);
4753 }
4754 case Instruction::ICmp:
4755 case Instruction::FCmp: {
4756 Type *ValTy = I->getOperand(0)->getType();
4757
4759 [[maybe_unused]] Instruction *Op0AsInstruction =
4760 dyn_cast<Instruction>(I->getOperand(0));
4761 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
4762 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
4763 "if both the operand and the compare are marked for "
4764 "truncation, they must have the same bitwidth");
4765 ValTy = IntegerType::get(ValTy->getContext(), InstrMinBWs);
4766 }
4767
4768 VectorTy = toVectorTy(ValTy, VF);
4769 return TTI.getCmpSelInstrCost(
4770 I->getOpcode(), VectorTy, CmpInst::makeCmpResultType(VectorTy),
4771 cast<CmpInst>(I)->getPredicate(), Config.CostKind,
4772 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, I);
4773 }
4774 case Instruction::Store:
4775 case Instruction::Load: {
4776 ElementCount Width = VF;
4777 if (Width.isVector()) {
4778 InstWidening Decision = getWideningDecision(I, Width);
4779 assert(Decision != CM_Unknown &&
4780 "CM decision should be taken at this point");
4783 if (Decision == CM_Scalarize)
4784 Width = ElementCount::getFixed(1);
4785 }
4786 VectorTy = toVectorTy(getLoadStoreType(I), Width);
4787 return getMemoryInstructionCost(I, VF);
4788 }
4789 case Instruction::BitCast:
4790 if (I->getType()->isPointerTy())
4791 return 0;
4792 [[fallthrough]];
4793 case Instruction::ZExt:
4794 case Instruction::SExt:
4795 case Instruction::FPToUI:
4796 case Instruction::FPToSI:
4797 case Instruction::FPExt:
4798 case Instruction::PtrToInt:
4799 case Instruction::IntToPtr:
4800 case Instruction::SIToFP:
4801 case Instruction::UIToFP:
4802 case Instruction::Trunc:
4803 case Instruction::FPTrunc: {
4804 // Computes the CastContextHint from a Load/Store instruction.
4805 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
4807 "Expected a load or a store!");
4808
4809 if (VF.isScalar() || !TheLoop->contains(I))
4811
4812 switch (getWideningDecision(I, VF)) {
4819 // TODO: Add 'Compressed' hint (not needed for any targets yet).
4820 if (Legal->isCompressedLoadOrStore(I))
4827 llvm_unreachable("Instr did not go through cost modelling?");
4830 }
4831
4832 llvm_unreachable("Unhandled case!");
4833 };
4834
4835 unsigned Opcode = I->getOpcode();
4837 // For Trunc, the context is the only user, which must be a StoreInst.
4838 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
4839 if (I->hasOneUse())
4840 if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
4841 CCH = ComputeCCH(Store);
4842 }
4843 // For Z/Sext, the context is the operand, which must be a LoadInst.
4844 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
4845 Opcode == Instruction::FPExt) {
4846 if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
4847 CCH = ComputeCCH(Load);
4848 }
4849
4850 // We optimize the truncation of induction variables having constant
4851 // integer steps. The cost of these truncations is the same as the scalar
4852 // operation.
4853 if (isOptimizableIVTruncate(I, VF)) {
4854 auto *Trunc = cast<TruncInst>(I);
4855 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
4856 Trunc->getSrcTy(), CCH, Config.CostKind,
4857 Trunc);
4858 }
4859
4860 Type *SrcScalarTy = I->getOperand(0)->getType();
4861 Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
4862 if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
4863 SrcScalarTy = IntegerType::get(SrcScalarTy->getContext(),
4864 MinBWs.lookup(Op0AsInstruction));
4865 Type *SrcVecTy =
4866 VectorTy->isVectorTy() ? toVectorTy(SrcScalarTy, VF) : SrcScalarTy;
4867
4869 // If the result type is <= the source type, there will be no extend
4870 // after truncating the users to the minimal required bitwidth.
4871 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
4872 (I->getOpcode() == Instruction::ZExt ||
4873 I->getOpcode() == Instruction::SExt))
4874 return 0;
4875 }
4876
4877 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH,
4878 Config.CostKind, I);
4879 }
4880 case Instruction::Call:
4881 return getVectorCallCost(cast<CallInst>(I), VF);
4882 case Instruction::ExtractValue:
4883 return TTI.getInstructionCost(I, Config.CostKind);
4884 case Instruction::Alloca:
4885 // We cannot easily widen alloca to a scalable alloca, as
4886 // the result would need to be a vector of pointers.
4887 if (VF.isScalable())
4889 return TTI.getArithmeticInstrCost(Instruction::Mul, RetTy, Config.CostKind);
4890 case Instruction::Freeze:
4891 return TTI::TCC_Free;
4892 default:
4893 // This opcode is unknown. Assume that it is the same as 'mul'.
4894 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
4895 Config.CostKind);
4896 } // end of switch.
4897}
4898
4900 // Ignore ephemeral values.
4902
4903 SmallVector<Value *, 4> DeadInterleavePointerOps;
4905
4906 // If a scalar epilogue is required, users outside the loop won't use
4907 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
4908 // that is the case.
4909 bool RequiresScalarEpilogue = requiresScalarEpilogue(true);
4910 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
4911 return RequiresScalarEpilogue &&
4912 !TheLoop->contains(cast<Instruction>(U)->getParent());
4913 };
4914
4915 // Returns true if the address of interleaved access I is used by its group.
4916 // VPlan uses the address of member 0 for store groups. For load groups, it
4917 // may use the address of the insert position instead.
4918 auto IsGroupAddress = [this](Instruction *I) {
4919 auto *Group = getInterleavedAccessGroup(I);
4920 return isa<StoreInst>(I) ? Group->getMember(0) == I
4921 : Group->getInsertPos() == I;
4922 };
4923
4925 DFS.perform(LI);
4926 for (BasicBlock *BB : reverse(make_range(DFS.beginRPO(), DFS.endRPO())))
4927 for (Instruction &I : reverse(*BB)) {
4928 if (VecValuesToIgnore.contains(&I) || ValuesToIgnore.contains(&I))
4929 continue;
4930
4931 // Add instructions that would be trivially dead and are only used by
4932 // values already ignored to DeadOps to seed worklist.
4934 all_of(I.users(), [this, IsLiveOutDead](User *U) {
4935 return VecValuesToIgnore.contains(U) ||
4936 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
4937 }))
4938 DeadOps.push_back(&I);
4939
4940 // For interleave groups, we only create a pointer for the start of the
4941 // interleave group. Queue up addresses of group members not used by the
4942 // group for further processing.
4943 if (isAccessInterleaved(&I)) {
4944 if (IsGroupAddress(&I))
4945 continue;
4946 Value *PointerOp = getLoadStorePointerOperand(&I);
4947 DeadInterleavePointerOps.push_back(PointerOp);
4948 }
4949
4950 // Queue branches for analysis. They are dead, if their successors only
4951 // contain dead instructions.
4952 if (isa<CondBrInst>(&I))
4953 DeadOps.push_back(&I);
4954 }
4955
4956 // Mark ops feeding interleave group members as free, if they are only used
4957 // by other dead computations.
4958 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
4959 auto *Op = dyn_cast<Instruction>(DeadInterleavePointerOps[I]);
4960 if (!Op || !TheLoop->contains(Op) ||
4961 any_of(Op->users(), [this, &IsGroupAddress](User *U) {
4962 Instruction *UI = cast<Instruction>(U);
4963 return !VecValuesToIgnore.contains(U) &&
4964 (!isAccessInterleaved(UI) || IsGroupAddress(UI));
4965 }))
4966 continue;
4967 VecValuesToIgnore.insert(Op);
4968 append_range(DeadInterleavePointerOps, Op->operands());
4969 }
4970
4971 // Mark ops that would be trivially dead and are only used by ignored
4972 // instructions as free.
4973 BasicBlock *Header = TheLoop->getHeader();
4974
4975 // Returns true if the block contains only dead instructions. Such blocks will
4976 // be removed by VPlan-to-VPlan transforms and won't be considered by the
4977 // VPlan-based cost model, so skip them in the legacy cost-model as well.
4978 auto IsEmptyBlock = [this](BasicBlock *BB) {
4979 return all_of(*BB, [this](Instruction &I) {
4980 return ValuesToIgnore.contains(&I) || VecValuesToIgnore.contains(&I) ||
4982 });
4983 };
4984 SmallPtrSet<Instruction *, 16> ProcessedDeadOps;
4985 for (unsigned I = 0; I != DeadOps.size(); ++I) {
4986 auto *Op = dyn_cast<Instruction>(DeadOps[I]);
4987
4988 // Check if the branch should be considered dead.
4989 if (auto *Br = dyn_cast_or_null<CondBrInst>(Op)) {
4990 BasicBlock *ThenBB = Br->getSuccessor(0);
4991 BasicBlock *ElseBB = Br->getSuccessor(1);
4992 // Don't considers branches leaving the loop for simplification.
4993 if (!TheLoop->contains(ThenBB) || !TheLoop->contains(ElseBB))
4994 continue;
4995 bool ThenEmpty = IsEmptyBlock(ThenBB);
4996 bool ElseEmpty = IsEmptyBlock(ElseBB);
4997 if ((ThenEmpty && ElseEmpty) ||
4998 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
4999 ElseBB->phis().empty()) ||
5000 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
5001 ThenBB->phis().empty())) {
5002 VecValuesToIgnore.insert(Br);
5003 DeadOps.push_back(Br->getCondition());
5004 }
5005 continue;
5006 }
5007
5008 // Skip any op that shouldn't be considered dead.
5009 if (!Op || !TheLoop->contains(Op) ||
5010 (isa<PHINode>(Op) && Op->getParent() == Header) ||
5012 any_of(Op->users(), [this, IsLiveOutDead](User *U) {
5013 return !VecValuesToIgnore.contains(U) &&
5014 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
5015 }))
5016 continue;
5017
5018 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
5019 // which applies for both scalar and vector versions. Otherwise it is only
5020 // dead in vector versions, so only add it to VecValuesToIgnore.
5021 bool BecameScalarDead = false;
5022 if (all_of(Op->users(),
5023 [this](User *U) { return ValuesToIgnore.contains(U); }))
5024 BecameScalarDead = ValuesToIgnore.insert(Op).second;
5025
5026 VecValuesToIgnore.insert(Op);
5027 // Shared operands may be queued more than once. Propagate deadness once,
5028 // and again if an instruction previously dead only in the vector loop
5029 // becomes dead in the scalar loop too.
5030 if (ProcessedDeadOps.insert(Op).second || BecameScalarDead)
5031 append_range(DeadOps, Op->operands());
5032 }
5033
5034 // Ignore type-promoting instructions we identified during reduction
5035 // detection.
5036 for (const RecurrenceDescriptor &RedDes :
5037 Legal->getReductionVars().values()) {
5038 VecValuesToIgnore.insert_range(RedDes.getCastInsts());
5039 }
5040 // Ignore type-casting instructions we identified during induction
5041 // detection.
5042 for (const InductionDescriptor &IndDes : Legal->getInductionVars().values())
5043 VecValuesToIgnore.insert_range(IndDes.getCastInsts());
5044}
5045
5046void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
5047 CM->collectValuesToIgnore();
5048 Config.collectElementTypesForWidening(&CM->ValuesToIgnore);
5049
5050 FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
5051 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
5052 return;
5053
5054 Config.collectInLoopReductions();
5055 // Cases that may be vectorized may be optimized by unit stride predicates.
5056 // TODO: Currently unit stride predicates are added unconditionally, even if
5057 // they are not used for the selected VF (e.g. when only interleaving).
5058 if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
5059 Legal->collectUnitStridePredicates();
5060
5061 auto VPlan1 = tryToBuildVPlan1(*CM);
5062 if (!VPlan1)
5063 return;
5064
5065 LLVM_DEBUG(dbgs() << "LV: VPlan created successfully. Loop can be "
5066 "vectorized.\n");
5067
5068 if (!OrigLoop->isInnermost()) {
5069 // For outer loops, computeMaxVF returns a single non-scalar VF; build a
5070 // plan for that VF only.
5071 ElementCount VF =
5072 MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
5073 buildVPlans(*VPlan1, VF, VF, *CM);
5075 return;
5076 }
5077
5078 // Compute the minimal bitwidths required for integer operations in the loop
5079 // for later use by the cost model.
5080 Config.computeMinimalBitwidths();
5081
5082 // Invalidate interleave groups if all blocks of loop will be predicated.
5083 if (CM->blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
5084 !useMaskedInterleavedAccesses(Opts, TTI)) {
5085 LLVM_DEBUG(dbgs() << "LV: Invalidate all interleaved groups due to "
5086 "predication which requires masked-interleaved "
5087 "support.\n");
5088 if (CM->InterleaveInfo.invalidateGroups())
5089 // Invalidating interleave groups also requires invalidating all decisions
5090 // based on them, which includes widening decisions and uniform and scalar
5091 // values.
5092 CM->invalidateCostModelingDecisions();
5093 }
5094
5095 if (CM->foldTailByMasking())
5096 Legal->prepareToFoldTailByMasking();
5097
5098 ElementCount MaxUserVF =
5099 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
5100 if (UserVF) {
5101 if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
5103 "UserVF ignored because it may be larger than the maximal safe VF",
5104 "InvalidUserVF", ORE, OrigLoop);
5105 } else {
5106 assert(UserVF.isPowerOf2() && "VF needs to be a power of two");
5107 // Collect the instructions (and their associated costs) that will be more
5108 // profitable to scalarize.
5109 CM->collectNonVectorizedAndSetWideningDecisions(UserVF);
5110 buildVPlans(*VPlan1, UserVF, UserVF, *CM);
5111
5112 // For scalar VF, skip VPlan cost check as VPlan cost is designed for
5113 // vector VFs only.
5114 if (!VPlans.empty() && (VPlans.front()->getSingleVF() == UserVF) &&
5115 (UserVF.isScalar() ||
5116 cost(*VPlans.front(), UserVF, /*RU=*/nullptr, *CM).isValid())) {
5117 // Plan for epilogue only if we succeeded in building the Vplan for the
5118 // main loop.
5119 // Try to build a tail-folded epilogue Vplan. Otherwise build a regular
5120 // plan for the epilogue:
5121 if (!planForEpilogueTF()) {
5123 if (EpilogueUserVF.isVector() &&
5124 ElementCount::isKnownLT(EpilogueUserVF, UserVF)) {
5125 CM->collectNonVectorizedAndSetWideningDecisions(EpilogueUserVF);
5126 buildVPlans(*VPlan1, EpilogueUserVF, EpilogueUserVF, *CM);
5127 }
5128 }
5129 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
5131 return;
5132 }
5133 VPlans.clear();
5134 reportVectorizationInfo("UserVF ignored because of invalid costs.",
5135 "InvalidCost", ORE, OrigLoop);
5136 }
5137 }
5138
5139 // Collect the Vectorization Factor Candidates.
5140 SmallVector<ElementCount> VFCandidates;
5141 for (auto VF = ElementCount::getFixed(1);
5142 ElementCount::isKnownLE(VF, MaxFactors.FixedVF); VF *= 2)
5143 VFCandidates.push_back(VF);
5144 for (auto VF = ElementCount::getScalable(1);
5145 ElementCount::isKnownLE(VF, MaxFactors.ScalableVF); VF *= 2)
5146 VFCandidates.push_back(VF);
5147
5148 for (const auto &VF : VFCandidates) {
5149 // Collect Uniform and Scalar instructions after vectorization with VF.
5150 CM->collectNonVectorizedAndSetWideningDecisions(VF);
5151 }
5152
5153 buildVPlans(*VPlan1, ElementCount::getFixed(1), MaxFactors.FixedVF, *CM);
5154 buildVPlans(*VPlan1, ElementCount::getScalable(1), MaxFactors.ScalableVF,
5155 *CM);
5156
5158}
5159
5160bool LoopVectorizationPlanner::planForEpilogueTF() {
5161 EpilogueLowering EpilogueTailLoweringStatus = getEpilogueTailLowering(
5162 Opts, *CM, OrigLoop, ORE, *Legal, Config.getHints(), &TTI);
5163 if (EpilogueTailLoweringStatus !=
5165 return false;
5166 LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
5167
5168 LoopVectorizationCostModel EpilogueTfCM(
5169 EpilogueTailLoweringStatus, OrigLoop, PSE, LI, Legal, TTI, TLI, CM->AC,
5170 ORE, CM->GetBFI, CM->TheFunction, CM->InterleaveInfo, Config);
5171
5172 assert(!EpilogueTfCM.isEpilogueAllowed() &&
5173 "Epilogue tail-folding is expected to be enabled");
5174
5175 LLVM_DEBUG(dbgs() << "LV: plan for tail-folded epilogue\n");
5176
5177 EpilogueTfCM.ValuesToIgnore.insert_range(CM->ValuesToIgnore);
5178 EpilogueTfCM.VecValuesToIgnore.insert_range(CM->VecValuesToIgnore);
5179
5180 FixedScalableVFPair MaxFactors =
5181 EpilogueTfCM.computeMaxVF(EpilogueVectorizationForceVF, /*UserIC*/ 1);
5182 // TODO: Validate the epilogue VF against MaxFactors.
5183 // That validation is ignored for now because computeMaxVF returns 0 for
5184 // max safe scalable VF when UserVF is not scalable, and at that case having
5185 // scalable epilogue VF is considered unsafe which is inaccurate.
5186 if (!MaxFactors || !EpilogueTfCM.foldTailByMasking()) {
5187 // Cases that should not be vectorized or tail-folded.
5188 reportVectorizationInfo("This case of epilogue loop can't be tail-folded",
5189 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5190 return false;
5191 }
5192
5193 if (EpilogueTfCM.getTailFoldingStyle() !=
5195 reportVectorizationInfo("Right now tail-folded epilogue is supported only "
5196 "for DataAndControlFlow tail-folding style.",
5197 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5198 return false;
5199 }
5200
5201 auto VPlan1 = tryToBuildVPlan1(EpilogueTfCM);
5202
5203 if (!VPlan1) {
5205 "Failed to build initial tail-folded epilogue VPlan",
5206 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5207 return false;
5208 }
5209
5210 Legal->prepareToFoldTailByMasking();
5211
5212 // Collect the instructions (and their associated costs) that will be more
5213 // profitable to scalarize.
5214 EpilogueTfCM.collectNonVectorizedAndSetWideningDecisions(
5216
5217 [[maybe_unused]] size_t NumPlansBefore = VPlans.size();
5218 buildVPlans(*VPlan1, EpilogueVectorizationForceVF,
5219 EpilogueVectorizationForceVF, EpilogueTfCM);
5220
5221 // Check that a Vplan is successfully built:
5222 assert(VPlans.size() == NumPlansBefore + 1 &&
5223 "Expected to build a single tail-folded VPlan");
5224 VPlan &Plan = *VPlans.back();
5226 Plan.hasTailFolded() &&
5227 "Expected to build a valid tail-folded VPlan with epilogue VF");
5228
5229 // Validate that VPlan has a mask to make sure that the epilogue loop
5230 // terminates correctly:
5234 "Tail-folded epilogue has no active-lane-mask exit, so it may not "
5235 "terminate, can't apply tail-folded epilogue",
5236 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5237 VPlans.pop_back();
5238 return false;
5239 }
5240
5241 if (!cost(Plan, EpilogueVectorizationForceVF, /*RU=*/nullptr, EpilogueTfCM)
5242 .isValid()) {
5243 VPlans.pop_back();
5244 reportVectorizationInfo("This case of epilogue loop can't be tail-folded "
5245 "- Invalid costs",
5246 "InvalidTailFoldedEpilogue", ORE, OrigLoop);
5247 return false;
5248 }
5249 LLVM_DEBUG(dbgs() << "LV: Tail-folded epilogue VPlan is created\n");
5250 return true;
5251}
5252
5256 bool ReusePrintingSlotTracker)
5257 : Opts(Config.getOpts()), TTI(Config.getTTI()), TLI(TLI),
5258 LLVMCtx(Plan.getContext()), CM(CM), Config(Config),
5259 CostKind(Config.CostKind), PSE(Config.getPSE()), L(Config.getLoop()) {
5260#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5261 if (ReusePrintingSlotTracker)
5262 PlanForSlotTracker = &Plan;
5263#endif
5264}
5265
5267 ElementCount VF) const {
5268 InstructionCost Cost = CM.getInstructionCost(UI, VF);
5269 if (Cost.isValid() && Opts.force_target_instruction_cost)
5270 return InstructionCost(*Opts.force_target_instruction_cost);
5271 return Cost;
5272}
5273
5274bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
5275 return CM.ValuesToIgnore.contains(UI) ||
5276 (IsVector && CM.VecValuesToIgnore.contains(UI)) ||
5277 SkipCostComputation.contains(UI);
5278}
5279
5285
5287 return CM.isScalarWithPredication(I, VF) ||
5288 CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
5289 (VF.isVector() && CM.isProfitableToScalarize(I, VF));
5290}
5291
5293 return CM.isMaskRequired(I);
5294}
5295
5299 return TC && TC->getValue().ule(VF.getKnownMinValue());
5300}
5301
5303LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
5304 VPCostContext &CostCtx) const {
5306
5307 // If the vector loop gets executed exactly once with the given VF, ignore the
5308 // costs of comparison and induction instructions, as they'll get simplified
5309 // away.
5310 // TODO: Remove this code after stepping away from the legacy cost model and
5311 // adding code to simplify VPlans before calculating their costs.
5312 auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
5313 if (TC == VF && !Plan.hasTailFolded())
5314 addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
5315 CostCtx.SkipCostComputation);
5316
5317 // Don't apply special costs when instruction cost is forced to make sure the
5318 // forced cost is used for each recipe.
5319 if (Opts.force_target_instruction_cost)
5320 return Cost;
5321
5322 // Pre-compute costs for instructions that are forced-scalar or profitable to
5323 // scalarize. For most such instructions, their scalarization costs are
5324 // accounted for here using the legacy cost model. However, some opcodes
5325 // are excluded from these precomputed scalarization costs and are instead
5326 // modeled later by the VPlan cost model (see UseVPlanCostModel below).
5327 for (Instruction *ForcedScalar : CostCtx.CM.ForcedScalars[VF]) {
5328 if (CostCtx.skipCostComputation(ForcedScalar, VF.isVector()))
5329 continue;
5330 CostCtx.SkipCostComputation.insert(ForcedScalar);
5331 InstructionCost ForcedCost = CostCtx.getLegacyCost(ForcedScalar, VF);
5332 LLVM_DEBUG({
5333 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
5334 << ": forced scalar " << *ForcedScalar << "\n";
5335 });
5336 Cost += ForcedCost;
5337 }
5338
5339 // Don't apply legacy scalarization costs if nothing remains scalar &
5340 // predicated.
5341 if (!hasReplicatorRegion(Plan))
5342 return Cost;
5343
5344 auto UseVPlanCostModel = [](Instruction *I) -> bool {
5345 switch (I->getOpcode()) {
5346 case Instruction::SDiv:
5347 case Instruction::UDiv:
5348 case Instruction::SRem:
5349 case Instruction::URem:
5350 return true;
5351 default:
5352 return false;
5353 }
5354 };
5355 // Only charge legacy scalarization costs for instructions VPlan replicates.
5356 SmallPtrSet<Instruction *, 8> Replicated;
5357 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
5359 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(*VPBB))
5360 Replicated.insert(RepR.getUnderlyingInstr());
5361 for (const auto &[Scalarized, ScalarCost] : CostCtx.CM.InstsToScalarize[VF]) {
5362 if (!Replicated.contains(Scalarized) || UseVPlanCostModel(Scalarized) ||
5363 CostCtx.skipCostComputation(Scalarized, VF.isVector()))
5364 continue;
5365 CostCtx.SkipCostComputation.insert(Scalarized);
5366 LLVM_DEBUG({
5367 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
5368 << ": profitable to scalarize " << *Scalarized << "\n";
5369 });
5370 Cost += ScalarCost;
5371 }
5372
5373 return Cost;
5374}
5375
5376#ifndef NDEBUG
5377/// Returns the frequency with which \p VPBB executes, as recorded on its
5378/// recipes. All recipes of a block share the same frequency.
5379static std::optional<VPExecutionFrequency>
5381 if (VPBB->empty())
5382 return std::nullopt;
5383 return cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
5384}
5385#endif
5386
5388LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
5389 VPRegisterUsage *RU,
5390 LoopVectorizationCostModel &PlanCM) const {
5391 VPCostContext CostCtx(*TLI, Plan, PlanCM, Config,
5392 /*ReusePrintingSlotTracker=*/true);
5393 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
5394 LLVM_DEBUG(dbgs() << "Precomputed costs for VF " << VF << ": " << Cost
5395 << '\n');
5396
5397 // Now compute and add the VPlan-based cost.
5398 Cost += Plan.cost(VF, CostCtx);
5399
5400 // Add the cost of spills due to excess register usage
5401 if (RU && Config.shouldConsiderRegPressureForVF(VF)) {
5402 InstructionCost SpillCost = RU->spillCost(
5403 TTI, Config.CostKind, Opts.force_target_num_vector_regs.value_or(0));
5404 LLVM_DEBUG(dbgs() << "Spill costs for VF " << VF << ": " << SpillCost
5405 << '\n');
5406 Cost += SpillCost;
5407 }
5408
5409#ifndef NDEBUG
5410 unsigned EstimatedWidth =
5411 estimateElementCount(VF, Config.getVScaleForTuning());
5412 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
5413 << " (Estimated cost per lane: ");
5414 if (Cost.isValid()) {
5415 APFloat CostPerLane(APFloat::IEEEdouble());
5416 APFloat EstimatedWidthAsAPFloat(APFloat::IEEEdouble());
5417 (void)CostPerLane.convertFromAPInt(APInt(64, (uint64_t)Cost.getValue()),
5418 false, APFloat::rmTowardZero);
5419 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5420 APInt(64, (uint64_t)EstimatedWidth), false, APFloat::rmTowardZero);
5421 (void)CostPerLane.divide(EstimatedWidthAsAPFloat, APFloat::rmTowardZero);
5422
5423 SmallString<16> Str;
5424 CostPerLane.toString(Str, 3);
5425 LLVM_DEBUG(dbgs() << Str);
5426 } else /* No point dividing an invalid cost - it will still be invalid */
5427 LLVM_DEBUG(dbgs() << "Invalid");
5428 LLVM_DEBUG(dbgs() << ")\n");
5429#endif
5430 return Cost;
5431}
5432
5433std::pair<VectorizationFactor, VPlan *>
5435 if (VPlans.empty())
5436 return {VectorizationFactor::Disabled(), nullptr};
5437 // If there is a single VPlan with a single VF, return it directly.
5438 VPlan &FirstPlan = *VPlans[0];
5439
5440 ElementCount UserVF = Config.getHints().getWidth();
5441 if (VPlans.size() == 1) {
5442 // For outer loops, the plan has a single vector VF determined by the
5443 // heuristic.
5444 assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
5445 FirstPlan.isOuterLoop()) &&
5446 "must have a single scalar VF, UserVF or an outer loop");
5447 return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
5448 }
5449
5450 if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
5451 assert(VPlans[0]->getSingleVF() == UserVF &&
5452 "expected second plan to be for the forced UserVF");
5453 assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
5454 "expected first plan to be for the forced epilogue VF");
5455 return {VectorizationFactor(UserVF, 0, 0), VPlans[0].get()};
5456 }
5457
5458 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
5459 << (Config.CostKind == TTI::TCK_RecipThroughput
5460 ? "Reciprocal Throughput\n"
5461 : Config.CostKind == TTI::TCK_Latency
5462 ? "Instruction Latency\n"
5463 : Config.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
5464 : Config.CostKind == TTI::TCK_SizeAndLatency
5465 ? "Code Size and Latency\n"
5466 : "Unknown\n"));
5467
5469 assert(FirstPlan.hasVF(ScalarVF) &&
5470 "More than a single plan/VF w/o any plan having scalar VF");
5471
5472 // TODO: Compute scalar cost using VPlan-based cost model.
5473 InstructionCost ScalarCost = CM->expectedCost(ScalarVF);
5474 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
5475 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
5476 VectorizationFactor BestFactor = ScalarFactor;
5477
5478 bool ForceVectorization =
5479 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
5480 if (ForceVectorization) {
5481 // Ignore scalar width, because the user explicitly wants vectorization.
5482 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5483 // evaluation.
5484 BestFactor.Cost = InstructionCost::getMax();
5485 }
5486
5487 VPlan *PlanForBestVF = &FirstPlan;
5488 ElementCount ExactTC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
5489
5490 for (auto &P : VPlans) {
5491 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
5492 P->vectorFactors().end());
5493
5494 // For loops where the Trip Count is below the TailFoldingThreshold, only
5495 // consider the largest VF to result in at most one vector iteration, and at
5496 // most one scalar iteration.
5497 // FIXME: Encode this decision directly in LVPlanner.
5498 if (!ForceVectorization && P->hasScalarTail() && ExactTC.isFixed() &&
5499 ExactTC.getFixedValue() > 0 &&
5500 ExactTC.getFixedValue() <= TTI.getMinTripCountTailFoldingThreshold()) {
5501 VFs = VFs.take_back(1);
5502 }
5503
5505 bool ConsiderRegPressure = any_of(VFs, [this](ElementCount VF) {
5506 return Config.shouldConsiderRegPressureForVF(VF);
5507 });
5508 if (ConsiderRegPressure)
5509 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI);
5510
5511 for (unsigned I = 0; I < VFs.size(); I++) {
5512 ElementCount VF = VFs[I];
5513 if (VF.isScalar())
5514 continue;
5515 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
5516 LLVM_DEBUG(
5517 dbgs()
5518 << "LV: Not considering vector loop of width " << VF
5519 << " because it will not generate any vector instructions.\n");
5520 continue;
5521 }
5522 if (Config.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
5523 LLVM_DEBUG(
5524 dbgs()
5525 << "LV: Not considering vector loop of width " << VF
5526 << " because it would cause replicated blocks to be generated,"
5527 << " which isn't allowed when optimizing for size.\n");
5528 continue;
5529 }
5530
5532 cost(*P, VF, ConsiderRegPressure ? &RUs[I] : nullptr, *CM);
5533 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
5534
5535 if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail())) {
5536 BestFactor = CurrentFactor;
5537 PlanForBestVF = P.get();
5538 }
5539
5540 // If profitable add it to ProfitableVF list.
5541 if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail()))
5542 ProfitableVFs.push_back(CurrentFactor);
5543 }
5544 }
5545
5546 VPlan &BestPlan = *PlanForBestVF;
5547
5548 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
5549 "when vectorizing, the scalar cost must be computed.");
5550
5551 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
5552 return {BestFactor, &BestPlan};
5553}
5554
5556 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
5558 std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
5560 std::function<const BranchProbabilityInfo &()> GetBPI)
5561 : Opts(Config.getOpts()), OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
5562 Legal(Legal), CM(std::move(CM)), Config(Config), PSE(PSE), ORE(ORE),
5563 GetBPI(GetBPI) {}
5564
5566
5568
5570 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
5571 bool HasRuntimeChecks, EpilogueVectorizationKind EpilogueVecKind,
5572 MutableArrayRef<VPBasicBlock *> TrackedBlocks) {
5573 assert(BestVPlan.hasVF(BestVF) &&
5574 "Trying to execute plan with unsupported VF");
5575 assert(BestVPlan.hasUF(BestUF) &&
5576 "Trying to execute plan with unsupported UF");
5577 if (BestVPlan.hasEarlyExit())
5578 ++LoopsEarlyExitVectorized;
5579
5581 *PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
5582 if (TTI.hasMultiVectorLoadStore(
5583 BestUF, BestVPlan.hasTailFolded()
5587 BestUF, TTI);
5588 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
5589 // cost model is complete for better cost estimates.
5590 RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
5594 bool HasBranchWeights =
5595 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator());
5596 if (HasBranchWeights) {
5597 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5599 BestVPlan, BestVF, VScale);
5600 }
5601
5602 if (vputils::findIncomingAliasMask(BestVPlan)) {
5603 assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
5605 *Legal->getRuntimePointerChecking()->getDiffChecks(),
5606 *PSE.getSE(), HasBranchWeights);
5607 ++LoopsPartialAliasVectorized;
5608 }
5609
5610 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
5611 VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
5612
5614 BestVF, BestUF, PSE);
5615 RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
5616 PSE);
5618 // Check if scalar epilogue is required, before simplifying constant branches.
5619 const bool RequiresScalarEpilogue = BestVPlan.requiresScalarEpilogue();
5620 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5622 /*OnlyLatches=*/false);
5624
5626 // Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
5628 // Regions are dissolved after optimizing for VF and UF, which completely
5629 // removes unneeded loop regions first.
5630 const bool HasTailFolded = BestVPlan.hasTailFolded();
5632 // Expand BranchOnTwoConds after dissolution, when latch has direct access to
5633 // its successors.
5635 // Convert loops with variable-length stepping after regions are dissolved.
5637 // Remove dead back-edges for single-iteration loops with BranchOnCond(true).
5638 // Only process loop latches to avoid removing edges from the middle block,
5639 // which may be needed for epilogue vectorization.
5641 /*OnlyLatches=*/true);
5642 std::optional<uint64_t> MaxRuntimeStep = getMaxRuntimeElementCount(
5643 BestVF * BestUF, *OrigLoop->getHeader()->getParent());
5644
5645 assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
5646 "loops not exiting via the latch without required epilogue?");
5648 VectorPH, HasTailFolded, RequiresScalarEpilogue,
5649 &BestVPlan.getVFxUF(), MaxRuntimeStep);
5651 BestVF);
5652 // Limit expansions to VPInstruction to when not vectorizing the epilogue.
5653 // Currently this code path still relies on code re-using SCEVs expanded
5654 // directly to IR instructions.
5655 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5657 *PSE.getSE());
5658 RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan, VPDominatorTree(BestVPlan));
5660 // Removing branches and incoming values may expose additional simplification
5661 // opportunities.
5663 /*OnlyLatches=*/EpilogueVecKind !=
5666 RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
5667 if (BestVPlan.getEntry()->getSingleSuccessor() ==
5668 BestVPlan.getScalarPreheader()) {
5669 // TODO: The vector loop would be dead, should not even try to vectorize.
5670 ORE->emit([&]() {
5671 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
5672 OrigLoop->getStartLoc(),
5673 OrigLoop->getHeader())
5674 << "Created vector loop never executes due to insufficient trip "
5675 "count.";
5676 });
5678 }
5679
5680 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
5681 // making any changes to the CFG.
5682 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
5683 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVs, BestVPlan, *PSE.getSE());
5684
5685 // Perform the actual loop transformation.
5686 IRBuilder<> Builder(PSE.getSE()->getModule());
5687 VPTransformState State(&TTI, BestVF, LI, DT, Legal->getAssumptionCache(),
5688 Builder, &BestVPlan, OrigLoop->getParentLoop());
5689
5690#ifdef EXPENSIVE_CHECKS
5691 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
5692#endif
5693
5694 // 1. Create the scalar preheader; all other blocks are created during VPlan
5695 // execution.
5696 State.CFG.PrevBB = OrigLoop->getLoopPreheader();
5697 assert(State.CFG.PrevBB && "Invalid loop structure");
5698 BasicBlock *ScalarPHBB = SplitBlock(
5699 State.CFG.PrevBB, State.CFG.PrevBB->getTerminator(), DT, LI, nullptr,
5700 EpilogueVecKind == EpilogueVectorizationKind::Epilogue
5701 ? "vec.epilog.scalar.ph"
5702 : "scalar.ph");
5703 if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
5704 replaceVPBBWithIRVPBB(ScalarPH, ScalarPHBB, &BestVPlan);
5706
5707 assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
5708
5709 // After vectorization, the exit blocks of the original loop will have
5710 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
5711 // looked through single-entry phis.
5712 ScalarEvolution &SE = *PSE.getSE();
5713 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
5714 if (!Exit->hasPredecessors())
5715 continue;
5716 for (VPRecipeBase &PhiR : Exit->phis())
5718 &cast<VPIRPhi>(PhiR).getIRPhi());
5719 }
5720
5721 // Query whether the target wants loops it vectorizes to remain eligible for
5722 // runtime unrolling. Do this here, on the original loop and before its SCEV
5723 // is forgotten below.
5725 TTI.getUnrollingPreferences(OrigLoop, SE, UP, ORE);
5726 bool UnrollVectorizedLoop = UP.UnrollVectorizedLoop;
5727
5728 // Forget the original loop and block dispositions.
5729 SE.forgetLoop(OrigLoop);
5731
5732 //===------------------------------------------------===//
5733 //
5734 // Notice: any optimization or new instruction that go
5735 // into the code below should also be implemented in
5736 // the cost-model.
5737 //
5738 //===------------------------------------------------===//
5739
5740 // Retrieve loop information before executing the plan, which may remove the
5741 // original loop, if it becomes unreachable.
5742 MDNode *LID = OrigLoop->getLoopID();
5743 unsigned OrigLoopInvocationWeight = 0;
5744 std::optional<unsigned> OrigAverageTripCount =
5745 getLoopEstimatedTripCount(OrigLoop, &OrigLoopInvocationWeight);
5746
5747 BestVPlan.execute(&State);
5748
5749 // 2.6. Maintain Loop Hints
5750 // Keep all loop hints from the original loop on the vector loop (we'll
5751 // replace the vectorizer-specific hints below).
5752 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(BestVPlan, State.VPDT);
5753 // Add metadata to disable runtime unrolling a scalar loop when there
5754 // are no runtime checks about strides and memory. A scalar loop that is
5755 // rarely used is not worth unrolling.
5756 bool DisableRuntimeUnroll = !HasRuntimeChecks && !BestVF.isScalar();
5758 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
5759 : nullptr,
5760 HeaderVPBB, BestVPlan,
5761 EpilogueVecKind == EpilogueVectorizationKind::Epilogue, LID,
5762 OrigAverageTripCount, OrigLoopInvocationWeight,
5763 estimateElementCount(BestVF * BestUF, Config.getVScaleForTuning()),
5764 DisableRuntimeUnroll, UnrollVectorizedLoop);
5765
5766 // 3. Remove redundant induction instructions, if a vector loop remains.
5767 if (HeaderVPBB)
5768 legacyCSE(State.CFG.VPBB2IRBB.lookup(HeaderVPBB));
5769
5770 // Wrap the generated blocks in VPIRBasicBlocks, so they can be used in the
5771 // epilogue plan.
5772 if (EpilogueVecKind == EpilogueVectorizationKind::MainLoop)
5774 vp_depth_first_shallow(BestVPlan.getEntry()))))
5775 if (!isa<VPIRBasicBlock>(VPBB)) {
5777 VPBB, State.CFG.VPBB2IRBB.at(VPBB), &BestVPlan);
5778 replace(TrackedBlocks, VPBB, IRVPBB);
5779 }
5780
5781 return ExpandedSCEVs;
5782}
5783
5785 return CM.isPredicatedInst(I);
5786}
5787
5789 return CM.TTI.prefersVectorizedAddressing();
5790}
5791
5793 VFRange &Range) {
5794 assert((VPI->getOpcode() == Instruction::Load ||
5795 VPI->getOpcode() == Instruction::Store) &&
5796 "Must be called with either a load or store");
5798
5799 auto WillWiden = [&](ElementCount VF) -> bool {
5801 CM.getWideningDecision(I, VF);
5803 "CM decision should be taken at this point.");
5805 return true;
5806 if (CM.isScalarAfterVectorization(I, VF) ||
5807 CM.isProfitableToScalarize(I, VF))
5808 return false;
5810 };
5811
5813 return nullptr;
5814
5815 // If a mask is not required, drop it - use unmasked version for safe loads.
5816 // TODO: Determine if mask is needed in VPlan.
5817 VPValue *Mask = CM.isMaskRequired(I) ? VPI->getMask() : nullptr;
5818
5819 // Determine if the pointer operand of the access is either consecutive or
5820 // reverse consecutive.
5822 CM.getWideningDecision(I, Range.Start);
5824 bool Consecutive =
5826
5827 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
5828 : VPI->getOperand(1);
5829 Builder.setInsertPoint(VPI);
5830 if (Consecutive) {
5831 Ptr = Builder.createConsecutiveVectorPointer(Ptr, getLoadStoreType(I),
5832 Reverse, VPI->getDebugLoc());
5833 }
5834
5835 if (Reverse && Mask)
5836 Mask = Builder.createNaryOp(VPInstruction::Reverse, Mask, I->getDebugLoc());
5837
5838 if (VPI->getOpcode() == Instruction::Load) {
5839 auto *Load = cast<LoadInst>(I);
5840 auto *LoadR =
5841 Builder.createWidenLoad(*Load, Ptr, Mask, Consecutive, Load->getAlign(),
5842 *VPI, Load->getDebugLoc());
5843 if (Reverse)
5844 return Builder.createNaryOp(VPInstruction::Reverse, LoadR,
5845 LoadR->getDebugLoc());
5846 return LoadR;
5847 }
5848
5850 VPValue *StoredVal = VPI->getOperand(0);
5851 if (Reverse)
5852 StoredVal = Builder.createNaryOp(VPInstruction::Reverse, StoredVal,
5853 Store->getDebugLoc());
5854 return Builder.createWidenStore(*Store, Ptr, StoredVal, Mask, Consecutive,
5855 Store->getAlign(), *VPI,
5856 Store->getDebugLoc());
5857}
5858
5859bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
5861 "Instruction should have been handled earlier");
5862 // Instruction should be widened, unless it is scalar after vectorization,
5863 // scalarization is profitable or it is predicated.
5864 auto WillScalarize = [this, I](ElementCount VF) -> bool {
5865 return CM.isScalarAfterVectorization(I, VF) ||
5866 CM.isProfitableToScalarize(I, VF) ||
5867 CM.isScalarWithPredication(I, VF);
5868 };
5870 Range);
5871}
5872
5873VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
5874 auto *I = VPI->getUnderlyingInstr();
5875 switch (VPI->getOpcode()) {
5876 default:
5877 return nullptr;
5878 case Instruction::SDiv:
5879 case Instruction::UDiv:
5880 case Instruction::SRem:
5881 case Instruction::URem:
5882 // If not provably safe, use a masked intrinsic.
5883 if (CM.isPredicatedInst(I))
5884 return new VPWidenIntrinsicRecipe(
5886 I->getType(), {}, {}, VPI->getDebugLoc());
5887 [[fallthrough]];
5888 case Instruction::Add:
5889 case Instruction::And:
5890 case Instruction::AShr:
5891 case Instruction::FAdd:
5892 case Instruction::FCmp:
5893 case Instruction::FDiv:
5894 case Instruction::FMul:
5895 case Instruction::FNeg:
5896 case Instruction::FRem:
5897 case Instruction::FSub:
5898 case Instruction::ICmp:
5899 case Instruction::LShr:
5900 case Instruction::Mul:
5901 case Instruction::Or:
5902 case Instruction::Select:
5903 case Instruction::Shl:
5904 case Instruction::Sub:
5905 case Instruction::Xor:
5906 case Instruction::Freeze:
5907 case Instruction::ExtractValue:
5908 return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
5909 VPI->getDebugLoc());
5910 };
5911}
5912
5914 if (VPI->getOpcode() != Instruction::Store)
5915 return nullptr;
5916
5917 const HistogramInfo *HistInfo =
5918 Legal->getHistogramInfo(cast<StoreInst>(VPI->getUnderlyingInstr()));
5919 if (!HistInfo)
5920 return nullptr;
5921
5923 // Bucket address.
5924 HGramOps.push_back(VPI->getOperand(1));
5925 // Increment value.
5926 HGramOps.push_back(Plan.getOrAddLiveIn(HistInfo->Update->getOperand(1)));
5927
5928 // In case of predicated execution (due to tail-folding, or conditional
5929 // execution, or both), pass the relevant mask.
5930 if (VPValue *Mask = VPI->getMask())
5931 HGramOps.push_back(Mask);
5932
5933 return new VPHistogramRecipe(HistInfo->UpdateKind, HGramOps,
5934 cast<VPIRMetadata>(*VPI), VPI->getDebugLoc());
5935}
5936
5937std::optional<CompressedPtrInfo>
5939 if (!is_contained({Instruction::Load, Instruction::Store}, VPI->getOpcode()))
5940 return std::nullopt;
5942 return Legal->getCompressedPtrInfo(getLoadStorePointerOperand(I));
5943}
5944
5947 assert(getCompressedPtrInfo(VPI) && "Expected a compressed load/store");
5948
5949 VPBuilder::InsertPointGuard Guard(Builder);
5950 Builder.setInsertPoint(VPI);
5951
5952 VPValue *Mask = VPI->getMask();
5953 Type *AccessTy = vputils::getLoadStoreValueType(VPI);
5955
5956 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
5957 : VPI->getOperand(1);
5958 Ptr = Builder.createConsecutiveVectorPointer(Ptr, AccessTy,
5959 /*Reverse=*/false,
5960 VPI->getDebugLoc());
5961
5962 // Note: For expandload we fill masked-off lanes with passthru poison.
5963 if (VPI->getOpcode() == Instruction::Load)
5964 return new VPWidenMemIntrinsicRecipe(
5965 Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(AccessTy)},
5966 AccessTy, Alignment, *VPI, VPI->getDebugLoc());
5967
5968 VPValue *StoredValue = VPI->getOperand(0);
5969 return new VPWidenMemIntrinsicRecipe(Intrinsic::masked_compressstore,
5970 {StoredValue, Ptr, Mask}, AccessTy,
5971 Alignment, *VPI, VPI->getDebugLoc());
5972}
5973
5975 VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
5976 StoreInst *SI;
5977 if ((SI = dyn_cast<StoreInst>(VPI->getUnderlyingInstr())) &&
5978 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
5979 // Only create recipe for the final invariant store of the reduction.
5980 if (Legal->isInvariantStoreOfReduction(SI)) {
5981 VPValue *Val = VPI->getOperand(0);
5982 VPValue *Addr = VPI->getOperand(1);
5983 // We need to store the exiting value of the reduction, so use the blend
5984 // if tail folded.
5986 Val = Blend;
5987 [[maybe_unused]] auto *Rdx =
5989 assert((isa<VPIRValue>(Val) || !Rdx || Rdx->getBackedgeValue() == Val) &&
5990 "Store of reduction thats not the backedge value?");
5991 auto *Recipe = new VPReplicateRecipe(
5992 SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
5993 VPI->getDebugLoc());
5994 FinalRedStoresBuilder.insert(Recipe);
5995 }
5996 VPI->eraseFromParent();
5997 return true;
5998 }
5999
6000 return false;
6001}
6002
6004 VFRange &Range) {
6005 auto *I = VPI->getUnderlyingInstr();
6007 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
6008 Range);
6009
6010 bool IsPredicated = CM.isPredicatedInst(I);
6011
6012 // Even if the instruction is not marked as uniform, there are certain
6013 // intrinsic calls that can be effectively treated as such, so we check for
6014 // them here. Conservatively, we only do this for scalable vectors, since
6015 // for fixed-width VFs we can always fall back on full scalarization.
6016 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(I)) {
6017 switch (cast<IntrinsicInst>(I)->getIntrinsicID()) {
6018 case Intrinsic::assume:
6019 case Intrinsic::lifetime_start:
6020 case Intrinsic::lifetime_end:
6021 // For scalable vectors if one of the operands is variant then we still
6022 // want to mark as uniform, which will generate one instruction for just
6023 // the first lane of the vector. We can't scalarize the call in the same
6024 // way as for fixed-width vectors because we don't know how many lanes
6025 // there are.
6026 //
6027 // The reasons for doing it this way for scalable vectors are:
6028 // 1. For the assume intrinsic generating the instruction for the first
6029 // lane is still be better than not generating any at all. For
6030 // example, the input may be a splat across all lanes.
6031 // 2. For the lifetime start/end intrinsics the pointer operand only
6032 // does anything useful when the input comes from a stack object,
6033 // which suggests it should always be uniform. For non-stack objects
6034 // the effect is to poison the object, which still allows us to
6035 // remove the call.
6036 IsUniform = true;
6037 break;
6038 default:
6039 break;
6040 }
6041 }
6042 VPValue *BlockInMask = nullptr;
6043 if (!IsPredicated) {
6044 // Finalize the recipe for Instr, first if it is not predicated.
6045 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
6046 } else {
6047 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
6048 // Instructions marked for predication are replicated and a mask operand is
6049 // added initially. Masked replicate recipes will later be placed under an
6050 // if-then construct to prevent side-effects. Generate recipes to compute
6051 // the block mask for this region.
6052 BlockInMask = VPI->getMask();
6053 }
6054
6055 // Note that there is some custom logic to mark some intrinsics as uniform
6056 // manually above for scalable vectors, which this assert needs to account for
6057 // as well.
6058 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6059 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
6060 "Should not predicate a uniform recipe");
6061 if (IsUniform) {
6063 VPI->getOpcode(), VPI->operandsWithoutMask(), BlockInMask, *VPI, *VPI,
6064 VPI->getDebugLoc(), VPI->getScalarType(), I);
6065 }
6066 auto *Recipe = new VPReplicateRecipe(I, VPI->operandsWithoutMask(),
6067 /*IsSingleScalar=*/false, BlockInMask,
6068 *VPI, *VPI, VPI->getDebugLoc());
6069 return Recipe;
6070}
6071
6074 VFRange &Range) {
6075 assert(!R->isPhi() && "phis must be handled earlier");
6076 auto *VPI = cast<VPInstruction>(R);
6077 assert(VPI->getOpcode() != Instruction::Call &&
6078 "Call should have been handled by makeCallWideningDecisions");
6079
6080 // All widen recipes below deal only with VF > 1.
6082 [&](ElementCount VF) { return VF.isScalar(); }, Range))
6083 return nullptr;
6084
6085 Instruction *Instr = R->getUnderlyingInstr();
6086 assert(!is_contained({Instruction::Load, Instruction::Store},
6087 VPI->getOpcode()) &&
6088 "Should have been handled prior to this!");
6089
6090 // An extractvalue takes the shape of its operand: we cannot extract a lane
6091 // from a widened struct, nor build a struct from replicated lanes.
6092 if (VPI->getOpcode() == Instruction::ExtractValue &&
6094 if (VPRecipeBase *OpR = VPI->getOperand(0)->getDefiningRecipe())
6096 return tryToWiden(VPI);
6097 if (VPI->getOpcode() == Instruction::ExtractValue)
6098 return new VPReplicateRecipe(Instr, VPI->operandsWithoutMask(),
6100 /*Mask=*/nullptr, *VPI, *VPI,
6101 VPI->getDebugLoc());
6102
6103 if (!shouldWiden(Instr, Range))
6104 return nullptr;
6105
6106 if (VPI->getOpcode() == Instruction::GetElementPtr) {
6107 auto *GEP = cast<GetElementPtrInst>(Instr);
6108 return new VPWidenGEPRecipe(GEP->getSourceElementType(),
6109 VPI->operandsWithoutMask(), *VPI,
6110 VPI->getDebugLoc(), GEP);
6111 }
6112
6113 if (Instruction::isCast(VPI->getOpcode())) {
6114 auto *CI = cast<CastInst>(Instr);
6115 return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
6116 VPI->getScalarType(), CI, *VPI, *VPI,
6117 VPI->getDebugLoc());
6118 }
6119
6120 return tryToWiden(VPI);
6121}
6122
6123// To allow RUN_VPLAN_PASS to print the VPlan after VF/UF independent
6124// optimizations.
6126
6127#ifndef NDEBUG
6128/// Cross-check the execution frequencies recorded in \p Plan against
6129/// BlockFrequencyInfo for the blocks of \p OrigLoop.
6130/// FIXME: Temporary verification aid, to be removed.
6131static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
6132 LoopInfo *LI,
6134 // Limited to loops with the latch as only exiting block
6135 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
6136 return true;
6137
6138 // Visit the loop body in the same order as recordExecutionFrequencies. Both
6139 // are reverse post-orders of the same CFG, so indices correspond.
6142 assert(Blocks.size() == OrigLoop->getNumBlocks() &&
6143 "loop body and original loop must have the same blocks");
6144
6145 LoopBlocksRPO OrigRPO(OrigLoop);
6146 OrigRPO.perform(LI);
6147
6148 // Only request the expensive BFI once the cheap bail-outs are past.
6149 BlockFrequencyInfo &BFI = CM.getBFI();
6150 uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
6151 if (HeaderFreq == 0)
6152 return true;
6153
6154 // BFI's fixed-point mass propagation loses up to 1 ULP per edge, so bound the
6155 // error by the number of edges in the region.
6156 uint64_t Edges = 0;
6157 for (const VPBasicBlock *VPBB : Blocks)
6158 Edges += VPBB->getNumSuccessors();
6159 uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
6160
6161 for (const auto &[VPBB, BB] :
6162 zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
6163 // Nothing to check for blocks without a recorded frequency.
6164 std::optional<VPExecutionFrequency> Freq =
6166 if (!Freq)
6167 continue;
6169
6170 // Clamp to the header's frequency, which BFI's rounding may exceed.
6173 std::min(BBFreq, HeaderFreq), HeaderFreq);
6174 if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
6175 Tolerance)
6176 continue;
6177
6178 errs() << "Block frequency mismatch for " << VPBB->getName() << ": VPlan "
6179 << Computed << ", BlockFrequencyInfo " << Expected << "\n";
6180 return false;
6181 }
6182 return true;
6183}
6184#endif
6185
6187LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
6188 bool IsInnerLoop = OrigLoop->isInnermost();
6189
6190 // Set up loop versioning for inner loops with memory runtime checks.
6191 // Outer loops don't have LoopAccessInfo since canVectorizeMemory() is not
6192 // called for them.
6193 std::optional<LoopVersioning> LVer;
6194 if (IsInnerLoop) {
6195 const LoopAccessInfo *LAI = Legal->getLAI();
6196 LVer.emplace(*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop,
6197 LI, DT, PSE.getSE());
6198 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
6200 // Only use noalias metadata when using memory checks guaranteeing no
6201 // overlap across all iterations.
6202 LVer->prepareNoAliasMetadata();
6203 }
6204 }
6205
6206 // Create initial base VPlan0, to serve as common starting point for all
6207 // candidates built later for specific VF ranges.
6208 auto VPlan0 = VPlanTransforms::buildVPlan0(
6209 OrigLoop, *LI, Legal->getWidestInductionType(), PSE,
6210 LVer ? &*LVer : nullptr, GetBPI);
6211
6212 VPDominatorTree VPDT(*VPlan0);
6213 if (const LoopAccessInfo *LAI = Legal->getLAI())
6215 LAI->getSymbolicStrides(), VPDT);
6218 if (IsInnerLoop) {
6220 assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, PlanCM) &&
6221 "execution frequencies do not match the loop's block frequencies");
6222 }
6223
6224 // Create recipes for header phis. For outer loops, reductions, recurrences
6225 // and in-loop reductions are empty since legality doesn't detect them.
6226 if (!RUN_VPLAN_PASS(
6227 VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE, *OrigLoop, ORE,
6228 VPDT, Legal->getInductionVars(), Legal->getReductionVars(),
6229 Legal->getConditionalInductions(), Legal->getFixedOrderRecurrences(),
6230 Config.getInLoopReductions(), Config.getHints().allowReordering())) {
6231 return nullptr;
6232 }
6233
6234 if (const LoopAccessInfo *LAI = Legal->getLAI())
6236 LAI->getSymbolicStrides(), VPDT);
6237
6238 // Without -vplan-outer-loop-legality-checks, trust the explicit vectorization
6239 // pragma that vectorizing the outer loop is safe.
6240 if (!IsInnerLoop && Opts.vplan_outer_loop_legality_checks &&
6241 !proveOuterLoopMemorySafety(*VPlan0, PSE, *Legal->getAA(), VPDT,
6242 OrigLoop)) {
6244 "Cannot prove outer loop memory accesses safe",
6245 "cannot prove memory accesses in outer loop are safe to vectorize",
6246 "CantProveOuterLoopMemSafety", ORE, OrigLoop);
6247 return nullptr;
6248 }
6249
6250 // Add surviving induction predicates to PSE and check constraints.
6251 bool ForceVectorization =
6252 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
6253 bool OptForSize = !ForceVectorization && PlanCM.EpilogueLoweringStatus ==
6255 unsigned SCEVCheckThreshold = ForceVectorization
6256 ? Opts.pragma_vectorize_scev_check_threshold
6257 : Opts.vectorize_scev_check_threshold;
6259 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6260 return nullptr;
6261
6263
6264 // If we're vectorizing a loop with an uncountable exit, make sure that the
6265 // recipes are safe to handle.
6266 // TODO: Remove this once we can properly check the VPlan itself for the
6267 // presence of an uncountable exit.
6268 if (Legal->hasUncountableEarlyExit()) {
6270 OrigLoop))
6271 return nullptr;
6272
6273 // TODO: Check target preference for style.
6276 ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
6277 EEStyle)) {
6278 return nullptr;
6279 }
6280 } else {
6282 }
6283
6285 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
6286 if (PlanCM.foldTailByMasking())
6288
6290
6291 return VPlan0;
6292}
6293
6294void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
6295 ElementCount MaxVF,
6297 if (ElementCount::isKnownGT(MinVF, MaxVF))
6298 return;
6299
6300 auto MaxVFTimes2 = MaxVF * 2;
6301 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
6302 VFRange SubRange = {VF, MaxVFTimes2};
6303 auto Plan = tryToBuildVPlan(std::unique_ptr<VPlan>(VPlan1.duplicate()),
6304 SubRange, PlanCM);
6305 VF = SubRange.End;
6306
6307 if (!Plan)
6308 continue;
6309
6310 // Now optimize the initial VPlan.
6314 Config.getMinimalBitwidths());
6316 // TODO: try to put addExplicitVectorLength close to addActiveLaneMask
6317 if (PlanCM.foldTailWithEVL()) {
6319 Config.getMaxSafeElements());
6321 VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
6323 }
6324
6325 if (auto P =
6327 VPlans.push_back(std::move(P));
6328
6331 useActiveLaneMask(Style),
6333
6335 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6336 VPlans.push_back(std::move(Plan));
6337 }
6338}
6339
6341LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, VFRange &Range,
6343
6344 // For outer loops, the plan only needs basic recipe conversion and induction
6345 // live-out optimization; the full inner-loop recipe building below does not
6346 // apply (no widening decisions, interleave groups, reductions, etc.).
6347 if (Plan->isOuterLoop()) {
6348 for (ElementCount VF : Range)
6349 Plan->addVF(VF);
6351 *Plan, *TLI, PSE, OrigLoop))
6352 return nullptr;
6354 OrigLoop);
6355 return Plan;
6356 }
6357
6358 using namespace llvm::VPlanPatternMatch;
6359 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6360
6361 // ---------------------------------------------------------------------------
6362 // Build initial VPlan: Scan the body of the loop in a topological order to
6363 // visit each basic block after having visited its predecessor basic blocks.
6364 // ---------------------------------------------------------------------------
6365
6366 bool RequiresScalarEpilogueCheck =
6368 [&PlanCM](ElementCount VF) {
6369 return !PlanCM.requiresScalarEpilogue(VF.isVector());
6370 },
6371 Range);
6372 // Update the branch in the middle block if a scalar epilogue is required.
6373 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6374 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6375 auto *BranchOnCond = cast<VPInstruction>(MiddleVPBB->getTerminator());
6376 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6377 "second successor must be scalar preheader");
6378 BranchOnCond->setOperand(0, Plan->getFalse());
6379 }
6380
6381 // Don't use getDecisionAndClampRange here, because we don't know the UF
6382 // so this function is better to be conservative, rather than to split
6383 // it up into different VPlans.
6384 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
6385 bool IVUpdateMayOverflow = false;
6386 for (ElementCount VF : Range)
6387 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(Opts, &PlanCM, VF);
6388
6390 // Use NUW for the induction increment if we proved that it won't overflow in
6391 // the vector loop or when not folding the tail. In the later case, we know
6392 // that the canonical induction increment will not overflow as the vector trip
6393 // count is >= increment and a multiple of the increment.
6394 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6395 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
6396 if (!HasNUW) {
6397 auto *IVInc =
6398 LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(0);
6399 assert(match(IVInc,
6400 m_VPInstruction<Instruction::Add>(
6401 m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
6402 "Did not find the canonical IV increment");
6403 LoopRegion->clearCanonicalIVNUW(cast<VPInstruction>(IVInc));
6404 }
6405
6406 // ---------------------------------------------------------------------------
6407 // Pre-construction: record ingredients whose recipes we'll need to further
6408 // process after constructing the initial VPlan.
6409 // ---------------------------------------------------------------------------
6410
6411 // For each interleave group which is relevant for this (possibly trimmed)
6412 // Range, add it to the set of groups to be later applied to the VPlan and add
6413 // placeholders for its members' Recipes which we'll be replacing with a
6414 // single VPInterleaveRecipe.
6415 for (InterleaveGroup<Instruction> *IG :
6417 auto ApplyIG = [IG, &PlanCM](ElementCount VF) -> bool {
6418 bool Result = (VF.isVector() && // Query is illegal for VF == 1
6419 PlanCM.getWideningDecision(IG->getInsertPos(), VF) ==
6421 // For scalable vectors, the interleave factors must be <= 8 since we
6422 // require the (de)interleaveN intrinsics instead of shufflevectors.
6423 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
6424 "Unsupported interleave factor for scalable vectors");
6425 return Result;
6426 };
6427 if (!getDecisionAndClampRange(ApplyIG, Range))
6428 continue;
6429 InterleaveGroups.insert(IG);
6430 }
6431
6432 // ---------------------------------------------------------------------------
6433 // Construct wide recipes and apply predication for original scalar
6434 // VPInstructions in the loop.
6435 // ---------------------------------------------------------------------------
6436 VPRecipeBuilder RecipeBuilder(*Plan, Legal, PlanCM, Builder);
6437
6439 RecipeBuilder, *PSE.getSE()))
6440 return nullptr;
6441
6443 Range.Start);
6444
6445 VPCostContext CostCtx(*TLI, *Plan, PlanCM, Config);
6446
6448 RecipeBuilder, CostCtx);
6449
6451
6452 // Widened calls may only use the first lane of some operands.
6454 RecipeBuilder, CostCtx))
6456
6458
6459 // Convert remaining VPInstructions to widen or replicate recipes.
6460 // TODO: This legacy code should eventually be migrated to VPlan.
6461 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6462 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6463 vp_depth_first_shallow(HeaderVPBB))) {
6464 // All types but VPInstructions are already widened and don't need extra
6465 // processing. We process VPInstructions below.
6466 assert(
6467 all_of(
6468 make_range(VPBB->getFirstNonPhi(), VPBB->end()),
6469 IsaPred<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6470 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6471 VPWidenCallRecipe, VPWidenIntrinsicRecipe,
6472 VPVectorPointerRecipe, VPVectorEndPointerRecipe,
6473 VPHistogramRecipe, VPInstruction>) &&
6474 "Unexpected recipe");
6475 for (VPInstruction &VPI :
6477 // We represent single-scalar casts directly as VPInstructions.
6478 if (Instruction::isCast(VPI.getOpcode()) &&
6480 continue;
6481
6482 // Only VPInstructions with an underlying value need to be processed.
6483 if (!VPI.getUnderlyingValue())
6484 continue;
6485
6486 Builder.setInsertPoint(&VPI);
6487
6488 VPRecipeBase *Recipe =
6489 RecipeBuilder.tryToCreateWidenNonPhiRecipe(&VPI, Range);
6490 if (!Recipe)
6491 Recipe = RecipeBuilder.handleReplication(&VPI, Range);
6492 Builder.insert(Recipe);
6493
6494 if (Recipe->getNumDefinedValues() == 1) {
6495 VPI.replaceAllUsesWith(Recipe->getVPSingleValue());
6496 } else {
6497 assert(Recipe->getNumDefinedValues() == 0 &&
6498 "Unexpected multidef recipe");
6499 }
6500 VPI.eraseFromParent();
6501 }
6502 }
6503
6504 assert(isa<VPRegionBlock>(LoopRegion) &&
6505 !LoopRegion->getEntryBasicBlock()->empty() &&
6506 "entry block must be set to a VPRegionBlock having a non-empty entry "
6507 "VPBasicBlock");
6508
6510 Range);
6511
6512 // ---------------------------------------------------------------------------
6513 // Transform initial VPlan: Apply previously taken decisions, in order, to
6514 // bring the VPlan to its final state.
6515 // ---------------------------------------------------------------------------
6516
6517 addReductionResultComputation(Plan, Range.Start, PlanCM);
6518
6519 // Optimize FindIV reductions to use sentinel-based approach when possible.
6521 *OrigLoop);
6523 OrigLoop);
6524
6525 // Apply mandatory transformation to handle reductions with multiple in-loop
6526 // uses if possible, bail out otherwise.
6528 OrigLoop))
6529 return nullptr;
6530 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
6531 // NaNs if possible, bail out otherwise.
6533 return nullptr;
6534
6535 // Create whole-vector selects for find-last recurrences.
6537 return nullptr;
6538
6540
6541 // Create partial reduction recipes for scaled reductions and transform
6542 // recipes to abstract recipes if it is legal and beneficial and clamp the
6543 // range for better cost estimation.
6545 Range);
6547 Range);
6548
6549 // Interleave memory: for each Interleave Group we marked earlier as relevant
6550 // for this VPlan, replace the Recipes widening its memory instructions with a
6551 // single VPInterleaveRecipe at its insertion point.
6553 InterleaveGroups, PlanCM.isEpilogueAllowed());
6554
6555 // Convert memory recipes to strided access recipes if the strided access is
6556 // legal and profitable.
6558 *OrigLoop, CostCtx, Range);
6559
6560 if (Legal->hasUncountableExitWithSideEffects())
6562 Range);
6563
6564 // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
6565 if (Range.Start.isScalar())
6566 Range.End = Range.Start * 2;
6567
6568 for (ElementCount VF : Range)
6569 Plan->addVF(VF);
6570 Plan->setName("Initial VPlan");
6571
6573
6574 if (PlanCM.maskPartialAliasing())
6576
6577 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6578 return Plan;
6579}
6580
6581void LoopVectorizationPlanner::addReductionResultComputation(
6582 VPlanPtr &Plan, ElementCount MinVF, LoopVectorizationCostModel &PlanCM) {
6583 using namespace VPlanPatternMatch;
6584 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6585 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6586 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
6587 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
6588 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
6589 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6590 for (VPRecipeBase &R : make_early_inc_range(
6591 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
6592 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
6593 if (!PhiR)
6594 continue;
6595
6596 // Clean up reductions that have become invariant.
6597 if (PhiR->getBackedgeValue() == PhiR) {
6598 PhiR->replaceAllUsesWith(PhiR->getStartValue());
6599 PhiR->eraseFromParent();
6600 continue;
6601 }
6602
6603 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
6604 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6606 Type *PhiTy = PhiR->getScalarType();
6607
6608 // Convert a VPBlendRecipe backedge to a select.
6609 if (auto *Blend = dyn_cast<VPBlendRecipe>(PhiR->getBackedgeValue())) {
6610 if (Blend->getNumIncomingValues() == 2 &&
6611 Blend->getMask(0) == HeaderMask) {
6612 auto *Sel = VPBuilder(Blend).createSelect(
6613 Blend->getMask(0), Blend->getIncomingValue(0),
6614 Blend->getIncomingValue(1), {}, "", *Blend);
6615 Blend->replaceAllUsesWith(Sel);
6616 Blend->eraseFromParent();
6617 }
6618 }
6619
6620 auto *OrigExitingVPV = PhiR->getBackedgeValue();
6621 auto *NewExitingVPV = OrigExitingVPV;
6622
6623 // Remove the predicated select if the target doesn't want it.
6624 VPValue *V;
6625 if (!PlanCM.usePredicatedReductionSelect(
6626 RecurrenceKind, PhiR->hasUsesOutsideReductionChain()) &&
6627 match(PhiR->getBackedgeValue(),
6628 m_Select(m_Specific(HeaderMask), m_VPValue(V), m_Specific(PhiR))))
6629 PhiR->setBackedgeValue(V);
6630
6631 // We want code in the middle block to appear to execute on the location of
6632 // the scalar loop's latch terminator because: (a) it is all compiler
6633 // generated, (b) these instructions are always executed after evaluating
6634 // the latch conditional branch, and (c) other passes may add new
6635 // predecessors which terminate on this line. This is the easiest way to
6636 // ensure we don't accidentally cause an extra step back into the loop while
6637 // debugging.
6638 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6639
6640 // TODO: At the moment ComputeReductionResult also drives creation of the
6641 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
6642 // even for in-loop reductions, until the reduction resume value handling is
6643 // also modeled in VPlan.
6644 VPInstruction *FinalReductionResult;
6645 VPBuilder::InsertPointGuard Guard(Builder);
6646 Builder.setInsertPoint(IP);
6647 // For AnyOf reductions, find the select among PhiR's users and convert
6648 // the reduction phi to operate on bools before creating the final
6649 // reduction result.
6650 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
6651 auto *AnyOfSelect = findSingleUser<VPSingleDefRecipe>(
6652 PhiR,
6653 m_Select(m_Unless(m_Specific(HeaderMask)), m_VPValue(), m_VPValue()));
6654 VPValue *Start = PhiR->getStartValue();
6655 bool TrueValIsPhi = AnyOfSelect->getOperand(1) == PhiR;
6656 // NewVal is the non-phi operand of the select.
6657 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(2)
6658 : AnyOfSelect->getOperand(1);
6659
6660 // Adjust AnyOf reductions; replace the reduction phi for the selected
6661 // value with a boolean reduction phi node to check if the condition is
6662 // true in any iteration. The final value is selected by the final
6663 // ComputeReductionResult.
6664 VPValue *Cmp = AnyOfSelect->getOperand(0);
6665 // If the compare is checking the reduction PHI node, adjust it to check
6666 // the start value.
6667 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
6668 CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
6669 Builder.setInsertPoint(AnyOfSelect);
6670
6671 // If the true value of the select is the reduction phi, the new value
6672 // is selected if the negated condition is true in any iteration.
6673 if (TrueValIsPhi)
6674 Cmp = Builder.createNot(Cmp);
6675
6676 // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
6677 // the exiting value flows through).
6678 auto *NewPhiR =
6679 PhiR->cloneWithOperands(Plan->getFalse(), Plan->getFalse());
6680 NewPhiR->insertBefore(PhiR);
6681 VPValue *NewExiting = Builder.createOr(NewPhiR, Cmp);
6682
6683 // The exiting value may flow through a chain of VPBlendRecipes and
6684 // select recipes (VPInstruction, VPWidenRecipe or VPReplicateRecipe with
6685 // Select opcode) before reaching OrigExitingVPV. Clone each chain link
6686 // in topological order so each clone refers to the already-rewritten i1
6687 // operands via Substitutions.
6688 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6689 {PhiR, NewPhiR}};
6690 std::function<void(VPSingleDefRecipe *)> CloneChain =
6691 [&](VPSingleDefRecipe *Old) {
6692 if (Substitutions.contains(Old))
6693 return;
6695 for (VPValue *Op : Old->operands()) {
6696 if (isa<VPBlendRecipe>(Op) ||
6698 CloneChain(cast<VPSingleDefRecipe>(Op));
6699 NewOps.push_back(Substitutions.lookup_or(Op, Op));
6700 }
6701 VPSingleDefRecipe *New;
6702 if (auto *B = dyn_cast<VPBlendRecipe>(Old))
6703 New = B->cloneWithOperands(NewOps);
6704 else if (auto *W = dyn_cast<VPWidenRecipe>(Old))
6705 New = W->cloneWithOperands(NewOps);
6706 else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Old))
6707 New = Rep->cloneWithOperands(NewOps);
6708 else
6709 New = cast<VPInstruction>(Old)->cloneWithOperands(NewOps);
6710 New->insertBefore(Old);
6711 Substitutions[Old] = New;
6712 };
6713
6714 if (OrigExitingVPV != AnyOfSelect) {
6715 CloneChain(cast<VPSingleDefRecipe>(OrigExitingVPV));
6716 NewExiting = Substitutions.lookup(OrigExitingVPV);
6717 }
6718 NewPhiR->setOperand(1, NewExiting);
6719 PhiR->replaceAllUsesWith(Plan->getPoison(PhiR->getScalarType()));
6720
6721 Builder.setInsertPoint(IP);
6722 FinalReductionResult =
6723 Builder.createAnyOfReduction(NewExiting, NewVal, Start, ExitDL);
6724 } else {
6725 // If the vector reduction can be performed in a smaller type, we
6726 // truncate then extend the loop exit value to enable InstCombine to
6727 // evaluate the entire expression in the smaller type.
6728 VPValue *ReductionOp = NewExitingVPV;
6729 Instruction::CastOps ExtendOpc = Instruction::CastOpsEnd;
6730 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
6731 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
6733 "Unexpected truncated min-max recurrence!");
6734 Type *RdxTy = RdxDesc.getRecurrenceType();
6735 ExtendOpc = RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
6736 {
6737 VPBuilder::InsertPointGuard Guard(Builder);
6738 Builder.setInsertPoint(
6739 std::next(NewExitingVPV->getDefiningRecipe()->getIterator()));
6740 ReductionOp =
6741 Builder.createWidenCast(Instruction::Trunc, NewExitingVPV, RdxTy);
6742 VPWidenCastRecipe *Extnd =
6743 Builder.createWidenCast(ExtendOpc, ReductionOp, PhiTy);
6744 if (PhiR->getOperand(1) == NewExitingVPV)
6745 PhiR->setOperand(1, Extnd);
6746 }
6747 }
6748
6749 VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
6750 PhiR->getFastMathFlagsOrNone());
6751 FinalReductionResult = Builder.createNaryOp(
6752 VPInstruction::ComputeReductionResult, {ReductionOp}, Flags, ExitDL);
6753 if (ExtendOpc != Instruction::CastOpsEnd)
6754 FinalReductionResult = Builder.createScalarCast(
6755 ExtendOpc, FinalReductionResult, PhiTy, {});
6756 }
6757
6758 // Update all users outside the vector region. Also replace redundant
6759 // extracts.
6760 for (auto *U : to_vector(OrigExitingVPV->users())) {
6761 auto *Parent = cast<VPRecipeBase>(U)->getParent();
6762 if (FinalReductionResult == U || Parent->getParent())
6763 continue;
6764 // Skip ComputeReductionResult and FindIV reductions when they are not the
6765 // final result.
6766 if (match(U, m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
6768 match(U, m_VPInstruction<Instruction::ICmp>())))
6769 continue;
6770 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
6771
6772 // Look through ExtractLastPart.
6774 U = cast<VPInstruction>(U)->getSingleUser();
6775
6778 cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
6779 }
6780
6781 RecurKind RK = PhiR->getRecurrenceKind();
6786 VPBuilder PHBuilder(Plan->getVectorPreheader());
6787 VPValue *Iden = Plan->getOrAddLiveIn(
6788 getRecurrenceIdentity(RK, PhiTy, PhiR->getFastMathFlagsOrNone()));
6789 auto *ScaleFactorVPV = Plan->getConstantInt(32, 1);
6790 VPValue *StartV = PHBuilder.createNaryOp(
6792 {PhiR->getStartValue(), Iden, ScaleFactorVPV}, *PhiR);
6793 PhiR->setOperand(0, StartV);
6794 }
6795 }
6796
6798}
6799
6801 VPlan &Plan, GeneratedRTChecks &RTChecks, ElementCount VF, unsigned UF,
6802 bool HasBranchWeights) const {
6803 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
6804 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
6805 assert((!Config.OptForSize ||
6806 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled) &&
6807 "Cannot SCEV check stride or overflow when optimizing for size");
6809 SCEVCheckBlock, HasBranchWeights);
6810 }
6811 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
6812 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
6813 // VPlan-native path does not do any analysis for memory runtime checks
6814 // currently.
6815 assert((!Opts.enable_vplan_native_path || !Plan.isOuterLoop()) &&
6816 "Memory runtime checks are not supported for outer loops yet");
6817
6818 if (Config.OptForSize) {
6819 assert(
6820 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled &&
6821 "Cannot emit memory checks when optimizing for size, unless forced "
6822 "to vectorize.");
6823 ORE->emit([&]() {
6824 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
6825 OrigLoop->getStartLoc(),
6826 OrigLoop->getHeader())
6827 << "Code-size may be reduced by not forcing "
6828 "vectorization, or by source-code modifications "
6829 "eliminating the need for runtime checks "
6830 "(e.g., adding 'restrict').";
6831 });
6832 }
6833 const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
6834 // Erase the temporary IR before recipe expansion can reuse its values.
6835 RTChecks.eraseMemCheckBlock();
6837 OrigLoop, *PSE.getSE(), VF, UF, HasBranchWeights);
6838 }
6839}
6840
6842 VPlan &Plan, ElementCount VF, unsigned UF,
6843 ElementCount MinProfitableTripCount, bool TailFoldedEpilogue) const {
6844 const uint32_t *BranchWeights =
6845 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())
6847 : nullptr;
6848 // If either the main vector loop or the epilogue vector loop is tail-folded,
6849 // it can handle any trip count, so the min iter check never needs
6850 // to branch to the scalar loop.
6852 MinProfitableTripCount, Plan.requiresScalarEpilogue(),
6853 Plan.hasTailFolded() || TailFoldedEpilogue, OrigLoop,
6854 BranchWeights,
6855 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
6856 PSE, Plan.getEntry());
6857}
6858
6859// Determine how to lower the epilogue, which depends on 1) optimising
6860// for minimum code-size, 2) tail-folding compiler options, 3) loop
6861// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
6862// is suitable for tail-folding.
6863// This function determines epilogue lowering for the main vector loop while
6864// epilogue lowering for the tail-folded epilogue path will be handled
6865// separately in getEpilogueTailLowering.
6866static EpilogueLowering
6867getEpilogueLowering(const VectorizeOptions &Opts, LoopVectorizeHints &Hints,
6868 bool OptForSize, TargetTransformInfo *TTI,
6870 InterleavedAccessInfo *IAI) {
6871 // 1) OptSize takes precedence over all other options, i.e. if this is set,
6872 // don't look at hints or options, and don't request an epilogue.
6873 if (OptForSize)
6875
6876 // 2) If set, obey the directives
6877 if (Opts.tail_folding_policy) {
6878 switch (*Opts.tail_folding_policy) {
6880 return CM_EpilogueAllowed;
6885 };
6886 }
6887
6888 // 3) If set, obey the hints
6889 switch (Hints.getPredicate()) {
6893 return CM_EpilogueAllowed;
6894 };
6895
6896 // 4) if the TTI hook indicates this is profitable, request tail-folding.
6897 TailFoldingInfo TFI(TLI, &LVL, IAI);
6898 if (TTI->preferTailFoldingOverEpilogue(&TFI))
6900
6901 return CM_EpilogueAllowed;
6902}
6903
6904// Emit a remark if there are stores to floats that required a floating point
6905// extension. If the vectorized loop was generated with floating point there
6906// will be a performance penalty from the conversion overhead and the change in
6907// the vector width.
6910 for (BasicBlock *BB : L->getBlocks()) {
6911 for (Instruction &Inst : *BB) {
6912 if (auto *S = dyn_cast<StoreInst>(&Inst)) {
6913 if (S->getValueOperand()->getType()->isFloatTy())
6914 Worklist.push_back(S);
6915 }
6916 }
6917 }
6918
6919 // Traverse the floating point stores upwards searching, for floating point
6920 // conversions.
6923 while (!Worklist.empty()) {
6924 auto *I = Worklist.pop_back_val();
6925 if (!L->contains(I))
6926 continue;
6927 if (!Visited.insert(I).second)
6928 continue;
6929
6930 // Emit a remark if the floating point store required a floating
6931 // point conversion.
6932 // TODO: More work could be done to identify the root cause such as a
6933 // constant or a function return type and point the user to it.
6934 if (isa<FPExtInst>(I) && EmittedRemark.insert(I).second)
6935 ORE->emit([&]() {
6936 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
6937 I->getDebugLoc(), L->getHeader())
6938 << "floating point conversion changes vector width. "
6939 << "Mixed floating point precision requires an up/down "
6940 << "cast that will negatively impact performance.";
6941 });
6942
6943 for (Use &Op : I->operands())
6944 if (auto *OpI = dyn_cast<Instruction>(Op))
6945 Worklist.push_back(OpI);
6946 }
6947}
6948
6949/// For loops with uncountable early exits, find the cost of doing work when
6950/// exiting the loop early, such as calculating the final exit values of
6951/// variables used outside the loop. The loop leaves through at most one of the
6952/// vector.early.exit blocks, so return the maximum of their costs.
6953/// TODO: This is currently overly pessimistic because the loop may not take
6954/// the early exit, but better to keep this conservative for now. In future,
6955/// it might be possible to relax this by using branch probabilities.
6957 VPlan &Plan, ElementCount VF) {
6958 InstructionCost Cost = 0;
6959 for (auto *ExitVPBB : Plan.getExitBlocks()) {
6960 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
6961 // If the predecessor is not the middle.block, then it must be the
6962 // vector.early.exit block, which may contain work to calculate the exit
6963 // values of variables used outside the loop.
6964 if (PredVPBB != Plan.getMiddleBlock()) {
6965 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
6966 << PredVPBB->getName() << ":\n");
6967 Cost = std::max(Cost, PredVPBB->cost(VF, CostCtx));
6968 }
6969 }
6970 }
6971 return Cost;
6972}
6973
6974/// This function determines whether or not it's still profitable to vectorize
6975/// the loop given the extra work we have to do outside of the loop:
6976/// 1. Perform the runtime checks before entering the loop to ensure it's safe
6977/// to vectorize.
6978/// 2. In the case of loops with uncountable early exits, we may have to do
6979/// extra work when exiting the loop early, such as calculating the final
6980/// exit values of variables used outside the loop.
6981/// 3. The middle block.
6982static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
6983 VectorizationFactor &VF, Loop *L,
6985 VPCostContext &CostCtx, VPlan &Plan,
6986 EpilogueLowering SEL,
6987 std::optional<unsigned> VScale) {
6988 InstructionCost RtC = Checks.getCost();
6989 if (!RtC.isValid())
6990 return false;
6991
6992 // When interleaving only scalar and vector cost will be equal, which in turn
6993 // would lead to a divide by 0. Fall back to hard threshold.
6994 if (VF.Width.isScalar()) {
6995 // TODO: Should we rename VectorizeMemoryCheckThreshold?
6997 LLVM_DEBUG(
6998 dbgs()
6999 << "LV: Interleaving only is not profitable due to runtime checks\n");
7000 return false;
7001 }
7002 return true;
7003 }
7004
7005 // The scalar cost should only be 0 when vectorizing with a user specified
7006 // VF/IC. In those cases, runtime checks should always be generated.
7007 uint64_t ScalarC = VF.ScalarCost.getValue();
7008 if (ScalarC == 0)
7009 return true;
7010
7011 // The vector loop is left through either a vector.early.exit block or the
7012 // middle block, so add the cost of the more expensive of the two.
7013 InstructionCost EarlyExitCost =
7014 calculateEarlyExitCost(CostCtx, Plan, VF.Width);
7015 InstructionCost MiddleCost = Plan.getMiddleBlock()->cost(VF.Width, CostCtx);
7016 InstructionCost TotalCost = RtC + std::max(EarlyExitCost, MiddleCost);
7017
7018 // First, compute the minimum iteration count required so that the vector
7019 // loop outperforms the scalar loop.
7020 // The total cost of the scalar loop is
7021 // ScalarC * TC
7022 // where
7023 // * TC is the actual trip count of the loop.
7024 // * ScalarC is the cost of a single scalar iteration.
7025 //
7026 // The total cost of the vector loop is
7027 // TotalCost + VecC * (TC / VF) + EpiC
7028 // where
7029 // * TotalCost is the sum of the costs of
7030 // - the generated runtime checks, i.e. RtC
7031 // - the most expensive way of leaving the vector loop, i.e. the maximum
7032 // of the costs of the middle block and, for loops with uncountable
7033 // early exits, the vector.early.exit blocks.
7034 // * VecC is the cost of a single vector iteration.
7035 // * TC is the actual trip count of the loop
7036 // * VF is the vectorization factor
7037 // * EpiCost is the cost of the generated epilogue, including the cost
7038 // of the remaining scalar operations.
7039 //
7040 // Vectorization is profitable once the total vector cost is less than the
7041 // total scalar cost:
7042 // TotalCost + VecC * (TC / VF) + EpiC < ScalarC * TC
7043 //
7044 // Now we can compute the minimum required trip count TC as
7045 // VF * (TotalCost + EpiC) / (ScalarC * VF - VecC) < TC
7046 //
7047 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
7048 // the computations are performed on doubles, not integers and the result
7049 // is rounded up, hence we get an upper estimate of the TC.
7050 unsigned IntVF = estimateElementCount(VF.Width, VScale);
7051 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
7052 uint64_t MinTC1 =
7053 Div == 0 ? 0 : divideCeil(TotalCost.getValue() * IntVF, Div);
7054
7055 // Second, compute a minimum iteration count so that the cost of the
7056 // runtime checks is only a fraction of the total scalar loop cost. This
7057 // adds a loop-dependent bound on the overhead incurred if the runtime
7058 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
7059 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
7060 // cost, compute
7061 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
7062 uint64_t MinTC2 = divideCeil(RtC.getValue() * 10, ScalarC);
7063
7064 // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
7065 // is allowed, choose the next closest multiple of VF. This should partly
7066 // compensate for ignoring the epilogue cost.
7067 uint64_t MinTC = std::max(MinTC1, MinTC2);
7068 if (SEL == CM_EpilogueAllowed)
7069 MinTC = alignTo(MinTC, IntVF);
7071
7072 LLVM_DEBUG(
7073 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
7074 << VF.MinProfitableTripCount << "\n");
7075
7076 // Skip vectorization if the expected trip count is less than the minimum
7077 // required trip count.
7078 if (auto ExpectedTC = getSmallBestKnownTC(CostCtx.Opts, PSE, L)) {
7079 if (ElementCount::isKnownLT(*ExpectedTC, VF.MinProfitableTripCount)) {
7080 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
7081 "trip count < minimum profitable VF ("
7082 << *ExpectedTC << " < " << VF.MinProfitableTripCount
7083 << ")\n");
7084
7085 return false;
7086 }
7087 }
7088 return true;
7089}
7090
7092 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced),
7093 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {}
7094
7095/// ResumeForEpilogue markers in the main plan, used by the epilogue plan.
7100
7101/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
7102/// vectorization.
7104 using namespace VPlanPatternMatch;
7105 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
7106 // introduce multiple uses of undef/poison. If the reduction start value may
7107 // be undef or poison it needs to be frozen and the frozen start has to be
7108 // used when computing the reduction result. We also need to use the frozen
7109 // value in the resume phi generated by the main vector loop, as this is also
7110 // used to compute the reduction result after the epilogue vector loop. The
7111 // epilogue plan re-uses the frozen values.
7112 VPBuilder Builder(MainPlan.getEntry());
7113 for (VPInstruction &VPI :
7115 VPValue *OrigStart;
7116 if (!matchFindIVResult(&VPI, m_VPValue(), m_VPValue(OrigStart)))
7117 continue;
7119 continue;
7120 VPInstruction *Freeze = Builder.createFreeze(OrigStart, {}, "fr");
7121 VPI.setOperand(2, Freeze);
7122 OrigStart->replaceUsesWithIf(Freeze, IsaPred<VPPhi>);
7123 }
7124
7125 VPValue *VectorTC = nullptr;
7126 auto *Term =
7128 [[maybe_unused]] bool MatchedTC =
7129 match(Term, m_BranchOnCount(m_VPValue(), m_VPValue(VectorTC)));
7130 assert(MatchedTC && "must match vector trip count");
7131
7132 VPBasicBlock *MiddleVPBB = MainPlan.getMiddleBlock();
7133 VPBuilder MiddleBuilder(MiddleVPBB->getFirstNonPhi());
7134 VPInstruction *VectorTCMarker = MiddleBuilder.createNaryOp(
7136 {VectorTC, MainPlan.getZero(VectorTC->getScalarType())});
7137
7138 // Replace the resume phis of the VPIRPhis in the scalar header of the main
7139 // plan by ResumeForEpilogue instructions for their incoming values and return
7140 // them so they can be used as resume values when vectorizing the epilogue.
7141 // The epilogue plan creates the resume phis after the main plan executed.
7142 VPRecipeBase *MiddleTerm = MiddleVPBB->getTerminator();
7143 MiddleBuilder.setInsertPoint(MiddleTerm ? MiddleTerm->getIterator()
7144 : MiddleVPBB->end());
7145 auto ResumeValues = to_vector(
7146 map_range(MainPlan.getScalarHeader()->phis(), [&](VPRecipeBase &R) {
7147 auto *ResumePhi = cast<VPPhi>(R.getOperand(0));
7148 assert(ResumePhi->getNumIncoming() == 2 &&
7149 ResumePhi->getIncomingBlock(0) == MiddleVPBB &&
7150 "expected resume phi merging middle block and bypass value");
7151 return MiddleBuilder.createNaryOp(VPInstruction::ResumeForEpilogue,
7152 ResumePhi->operands());
7153 }));
7154 for (VPRecipeBase &R :
7156 auto *ResumePhi = cast<VPPhi>(&R);
7157 ResumePhi->replaceAllUsesWith(ResumePhi->getOperand(1));
7158 ResumePhi->eraseFromParent();
7159 }
7160 return {VectorTCMarker, std::move(ResumeValues)};
7161}
7162
7163/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
7164/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
7166 VPlan &MainPlan, VPlan &Plan, const SCEV2ValueTy &ExpandedSCEVs,
7167 ElementCount MainLoopVF, unsigned MainLoopUF, ElementCount EpilogueVF,
7169 const MainPlanResumeMarkers &Markers, VPBasicBlock *MainCheck) {
7170 // Build a map from the scalar-header PHI to the ResumeForEpilogue markers
7171 // from the main plan.
7172 // TODO: Replace the IR PHI key.
7173 DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
7174 for (auto [HeaderPhi, ResumeForEpi] :
7175 zip_equal(MainPlan.getScalarHeader()->phis(), Markers.ResumeValues))
7176 IRPhiToResumeForEpi[&cast<VPIRPhi>(HeaderPhi).getIRPhi()] = ResumeForEpi;
7177 VPIRBasicBlock *EpilogueCheck = RUN_VPLAN_PASS(
7178 VPlanTransforms::modelGeneratedMainLoopBlocks, Plan, MainPlan, MainCheck);
7179 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
7180 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
7181 Header->setName("vec.epilog.vector.body");
7182
7183 // When skipping the epilogue vector loop, the scalar loop resumes at the
7184 // values computed by the main vector loop.
7185 for (auto [R, ResumeForEpi] :
7186 zip_equal(Plan.getScalarHeader()->phis(), Markers.ResumeValues)) {
7187 // A tail-folded epilogue's middle block never branches to the scalar
7188 // preheader, so it has no resume phis and the scalar loop starts at the
7189 // original start values. The epilogue check never skips a tail-folded
7190 // epilogue either, so there is no resume value to set.
7191 auto *ResumePhi = dyn_cast<VPPhi>(R.getOperand(0));
7192 if (!ResumePhi) {
7193 assert(Plan.hasTailFolded() &&
7194 "expected resume phis unless the epilogue is tail-folded");
7195 continue;
7196 }
7197 ResumePhi->setIncomingValueForBlock(
7198 EpilogueCheck, Plan.getOrAddLiveIn(ResumeForEpi->getUnderlyingValue()));
7199 }
7200
7201 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
7202 assert(VectorPH->getNumPredecessors() == 2 &&
7203 VectorPH->getPredecessors()[0] == EpilogueCheck &&
7204 "vector preheader must be entered from the epilogue check first");
7205 // Returns the value to resume at for \p ResumeForEpi: a phi in the vector
7206 // preheader merging the value computed by the main vector loop with the
7207 // value used when the main vector loop has been bypassed, or the former if
7208 // both are equal.
7209 auto GetResumeValue = [&](VPInstruction *ResumeForEpi,
7210 const Twine &Name) -> VPValue * {
7211 VPValue *MainV = Plan.getOrAddLiveIn(ResumeForEpi->getUnderlyingValue());
7212 VPValue *BypassV =
7213 Plan.getOrAddLiveIn(ResumeForEpi->getOperand(1)->getUnderlyingValue());
7214 if (MainV == BypassV)
7215 return MainV;
7216 return VPBuilder(VectorPH->getFirstNonPhi())
7217 .createScalarPhi({MainV, BypassV}, {}, Name);
7218 };
7219
7220 VPValue *IV = VectorLoop->getCanonicalIV();
7221 // When vectorizing the epilogue loop, the canonical induction needs to start
7222 // at the resume value from the main vector loop. Find the resume value
7223 // created during execution of the main VPlan. Add this resume value as an
7224 // offset to the canonical IV of the epilogue loop.
7225 VPValue *VPV = GetResumeValue(Markers.VectorTC, "vec.epilog.resume.val");
7226 assert(all_of(IV->users(),
7227 [](const VPUser *U) {
7228 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7229 return true;
7230 auto *VPI = dyn_cast<VPInstruction>(U);
7231 if (!VPI)
7232 return false;
7233 // Only users linear in the canonical IV (e.g. scaling it by
7234 // a stride) stay correct when offsetting it below; others,
7235 // like header masks, may need an epilogue-relative IV.
7236 // The active-lane-masks of a tail-folded epilogue compare
7237 // against the full trip count, so they expect the offset
7238 // IV, like the active-lane-mask phi's entry value.
7239 unsigned Opc = VPI->getOpcode();
7240 return Instruction::isCast(Opc) ||
7241 is_contained({Instruction::Add, Instruction::Mul,
7242 Instruction::Shl},
7243 Opc) ||
7245 }) &&
7246 "the canonical IV should only be used by its increment, casts, "
7247 "stride scaling, ScalarIVSteps or DerivedIV when resetting the start "
7248 "value");
7249 VPBuilder Builder(Header->getFirstNonPhi());
7250 VPInstruction *Add = Builder.createAdd(IV, VPV);
7251 // Replace all users of the canonical IV and its increment with the offset
7252 // version, except for the Add itself and the canonical IV increment.
7254 assert(Increment && "Must have a canonical IV increment at this point");
7255 IV->replaceUsesWithIf(
7256 Add, [Add, Increment](VPUser *U) { return U != Add && U != Increment; });
7257 VPInstruction *OffsetIVInc =
7259 Increment->replaceAllUsesWith(OffsetIVInc);
7260 OffsetIVInc->setOperand(0, Increment);
7261
7262 // Resume values must be created in the vector preheader.
7263 VPBuilder PHBuilder(VectorPH->getFirstNonPhi());
7264
7265 // Ensure that the start values for all header phi recipes are updated before
7266 // vectorizing the epilogue loop.
7267 for (VPRecipeBase &R : Header->phis()) {
7268 VPValue *ResumeVPV = nullptr;
7269 // TODO: Move setting of resume values to prepareToExecute.
7270 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
7271 // Find the reduction result by searching users of the phi or its backedge
7272 // value.
7273 auto IsReductionResult = [](VPRecipeBase *R) {
7274 auto *VPI = dyn_cast<VPInstruction>(R);
7275 return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
7276 };
7277 auto *RdxResult = cast<VPInstruction>(
7278 vputils::findRecipe(ReductionPhi->getBackedgeValue(), IsReductionResult));
7279 assert(RdxResult && "expected to find reduction result");
7280
7281 VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
7282 cast<PHINode>(ReductionPhi->getUnderlyingInstr()));
7283
7284 // Check for FindIV pattern by looking for icmp user of RdxResult.
7285 // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
7286 using namespace VPlanPatternMatch;
7287 VPValue *SentinelVPV = nullptr;
7288 bool IsFindIV = any_of(RdxResult->users(), [&](VPUser *U) {
7289 return match(U, VPlanPatternMatch::m_SpecificICmp(
7290 ICmpInst::ICMP_NE, m_Specific(RdxResult),
7291 m_VPValue(SentinelVPV)));
7292 });
7293
7294 RecurKind RK = ReductionPhi->getRecurrenceKind();
7295 ResumeVPV = GetResumeValue(ResumeForEpi, "bc.merge.rdx");
7296 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) || IsFindIV) {
7297 VPValue *BypassOp = ResumeForEpi->getOperand(1);
7298 assert((isa<VPIRValue>(BypassOp) ||
7300 BypassOp,
7301 m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
7302 "expected live-in or Freeze");
7303 VPValue *StartV = Plan.getOrAddLiveIn(BypassOp->getUnderlyingValue());
7305 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
7306 // start value; compare the final value from the main vector loop
7307 // to the start value.
7308 ResumeVPV = PHBuilder.createICmp(CmpInst::ICMP_NE, ResumeVPV, StartV);
7309 } else {
7310 assert(isa<VPIRValue>(SentinelVPV) &&
7311 "sentinel must be a live-in to be used in the preheader");
7312 // Use the start value frozen for the main plan, if any, when
7313 // computing the reduction result.
7314 if (match(BypassOp, m_Freeze(m_VPValue())))
7315 for (VPUser *U : RdxResult->users()) {
7316 auto *VPI = dyn_cast<VPInstruction>(U);
7317 if (VPI && matchFindIVResult(VPI, m_VPValue(), m_VPValue()))
7318 VPI->setOperand(2, StartV);
7319 }
7320
7321 // Adjust resume: select(icmp eq ResumeVPV, StartV), Sentinel,
7322 // ResumeVPV
7323 VPValue *Cmp =
7324 PHBuilder.createICmp(CmpInst::ICMP_EQ, ResumeVPV, StartV);
7325 ResumeVPV = PHBuilder.createSelect(Cmp, SentinelVPV, ResumeVPV);
7326 }
7327 // TODO: materializeBroadcasts does not cover values in the vector
7328 // preheader.
7329 ReductionPhi->setStartValue(
7330 PHBuilder.createNaryOp(VPInstruction::Broadcast, ResumeVPV));
7331 continue;
7332 }
7333 if (auto *VPI = dyn_cast<VPInstruction>(ReductionPhi->getStartValue())) {
7335 "unexpected start value");
7336 // Partial sub-reductions always start at 0 and account for the
7337 // reduction start value in a final subtraction. Update it to use the
7338 // resume value from the main vector loop.
7339 if (ReductionPhi->getVFScaleFactor() > 1 &&
7341 auto *Sub = cast<VPInstruction>(RdxResult->getSingleUser());
7342 assert((Sub->getOpcode() == Instruction::Sub ||
7343 Sub->getOpcode() == Instruction::FSub) &&
7344 "Unexpected opcode");
7345 assert(isa<VPIRValue>(Sub->getOperand(0)) &&
7346 "Expected operand to match the original start value of the "
7347 "reduction");
7348 // For integer sub-reductions, verify start value is zero.
7349 // For FP sub-reductions, verify start value is negative zero.
7350 [[maybe_unused]] auto StartValueIsIdentity = [&] {
7351 Value *IdentityValue =
7352 getRecurrenceIdentity(RK, ResumeVPV->getScalarType(),
7353 ReductionPhi->getFastMathFlagsOrNone());
7354 auto *StartValue = dyn_cast<VPIRValue>(VPI->getOperand(0));
7355 return StartValue && StartValue->getValue() == IdentityValue;
7356 };
7357 assert(StartValueIsIdentity() &&
7358 "Expected start value for partial sub-reduction to be zero "
7359 "(or negative zero)");
7360
7361 Sub->setOperand(0, ResumeVPV);
7362 } else {
7363 // The start vector may be shared by multiple reductions, so create
7364 // a new one using the resume.
7365 auto *NewStart = cast<VPInstruction>(VPI->clone());
7366 NewStart->setOperand(0, ResumeVPV);
7367 NewStart->insertBefore(VPI);
7368 ReductionPhi->setStartValue(NewStart);
7369 }
7370 continue;
7371 }
7372 } else if (isa<VPActiveLaneMaskPHIRecipe>(&R)) {
7373 // The active-lane-mask phi's entry value was computed assuming the
7374 // epilogue vector loop starts at 0. Rebuild it using the resume value
7375 // instead, so the mask reflects how many elements the main vector loop
7376 // already processed.
7377 VPBuilder EntryBuilder(Plan.getVectorPreheader());
7378 auto *EntryALM = EntryBuilder.createNaryOp(
7379 VPInstruction::ActiveLaneMask, {VPV, Plan.getTripCount()},
7380 R.getDebugLoc(), "active.lane.mask.entry");
7381 cast<VPHeaderPHIRecipe>(&R)->setStartValue(EntryALM);
7382 continue;
7383 } else {
7384 // Retrieve the induction resume value via ResumeForEpilogue.
7385 PHINode *IndPhi =
7387 ? cast<VPWidenInductionRecipe>(&R)->getPHINode()
7388 : cast<VPConditionalInductionPHIRecipe>(&R)->getPHINode();
7389 ResumeVPV =
7390 GetResumeValue(IRPhiToResumeForEpi.at(IndPhi), "bc.resume.val");
7391 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R);
7392 if (IntOrFpIV && IntOrFpIV->isTruncated()) {
7393 // The resume value is for the original induction. Preserve any offset
7394 // folded into the truncated induction's start.
7395 // TODO: This is a short-term solution, as it relies on the induction
7396 // descriptor's IR start value. Compute the resume value with
7397 // createDerivedIV from the recipe's start and step instead.
7398 VPValue *OrigStart = Plan.getOrAddLiveIn(
7399 IntOrFpIV->getInductionDescriptor().getStartValue());
7400 VPValue *NarrowStart = IntOrFpIV->getStartValue();
7401 if (NarrowStart != OrigStart) {
7402 VPBuilder OffsetBuilder(VectorPH);
7403 VPValue *Offset = OffsetBuilder.createSub(NarrowStart, OrigStart);
7404 ResumeVPV = OffsetBuilder.createAdd(ResumeVPV, Offset);
7405 }
7406 }
7407 }
7408 assert(ResumeVPV && "Must have a resume value");
7409 cast<VPHeaderPHIRecipe>(&R)->setStartValue(ResumeVPV);
7410 }
7411
7412 // Re-use the SCEVs expanded for the main loop instead of expanding them
7413 // again, which also lets simplifyLiveInsWithSCEV fold their users.
7414 // TODO: Remove once the epilogue plan uses the main plan's VPValues directly.
7416 make_isa_range<VPExpandSCEVRecipe>(*Plan.getEntry()))) {
7417 assert(ExpandedSCEVs.contains(ExpandR.getSCEV()) &&
7418 "Epilogue plan needs a SCEV not expanded for the main loop");
7419 VPValue *ExpandedVal =
7420 Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR.getSCEV()));
7421 ExpandR.replaceAllUsesWith(ExpandedVal);
7422 if (Plan.getTripCount() == &ExpandR)
7423 Plan.resetTripCount(ExpandedVal);
7424 ExpandR.eraseFromParent();
7425 }
7426
7427 auto VScale = Config.getVScaleForTuning();
7428 unsigned MainLoopStep = estimateElementCount(MainLoopVF * MainLoopUF, VScale);
7429 unsigned EpilogueLoopStep = estimateElementCount(EpilogueVF, VScale);
7431 Plan.getOrAddLiveIn(Markers.VectorTC->getUnderlyingValue()),
7432 Plan.requiresScalarEpilogue(), EpilogueVF, MainLoopStep,
7433 EpilogueLoopStep, SE);
7434}
7435
7437 const VectorizeOptions &Opts = getVectorizeOptions(*L);
7438 assert((Opts.enable_vplan_native_path || L->isInnermost()) &&
7439 "VPlan-native path is not enabled. Only process inner loops.");
7440
7441 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
7442 << L->getHeader()->getParent()->getName() << "' from "
7443 << L->getLocStr() << "\n");
7444
7445 LoopVectorizeHints Hints(
7446 Opts, L, InterleaveOnlyWhenForced || !Opts.interleave_loops, *ORE, TTI);
7447
7448 LLVM_DEBUG(
7449 dbgs() << "LV: Loop hints:"
7450 << " force="
7452 ? "disabled"
7454 ? "enabled"
7455 : "?"))
7456 << " width=" << Hints.getWidth()
7457 << " interleave=" << Hints.getInterleave() << "\n");
7458
7459 // Function containing loop
7460 Function *F = L->getHeader()->getParent();
7461
7462 // Looking at the diagnostic output is the only way to determine if a loop
7463 // was vectorized (other than looking at the IR or machine code), so it
7464 // is important to generate an optimization remark for each loop. Most of
7465 // these messages are generated as OptimizationRemarkAnalysis. Remarks
7466 // generated as OptimizationRemark and OptimizationRemarkMissed are
7467 // less verbose reporting vectorized loops and unvectorized loops that may
7468 // benefit from vectorization, respectively.
7469
7470 if (!Hints.allowVectorization(L, VectorizeOnlyWhenForced ||
7471 !Opts.vectorize_loops)) {
7472 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
7473 return false;
7474 }
7475
7476 PredicatedScalarEvolution PSE(*SE, *L);
7477
7478 // Query this against the original loop and save it here because the profile
7479 // of the original loop header may change as the transformation happens.
7480 bool OptForSize = llvm::shouldOptimizeForSize(
7481 L->getHeader(), PSI,
7482 PSI && PSI->hasProfileSummary() ? &GetBFI() : nullptr,
7485
7486 // Check if it is legal to vectorize the loop.
7487 LoopVectorizationRequirements Requirements;
7488 LoopVectorizationLegality LVL(Opts, L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
7489 &Requirements, &Hints, DB, AC,
7490 /*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
7491 if (!LVL.canVectorize(Opts.enable_vplan_native_path)) {
7492 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
7493 Hints.emitRemarkWithHints();
7494 return false;
7495 }
7496
7497 bool IsInnerLoop = L->isInnermost();
7498
7499 // Outer loops require a computable trip count.
7500 if (!IsInnerLoop && isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
7501 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
7502 return false;
7503 }
7504
7505 if (LVL.hasUncountableEarlyExit()) {
7506 if (!Opts.enable_early_exit_vectorization) {
7507 reportVectorizationFailure("Auto-vectorization of loops with uncountable "
7508 "early exit is not enabled",
7509 "UncountableEarlyExitLoopsDisabled", ORE, L);
7510 return false;
7511 }
7513 !Opts.enable_early_exit_vectorization_with_side_effects) {
7514 reportVectorizationFailure("Auto-vectorization of loops with uncountable "
7515 "early exit and side effects is not enabled",
7516 "UncountableEarlyExitSideEffectLoopsDisabled",
7517 ORE, L);
7518 return false;
7519 }
7520 }
7521
7522 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
7523 // If an override option has been passed in for interleaved accesses, use it.
7524 bool UseInterleaved =
7525 IsInnerLoop && valueOr(Opts.enable_interleaved_mem_accesses,
7526 TTI->enableInterleavedAccessVectorization());
7527
7528 // Analyze interleaved memory accesses.
7529 if (UseInterleaved) {
7531 // If the loop exits from a block other than the latch, the scalar
7532 // iteration reserved for the exit may leave the loop before reaching the
7533 // grouped accesses.
7534 if (L->getExitingBlock() != L->getLoopLatch() &&
7537 }
7538
7539 if (LVL.hasUncountableEarlyExit()) {
7540 BasicBlock *LoopLatch = L->getLoopLatch();
7541 if (IAI.requiresScalarEpilogue() ||
7542 any_of(LVL.getCountableExitingBlocks(), not_equal_to(LoopLatch))) {
7543 reportVectorizationFailure("Auto-vectorization of early exit loops "
7544 "requiring a scalar epilogue is unsupported",
7545 "UncountableEarlyExitUnsupported", ORE, L);
7546 return false;
7547 }
7548 }
7549
7550 EpilogueLowering SEL =
7551 getEpilogueLowering(Opts, Hints, OptForSize, TTI, TLI, LVL, &IAI);
7552
7553 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
7554 // count by optimizing for size, to minimize overheads.
7555 auto ExpectedTC = getSmallBestKnownTC(Opts, PSE, L);
7556 if (ExpectedTC && ExpectedTC->isFixed() &&
7557 ExpectedTC->getFixedValue() < Opts.vectorizer_min_trip_count) {
7558 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
7559 << "This loop is worth vectorizing only if no scalar "
7560 << "epilogue is required.");
7562 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
7563 else {
7564 LLVM_DEBUG(dbgs() << "\n");
7565 // For loops with very small trip counts, require that no scalar epilogue
7566 // is needed. Tail-folded loops are still allowed since they avoid
7567 // epilogues, unless the trip count is below the target's minimum for
7568 // tail-folding. In that case the tail cannot be folded, so the loop
7569 // should be treated like any other low trip count loop.
7570 // TODO: Remove the check for CM_EpilogueNotAllowedOptSize here. The
7571 // EpilogueLowering enum currently conflates several pieces of
7572 // information:
7573 // * Whether a scalar epilogue is permitted
7574 // * Whether tail folding is preferred or required
7575 // * Whether optimizing for code size
7576 // Overwriting it here can drop OptForSize, breaking the
7577 // runtimeChecksRequired() bail-out in computeMaxVF. Split
7578 // EpilogueLowering into separate fields instead.
7579 if (SEL != CM_EpilogueNotAllowedOptSize &&
7581 ExpectedTC->getFixedValue() <=
7582 TTI->getMinTripCountTailFoldingThreshold()))
7584 }
7585 }
7586
7587 // Check the function attributes to see if implicit floats or vectors are
7588 // allowed.
7589 if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
7591 "Can't vectorize when the NoImplicitFloat attribute is used",
7592 "loop not vectorized due to NoImplicitFloat attribute",
7593 "NoImplicitFloat", ORE, L);
7594 Hints.emitRemarkWithHints();
7595 return false;
7596 }
7597
7598 // Check if the target supports potentially unsafe FP vectorization.
7599 // FIXME: Add a check for the type of safety issue (denormal, signaling)
7600 // for the target we're vectorizing for, to make sure none of the
7601 // additional fp-math flags can help.
7602 if (Hints.isPotentiallyUnsafe() &&
7603 TTI->isFPVectorizationPotentiallyUnsafe()) {
7605 "Potentially unsafe FP op prevents vectorization",
7606 "loop not vectorized due to unsafe FP support.", "UnsafeFP", ORE, L);
7607 Hints.emitRemarkWithHints();
7608 return false;
7609 }
7610
7611 // If the flag is set, use that instead and override the TTI behaviour.
7612 bool AllowOrderedReductions =
7613 valueOr(Opts.force_ordered_reductions, TTI->enableOrderedReductions());
7614 if (!LVL.canVectorizeFPMath(AllowOrderedReductions)) {
7615 ORE->emit([&]() {
7616 auto *ExactFPMathInst = Requirements.getExactFPInst();
7617 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
7618 ExactFPMathInst->getDebugLoc(),
7619 ExactFPMathInst->getParent())
7620 << "loop not vectorized: cannot prove it is safe to reorder "
7621 "floating-point operations";
7622 });
7623 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
7624 "reorder floating-point operations\n");
7625 Hints.emitRemarkWithHints();
7626 return false;
7627 }
7628
7629 // Use the cost model.
7630 VFSelectionContext Config(Opts, *TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
7631 OptForSize);
7632 // Use the planner for vectorization.
7634 L, LI, DT, TLI, *TTI, &LVL,
7635 std::make_unique<LoopVectorizationCostModel>(
7636 SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE, GetBFI, F, IAI, Config),
7637 Config, PSE, ORE, GetBPI);
7638
7639 // Get user vectorization factor and interleave count.
7640 ElementCount UserVF = Hints.getWidth();
7641 unsigned UserIC = Hints.getInterleave();
7642 // Outer loops don't have LoopAccessInfo, so skip the safety check and reset
7643 // UserIC (interleaving is not supported for outer loops).
7644 if (!IsInnerLoop)
7645 UserIC = 0;
7646 else if (UserIC > 1 && !LVL.isSafeForAnyVectorWidth())
7647 UserIC = 1;
7648
7649 // Plan how to best vectorize.
7650 LVP.plan(UserVF, UserIC);
7651 auto [VF, BestPlanPtr] = LVP.computeBestVF();
7652 unsigned IC = 1;
7653
7654 // For VPlan build stress testing of outer loops, bail after plan
7655 // construction.
7656 if (!IsInnerLoop && Opts.vplan_build_outerloop_stress_test)
7657 return false;
7658
7659 if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
7661
7662 assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
7663 "Did not expect to alias-mask outer loop");
7664
7665 GeneratedRTChecks Checks(Opts, PSE, DT, LI, TTI, Config.CostKind,
7667 if (LVP.hasPlanWithVF(VF.Width)) {
7668 // Select the interleave count. Outer loops are not interleaved.
7669 if (IsInnerLoop)
7670 IC = LVP.selectInterleaveCount(*BestPlanPtr, VF.Width, VF.Cost);
7671
7672 unsigned SelectedIC = UserIC > 0 ? UserIC : IC;
7673 // Optimistically generate runtime checks if they are needed. Drop them if
7674 // they turn out to not be profitable.
7675 if (VF.Width.isVector() || SelectedIC > 1) {
7676 Checks.create(L, LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC,
7677 *ORE);
7678
7679 // Bail out early if either the SCEV or memory runtime checks are known to
7680 // fail. In that case, the vector loop would never execute.
7681 using namespace llvm::PatternMatch;
7682 if ((Checks.getSCEVChecks().first &&
7683 match(Checks.getSCEVChecks().first, m_One())) ||
7684 (Checks.getMemRuntimeChecks().first &&
7685 match(Checks.getMemRuntimeChecks().first, m_One()))) {
7687 "runtime checks are known to fail, so we will never enter the "
7688 "vector loop",
7689 "RuntimeChecksNeverEnterVectorLoop", ORE, L);
7690 return false;
7691 }
7692 }
7693
7694 // Check if it is profitable to vectorize with runtime checks.
7695 bool ForceVectorization =
7697 assert((IsInnerLoop || ForceVectorization) &&
7698 "Outer loops must be forced to vectorize");
7699 VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
7700 /*ReusePrintingSlotTracker=*/true);
7701 if (!ForceVectorization &&
7702 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, *BestPlanPtr,
7703 SEL, Config.getVScaleForTuning())) {
7704 ORE->emit([&]() {
7706 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
7707 L->getHeader())
7708 << "loop not vectorized: cannot prove it is safe to reorder "
7709 "memory operations";
7710 });
7711 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
7712 Hints.emitRemarkWithHints();
7713 return false;
7714 }
7715 }
7716
7717 // Identify the diagnostic messages that should be produced.
7718 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
7719 bool VectorizeLoop = true, InterleaveLoop = true;
7720 if (VF.Width.isScalar()) {
7721 if (LVP.hasVectorPlan()) {
7722 LLVM_DEBUG(
7723 dbgs() << "LV: Vectorization is possible but not beneficial.\n");
7724 VecDiagMsg = {
7725 "VectorizationNotBeneficial",
7726 "the cost-model indicates that vectorization is not beneficial"};
7727 } else {
7728 LLVM_DEBUG(dbgs() << "LV: Vectorization is not possible. Failed to "
7729 "create any vector VPlans.\n");
7730 VecDiagMsg = {"VectorizationNotPossible",
7731 "vectorization is not possible"};
7732 }
7733 VectorizeLoop = false;
7734 }
7735
7736 if (UserIC == 1 && Hints.getInterleave() > 1) {
7738 "UserIC should only be ignored due to unsafe dependencies");
7739 LLVM_DEBUG(dbgs() << "LV: Ignoring user-specified interleave count.\n");
7740 IntDiagMsg = {"InterleavingUnsafe",
7741 "Ignoring user-specified interleave count due to possibly "
7742 "unsafe dependencies in the loop."};
7743 InterleaveLoop = false;
7744 } else if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
7745 // Tell the user interleaving was avoided up-front, despite being explicitly
7746 // requested.
7747 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
7748 "interleaving should be avoided up front\n");
7749 IntDiagMsg = {"InterleavingAvoided",
7750 "Ignoring UserIC, because interleaving was avoided up front"};
7751 InterleaveLoop = false;
7752 } else if (IC == 1 && UserIC <= 1) {
7753 // Tell the user interleaving is not beneficial.
7754 if (LVP.hasAPlan()) {
7755 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
7756 IntDiagMsg = {
7757 "InterleavingNotBeneficial",
7758 "the cost-model indicates that interleaving is not beneficial"};
7759 if (UserIC == 1) {
7760 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
7761 IntDiagMsg.second +=
7762 " and is explicitly disabled or interleave count is set to 1";
7763 }
7764 } else {
7765 LLVM_DEBUG(dbgs() << "LV: Interleaving is not possible. Failed to create"
7766 << " any vplans\n");
7767 IntDiagMsg = {"InterleavingNotPossible", "interleaving is not possible"};
7768 }
7769 InterleaveLoop = false;
7770 } else if (IC > 1 && UserIC == 1) {
7771 // Tell the user interleaving is beneficial, but it explicitly disabled.
7772 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
7773 "disabled.\n");
7774 IntDiagMsg = {"InterleavingBeneficialButDisabled",
7775 "the cost-model indicates that interleaving is beneficial "
7776 "but is explicitly disabled or interleave count is set to 1"};
7777 InterleaveLoop = false;
7778 }
7779
7780 // If there is a histogram in the loop, do not just interleave without
7781 // vectorizing. The order of operations will be incorrect without the
7782 // histogram intrinsics, which are only used for recipes with VF > 1.
7783 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
7784 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
7785 << "to histogram operations.\n");
7786 IntDiagMsg = {
7787 "HistogramPreventsScalarInterleaving",
7788 "Unable to interleave without vectorization due to constraints on "
7789 "the order of histogram operations"};
7790 InterleaveLoop = false;
7791 }
7792
7793 // Override IC if user provided an interleave count.
7794 IC = UserIC > 0 ? UserIC : IC;
7795
7796 if (LVP.getCostModel().maskPartialAliasing()) {
7797 LLVM_DEBUG(
7798 dbgs()
7799 << "LV: Not interleaving due to partial aliasing vectorization.\n");
7800 IntDiagMsg = {
7801 "PartialAliasingVectorization",
7802 "Unable to interleave due to partial aliasing vectorization."};
7803 InterleaveLoop = false;
7804 IC = 1;
7805 }
7806
7807 // FIXME: Enable interleaving for EE-with-side-effects.
7808 if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
7809 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
7810 IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
7811 "Unable to interleave due to early exit with side effects."};
7812 InterleaveLoop = false;
7813 IC = 1;
7814 }
7815
7816 if (InterleaveLoop && LVL.hasConditionalInductions()) {
7817 LLVM_DEBUG(
7818 dbgs() << "LV: Not interleaving due to conditional inductions.\n");
7819 IntDiagMsg = {"ConditionalInductionsPreventInterleaving",
7820 "Unable to interleave due to conditional inductions."};
7821 InterleaveLoop = false;
7822 IC = 1;
7823 }
7824
7825 // Emit diagnostic messages, if any.
7826 if (!VectorizeLoop && !InterleaveLoop) {
7827 // Do not vectorize or interleaving the loop.
7828 ORE->emit([&]() {
7829 return OptimizationRemarkMissed(LV_NAME, VecDiagMsg.first,
7830 L->getStartLoc(), L->getHeader())
7831 << VecDiagMsg.second;
7832 });
7833 ORE->emit([&]() {
7834 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
7835 L->getStartLoc(), L->getHeader())
7836 << IntDiagMsg.second;
7837 });
7838 return false;
7839 }
7840
7841 if (!VectorizeLoop && InterleaveLoop) {
7842 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
7843 ORE->emit([&]() {
7844 return OptimizationRemarkAnalysis(LV_NAME, VecDiagMsg.first,
7845 L->getStartLoc(), L->getHeader())
7846 << VecDiagMsg.second;
7847 });
7848 } else if (VectorizeLoop && !InterleaveLoop) {
7849 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
7850 << ") in " << L->getLocStr() << '\n');
7851 ORE->emit([&]() {
7852 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
7853 L->getStartLoc(), L->getHeader())
7854 << IntDiagMsg.second;
7855 });
7856 } else if (VectorizeLoop && InterleaveLoop) {
7857 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
7858 << ") in " << L->getLocStr() << '\n');
7859 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
7860 }
7861
7862 // Report the vectorization decision.
7863 if (VF.Width.isScalar()) {
7864 using namespace ore;
7865 assert(IC > 1);
7866 ORE->emit([&]() {
7867 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
7868 L->getHeader())
7869 << "interleaved loop (interleaved count: "
7870 << NV("InterleaveCount", IC) << ")";
7871 });
7872 } else {
7873 // Report the vectorization decision.
7874 reportVectorization(ORE, L, VF.Width, IC);
7875 }
7876 if (ORE->allowExtraAnalysis(LV_NAME))
7878
7879 // If we decided that it is *legal* to interleave or vectorize the loop, then
7880 // do it.
7881
7882 // Whether a scalar epilogue may be created is decided by the epilogue
7883 // lowering policy.
7884 // TODO: Also move check to be based on VPlan.
7885 bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
7886
7887 // Destroy the cost model before executing any plan, so that code generation
7888 // cannot rely on cost-modeling decisions.
7889 LVP.clearCostModel();
7890
7891 VPlan &BestPlan = *BestPlanPtr;
7892 // Consider vectorizing the epilogue too if it's profitable.
7893 std::unique_ptr<VPlan> EpiPlan =
7894 LVP.selectBestEpiloguePlan(BestPlan, VF.Width, IC, ScalarEpilogueAllowed);
7895 bool HasBranchWeights =
7896 hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
7897 if (EpiPlan) {
7898 VPlan &BestEpiPlan = *EpiPlan;
7899 VPlan &BestMainPlan = BestPlan;
7900 ElementCount EpilogueVF = BestEpiPlan.getSingleVF();
7901
7902 // The first pass vectorizes the main loop and creates a scalar epilogue
7903 // to be vectorized by executing the plan (potentially with a different
7904 // factor) again shortly afterwards.
7905 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
7906 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
7908
7909 // Add minimum iteration check for the epilogue plan, followed by runtime
7910 // checks for the main plan.
7911 LVP.addMinimumIterationCheck(BestMainPlan, EpilogueVF, /*UF=*/1,
7913 BestEpiPlan.hasTailFolded());
7914 LVP.attachRuntimeChecks(BestMainPlan, Checks, VF.Width, IC,
7915 HasBranchWeights);
7917 VF.Width, IC, BestMainPlan.requiresScalarEpilogue(), L,
7918 HasBranchWeights ? MinItersBypassWeights : nullptr,
7919 L->getLoopPredecessor()->getTerminator()->getDebugLoc(),
7920 PSE);
7921
7922 LLVM_DEBUG({
7923 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7924 << "Main Loop VF:" << VF.Width << ", Main Loop UF:" << IC
7925 << ", Epilogue Loop VF:" << EpilogueVF << ", Epilogue Loop UF:1\n";
7926 });
7927 // Keep track of the iteration count check of the main vector loop.
7928 VPBasicBlock *MainBlocks[] = {cast<VPBasicBlock>(
7929 BestMainPlan.getVectorPreheader()->getSinglePredecessor())};
7930 auto ExpandedSCEVs = LVP.executePlan(
7931 VF.Width, IC, BestMainPlan, Checks.hasChecks(),
7933 MainBlocks);
7934 VPBasicBlock *MainCheck = MainBlocks[0];
7935 ++LoopsVectorized;
7937 dbgs() << "intermediate fn:\n" << *L->getHeader()->getParent() << "\n";
7938 });
7939
7940 BasicBlock *EntryBB =
7941 cast<VPIRBasicBlock>(BestMainPlan.getEntry())->getIRBasicBlock();
7942 EntryBB->setName("iter.check");
7944 ->getIRBasicBlock()
7945 ->setName("vec.epilog.iter.check");
7946
7947 // Second pass vectorizes the epilogue and adjusts the control flow
7948 // edges from the first pass.
7949 preparePlanForEpilogueVectorLoop(BestMainPlan, BestEpiPlan, ExpandedSCEVs,
7950 VF.Width, IC, EpilogueVF, Config,
7951 *PSE.getSE(), Markers, MainCheck);
7953 LLVM_DEBUG({
7954 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
7955 << "Epilogue Loop VF:" << EpilogueVF << ", Epilogue Loop UF:1\n";
7956 });
7957 LVP.executePlan(
7958 EpilogueVF, /*BestUF=*/1, BestEpiPlan, Checks.hasChecks(),
7961 dbgs() << "final fn:\n" << *L->getHeader()->getParent() << "\n";
7962 });
7963 ++LoopsEpilogueVectorized;
7964 } else {
7965 LVP.addMinimumIterationCheck(BestPlan, VF.Width, IC,
7966 VF.MinProfitableTripCount,
7967 /*TailFoldedEpilogue=*/false);
7968 LVP.attachRuntimeChecks(BestPlan, Checks, VF.Width, IC, HasBranchWeights);
7969
7970 if (!IsInnerLoop)
7971 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
7972 << "\"\n");
7973 LVP.executePlan(VF.Width, IC, BestPlan, Checks.hasChecks());
7974 ++LoopsVectorized;
7975 }
7976
7977#ifdef EXPENSIVE_CHECKS
7978 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
7979 "DT not preserved correctly");
7980#endif
7981
7982 return true;
7983}
7984
7986 CFGChanged = false;
7987
7988 // Don't attempt if
7989 // 1. the target claims to have no vector registers, and
7990 // 2. interleaving won't help ILP.
7991 //
7992 // The second condition is necessary because, even if the target has no
7993 // vector registers, loop vectorization may still enable scalar
7994 // interleaving.
7995 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
7996 (TTI->getMaxInterleaveFactor(ElementCount::getFixed(1), false) < 2 ||
7997 TTI->getMaxInterleaveFactor(ElementCount::getFixed(1), true) < 2))
7998 return LoopVectorizeResult(false, false);
7999
8000 bool Changed = false;
8001
8002 // The vectorizer requires loops to be in simplified form.
8003 // Since simplification may add new inner loops, it has to run before the
8004 // legality and profitability checks. This means running the loop vectorizer
8005 // will simplify all loops, regardless of whether anything end up being
8006 // vectorized.
8007 for (const auto &L : *LI)
8008 Changed |= CFGChanged |= simplifyLoop(L, DT, LI, SE, AC, nullptr,
8009 false /* PreserveLCSSA */, TTI);
8010
8011 // Build up a worklist of inner-loops to vectorize. This is necessary as
8012 // the act of vectorizing or partially unrolling a loop creates new loops
8013 // and can invalidate iterators across the loops.
8014 SmallVector<Loop *, 8> Worklist;
8015
8016 const VectorizeOptions &Opts = getVectorizeOptions(F);
8017 for (Loop *L : *LI)
8018 collectSupportedLoops(Opts, *L, LI, ORE, Worklist);
8019
8020 LoopsAnalyzed += Worklist.size();
8021
8022 // Now walk the identified inner loops.
8023 while (!Worklist.empty()) {
8024 Loop *L = Worklist.pop_back_val();
8025
8026 // For the inner loops we actually process, form LCSSA to simplify the
8027 // transform.
8028 Changed |= formLCSSARecursively(*L, *DT, LI, SE);
8029
8031
8032 if (Changed) {
8033 LAIs->clear();
8034
8035#ifndef NDEBUG
8036 if (VerifySCEV)
8037 SE->verify();
8038#endif
8039 }
8040 }
8041
8042 // Verify once per function rather than once per processed loop, which would
8043 // make the pass quadratic in the number of loops.
8044 assert((!Changed || DT->verify(DominatorTree::VerificationLevel::Fast)) &&
8045 "DT not preserved correctly");
8046 assert((!Changed || !verifyFunction(F, &dbgs())) &&
8047 "Invalid IR produced by LoopVectorize");
8048
8049 // Process each loop nest in the function.
8051}
8052
8055 LI = &AM.getResult<LoopAnalysis>(F);
8056 // There are no loops in the function. Return before computing other
8057 // expensive analyses.
8058 if (LI->empty())
8059 return PreservedAnalyses::all();
8068 AA = &AM.getResult<AAManager>(F);
8069
8070 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
8071 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
8072 // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
8073 // Both BlockFrequencyAnalysis and BranchProbabilityAnalysis depend on it, so
8074 // drop the stale result before either is (re-)computed.
8075 auto ClearStaleCycleInfo = [this, &AM, &F] {
8078 };
8079 GetBFI = [&AM, &F, ClearStaleCycleInfo]() -> BlockFrequencyInfo & {
8080 ClearStaleCycleInfo();
8082 };
8083 GetBPI = [&AM, &F, ClearStaleCycleInfo]() -> const BranchProbabilityInfo & {
8084 ClearStaleCycleInfo();
8086 };
8087 LoopVectorizeResult Result = runImpl(F);
8088 if (!Result.MadeAnyChange)
8089 return PreservedAnalyses::all();
8091
8092 if (isAssignmentTrackingEnabled(*F.getParent())) {
8093 for (auto &BB : F)
8095 }
8096
8097 PA.preserve<LoopAnalysis>();
8101
8102 if (Result.MadeCFGChange) {
8103 // Making CFG changes likely means a loop got vectorized. Indicate that
8104 // extra simplification passes should be run.
8105 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
8106 // be run if runtime checks have been added.
8109 } else {
8111 }
8112 return PA;
8113}
8114
8116 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
8117 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
8118 OS, MapClassName2PassName);
8119
8120 OS << '<';
8121 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
8122 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
8123 OS << '>';
8124}
static unsigned getIntrinsicID(const SDNode *N)
const TargetLowering & TLI
SDValue Pred
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
AMDGPU Lower Kernel Arguments
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static InstructionCost getCost(const AnalysisOptions &Opts, Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI)
Definition CostModel.cpp:33
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
This file defines the DenseMap class.
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static cl::opt< ElementCount, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
#define LV_NAME
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE, Loop *L)
Get the maximum trip count for L from the SCEV unsigned range, excluding zero from the range.
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static void legacyCSE(BasicBlock *BB)
FIXME: This legacy common-subexpression-elimination routine is scheduled for removal,...
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode)
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static EpilogueLowering getEpilogueTailLowering(const VectorizeOptions &Opts, const LoopVectorizationCostModel &MainCM, const Loop *L, OptimizationRemarkEmitter *ORE, LoopVectorizationLegality &LVL, const LoopVectorizeHints &Hints, const TargetTransformInfo *TTI)
Determine how to lower the epilogue for the vector epilogue loop.
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static std::optional< VPExecutionFrequency > getRecordedExecutionFrequency(const VPBasicBlock *VPBB)
Returns the frequency with which VPBB executes, as recorded on its recipes.
static void collectSupportedLoops(const VectorizeOptions &Opts, Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF, bool MaskRequired, const TargetLibraryInfo *TLI)
Returns true iff CI has a library vector variant usable at VF.
static constexpr uint32_t MinItersBypassWeights[]
static Function * getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF, bool MaskRequired, const TargetLibraryInfo *TLI)
Returns the vector library variant function of CI usable at VF, respecting MaskRequired,...
static bool isExplicitVecOuterLoop(const VectorizeOptions &Opts, Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static void preparePlanForEpilogueVectorLoop(VPlan &MainPlan, VPlan &Plan, const SCEV2ValueTy &ExpandedSCEVs, ElementCount MainLoopVF, unsigned MainLoopUF, ElementCount EpilogueVF, VFSelectionContext &Config, ScalarEvolution &SE, const MainPlanResumeMarkers &Markers, VPBasicBlock *MainCheck)
Prepare Plan for vectorizing the epilogue loop.
static bool hasForcedEpilogueVF()
static bool hasConditionalInductionPhi(VPlan &Plan)
Returns true if the VPlan contains a VPConditionalInductionPHIRecipe.
static void printOptimizedVPlan(VPlan &)
static bool isIndvarOverflowCheckKnownFalse(const VectorizeOptions &Opts, const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static const SCEV * getAddressAccessSCEV(Value *Ptr, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets the address access SCEV for Ptr, if it should be used for cost modeling according to isAddressSC...
static std::optional< ElementCount > getSmallBestKnownTC(const VectorizeOptions &Opts, PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true, bool CanExcludeZeroTrips=false, bool ComputeUpperBoundOnly=false)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static bool useMaskedInterleavedAccesses(const VectorizeOptions &Opts, const TargetTransformInfo &TTI)
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static bool hasFindLastReductionPhi(VPlan &Plan)
Returns true if the VPlan contains a VPReductionPHIRecipe with FindLast recurrence kind.
static EpilogueLowering getEpilogueLowering(const VectorizeOptions &Opts, LoopVectorizeHints &Hints, bool OptForSize, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop, LoopInfo *LI, LoopVectorizationCostModel &CM)
Cross-check the execution frequencies recorded in Plan against BlockFrequencyInfo for the blocks of O...
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, EpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static cl::opt< ElementCount > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(ElementCount::getFixed(1)), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops. Note: This allows all scalable VFs >= vscale x 1."))
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
This file contains the declarations for profiling metadata utility functions.
R600 Emit Clause Markers
SmallVector< MachineOperand, 4 > Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
SI Fold Operands
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
This file contains some templates that are useful if you are working with the STL at all.
#define OP(OPC)
Definition Instruction.h:46
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
static const CodeGenOptions & getOpts(const TargetMachine &TM)
This pass exposes codegen information to IR-level passes.
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file provides utility VPlan to VPlan transformations.
#define RUN_VPLAN_PASS(PASS,...)
#define RUN_VPLAN_PASS_NO_VERIFY(PASS,...)
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
A manager for alias analyses.
static constexpr roundingMode rmTowardZero
Definition APFloat.h:365
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1532
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition APInt.h:1115
void clearAnalysis(IRUnitT &IR)
Directly clear a cached analysis for an IR unit.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
ArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
Definition ArrayRef.h:225
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:509
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
LLVM_ABI BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
Analysis pass which computes BranchProbabilityInfo.
Analysis providing branch probability information.
static LLVM_ABI BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
static uint32_t getDenominator()
uint32_t getNumerator() const
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ ICMP_NE
not equal
Definition InstrTypes.h:762
Conditional Branch instruction.
BasicBlock * getSuccessor(unsigned i) const
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
This class represents a range of values.
LLVM_ABI APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
Analysis pass which computes a CycleInfo.
A debug info location.
Definition DebugLoc.h:126
static DebugLoc getTemporary()
Definition DebugLoc.h:152
static DebugLoc getUnknown()
Definition DebugLoc.h:153
An analysis that produces DemandedBits for a function.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:757
void insert_range(Range &&R)
Inserts a range of 'std::pair<KeyT, ValueT>' values into the map.
Definition DenseMap.h:894
ValueT lookup_or(const_arg_type_t< KeyT > Val, U &&Default) const
Definition DenseMap.h:803
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:793
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:856
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
constexpr bool isVector() const
One or more elements.
Definition TypeSize.h:321
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:309
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:306
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:312
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:317
constexpr bool isPowerOf2() const
Definition TypeSize.h:324
Tagged union holding either a T or a Error.
Definition Error.h:485
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:23
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
A struct for saving information about induction variables.
@ IK_PtrInduction
Pointer induction var. Step = C.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
bool isCast() const
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
iterator_range< user_iterator > users()
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
Definition Type.cpp:357
The group of interleaved loads/stores sharing the same stride and close to each other.
auto members() const
Return an iterator range over the non-null members of this group, in index order.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool hasGroups() const
Returns true if we have any interleave groups.
iterator_range< SmallPtrSetIterator< llvm::InterleaveGroup< Instruction > * > > getInterleaveGroups()
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
LLVM_ABI void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
const SymbolicStrideMap & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:598
bool contains(const LoopT *L) const
Return true if the specified loop is contained within this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
unsigned getNumBlocks() const
Get the number of blocks in this loop in constant time.
BlockT * getHeader() const
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
LLVM_ABI void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
BlockFrequencyInfo * BFI
The BlockFrequencyInfo returned from GetBFI.
BlockFrequencyInfo & getBFI()
Returns the BlockFrequencyInfo for the function if cached, otherwise fetches it via GetBFI.
bool isForcedScalar(Instruction *I, ElementCount VF) const
Returns true if I has been forced to be scalarized at VF.
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
static constexpr StringLiteral getInstWideningStr(InstWidening W)
bool isMaskRequired(Instruction *I) const
Wrapper function for LoopVectorizationLegality::isMaskRequired, that passes the Instruction I and if ...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const TargetTransformInfo & TTI
Vector target information.
LoopVectorizationLegality * Legal
Vectorization legality.
uint64_t getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB)
A helper function that returns how much we should divide the cost of a predicated block by.
std::optional< InstWidening > memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
If I is a memory instruction with a consecutive pointer that can be widened, returns the widening kin...
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool maskPartialAliasing() const
Returns true if all loop blocks should have partial aliases masked.
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const
Returns true if the target machine supports gather or scatter for I's data type and alignment.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
Loop * TheLoop
The loop that we evaluate.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, std::function< BlockFrequencyInfo &()> GetBFI, const Function *F, InterleavedAccessInfo &IAI, VFSelectionContext &Config)
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool usePredicatedReductionSelect(RecurKind RecurrenceKind, bool HasUsesOutsideReductionChain) const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const
Returns true if an artificially high cost for emulated masked memrefs should be used.
bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const
Returns true if the target machine supports masked loads or stores for I's data type and alignment.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
bool isEpilogueAllowed() const
Returns true if an epilogue is allowed (e.g., not prevented by optsize or a loop hint annotation).
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
@ CM_InvalidatedDecision
A widening decision that has been invalidated after replacing the corresponding recipe during VPlan t...
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF)
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool isScalarWithPredication(Instruction *I, ElementCount VF)
Returns true if I is an instruction which requires predication and for which our chosen predication s...
std::function< BlockFrequencyInfo &()> GetBFI
A function to lazily fetch BlockFrequencyInfo.
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost MaskedCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
TailFoldingStyle getTailFoldingStyle() const
Returns the TailFoldingStyle that is best for the current loop.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
RecurrenceSet & getFixedOrderRecurrences()
Return the fixed-order recurrences found in the loop.
LLVM_ABI bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool hasUncountableExitWithSideEffects() const
Returns true if this is an early exit loop with state-changing or potentially-faulting operations and...
LLVM_ABI bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
LLVM_ABI bool isFixedOrderRecurrence(const PHINode *Phi) const
Returns True if Phi is a fixed-order recurrence in this loop.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const HistogramInfo * getHistogramInfo(Instruction *I) const
Returns a HistogramInfo* for the given instruction if it was determined to be part of a load -> updat...
const ReductionList & getReductionVars() const
Returns the reduction variables found in the loop.
bool hasUncountableEarlyExit() const
Returns true if the loop has uncountable early exits, i.e.
LLVM_ABI bool isUniformMemOp(Instruction &I, std::optional< ElementCount > VF) const
A uniform memory op is a load or store which accesses the same memory location on all VF lanes,...
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
LLVM_ABI bool isUniform(Value *V, std::optional< ElementCount > VF) const
Returns true if value V is uniform across VF lanes, when VF is provided, and otherwise if V is invari...
Planner drives the vectorization process after having passed Legality checks.
EpilogueVectorizationKind
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
@ MainLoop
Vectorizing the main loop of epilogue vectorization.
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool TailFoldedEpilogue) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
void clearCostModel()
Destroy the cost model.
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
Definition VPlan.cpp:1655
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll, bool UnrollVectorizedLoop)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
Definition VPlan.cpp:1706
bool hasAPlan() const
Returns true if there is at least one VPlan.
LoopVectorizationCostModel & getCostModel()
Return the cost model. Must not be called after clearCostModel().
LoopVectorizationPlanner(Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI, const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal, std::unique_ptr< LoopVectorizationCostModel > CM, VFSelectionContext &Config, PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE, std::function< const BranchProbabilityInfo &()> GetBPI)
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks, ElementCount VF, unsigned UF, bool HasBranchWeights) const
Attach the runtime checks of RTChecks to Plan.
bool hasVectorPlan() const
Return true if there is at least one VPlan for a vector VF.
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
Definition VPlan.cpp:1641
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, bool HasRuntimeChecks, EpilogueVectorizationKind EpilogueVecKind=EpilogueVectorizationKind::None, MutableArrayRef< VPBasicBlock * > TrackedBlocks={})
void printPlans(raw_ostream &O)
Definition VPlan.cpp:1810
std::unique_ptr< VPlan > selectBestEpiloguePlan(VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC, bool ScalarEpilogueAllowed)
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
std::pair< VectorizationFactor, VPlan * > computeBestVF()
Compute and return the most profitable vectorization factor and the corresponding best VPlan.
This holds vectorization requirements that must be verified late in the process.
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
LLVM_ABI void emitRemarkWithHints() const
Dumps all the hint information.
LLVM_ABI bool allowVectorization(Loop *L, bool VectorizeOnlyWhenForced) const
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
Metadata node.
Definition Metadata.h:1087
bool empty() const
Definition MapVector.h:80
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition MapVector.h:127
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition Module.cpp:254
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
Diagnostic information for optimization analysis remarks related to pointer aliasing.
Diagnostic information for optimization analysis remarks related to floating-point non-commutativity.
Diagnostic information for optimization analysis remarks.
The optimization diagnostic interface.
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:152
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:133
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Type * getRecurrenceType() const
Returns the type of the recurrence.
static bool isFindLastRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static LLVM_ABI bool isSubRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is for a sub operation.
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
bool Need
This flag indicates if we need to add the runtime check.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
LLVM_ABI void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEVFlags Flags=SCEV::FlagNone)
LLVM_ABI const SCEV * getURemExpr(SCEVUse LHS, SCEVUse RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI SCEVUse getAddExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI SCEVUse getMulExpr(SmallVectorImpl< SCEVUse > &Ops, SCEVFlagsPair Flags={}, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
void insert_range(Range &&R)
Definition SetVector.h:182
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
Definition SetVector.h:268
bool contains(const_arg_type key) const
Check if the SetVector contains the given key.
Definition SetVector.h:258
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:346
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition StringRef.h:888
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)
Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
@ ActiveLaneMask
The operation is masked with a contiguous active lane mask.
llvm::VectorInstrContext VectorInstrContext
@ TCC_Free
Expected to fold away in lowering.
LLVM_ABI InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_Reverse
Reverse the order of the vector.
CastContextHint
Represents a hint about the context in which a cast is used.
@ Reversed
The cast is used with a reversed load/store.
@ Masked
The cast is used with a masked load/store.
@ Normal
The cast is used with a normal load/store.
@ Interleave
The cast is used with an interleaved load/store.
@ GatherScatter
The cast is used with a gather/scatter.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
Definition TypeSwitch.h:89
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
Definition TypeSwitch.h:98
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:283
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:267
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.h:354
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:370
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:291
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:141
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
iterator_range< op_iterator > op_range
Definition User.h:256
Value * getOperand(unsigned i) const
Definition User.h:207
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
Definition VectorUtils.h:76
Holds state needed to make cost decisions before computing costs per-VF, including the maximum VFs.
const TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
const LoopVectorizeHints & getHints() const
bool isEpilogueVectorizationProfitable(ElementCount VF, unsigned IC) const
Returns true if epilogue vectorization is considered profitable for a main loop with vectorization fa...
std::optional< unsigned > getVScaleForTuning() const
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:4540
RecipeListTy::iterator iterator
Instruction iterators...
Definition VPlan.h:4571
iterator end()
Definition VPlan.h:4581
iterator begin()
Recipe iterator methods.
Definition VPlan.h:4579
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition VPlan.h:4634
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override
Return the cost of this VPBasicBlock.
Definition VPlan.cpp:734
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition VPlan.cpp:224
const VPRecipeBase & front() const
Definition VPlan.h:4591
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
Definition VPlan.cpp:604
bool empty() const
Definition VPlan.h:4590
const VPBasicBlock * getExitingBasicBlock() const
Definition VPlan.cpp:194
void setName(const Twine &newName)
Definition VPlan.h:186
size_t getNumPredecessors() const
Definition VPlan.h:244
const VPBlocksTy & getPredecessors() const
Definition VPlan.h:228
VPlan * getPlan()
Definition VPlan.h:197
VPBlockBase * getSinglePredecessor() const
Definition VPlan.h:239
const VPBasicBlock * getEntryBasicBlock() const
Definition VPlan.cpp:179
VPBlockBase * getSingleSuccessor() const
Definition VPlan.h:233
static auto blocksAs(T &&Range)
Return an iterator range over Range with each block cast to BlockTy.
Definition VPlanUtils.h:464
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
Definition VPlanUtils.h:427
static auto blocksOnly(T &&Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition VPlanUtils.h:457
static std::pair< VPBasicBlock *, VPBasicBlock * > getPlainCFGHeaderAndLatch(const VPlan &Plan)
Returns the header and latch of the outermost loop of Plan in plain CFG form (before regions are form...
A recipe for generating conditional branches on the bits of a mask.
Definition VPlan.h:3595
static VPBuilderBase getToInsertAfter(VPRecipeBase *R)
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", std::optional< VPIRFlags > Flags=std::nullopt, Type *ResultTy=nullptr)
Create a phi with IncomingValues, using the default flags for the result type, unless Flags is set.
T * insert(T *R)
Insert R at the current insertion point. Returns R unchanged.
VPInstruction * createAdd(VPValue *LHS, VPValue *RHS, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", VPRecipeWithIRFlags::WrapFlagsTy WrapFlags={false, false})
VPInstruction * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", std::optional< VPIRFlags > Flags=std::nullopt)
Create a select of TrueVal and FalseVal based on Cond, using the default flags for the result type,...
static VPSingleDefRecipe * createSingleScalarOp(unsigned Opcode, ArrayRef< VPValue * > Operands, VPValue *Mask, const VPIRFlags &Flags, const VPIRMetadata &Metadata, DebugLoc DL, Type *ResultTy, Instruction *UV)
void setInsertPoint(const VPInsertPoint &IP)
Set the current insert point.
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="", Type *ResultTy=nullptr, const VPIRAttributes &Attrs={})
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition VPlanValue.h:572
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition VPlanValue.h:545
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
Recipe to expand a SCEV expression.
Definition VPlan.h:4134
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition VPlan.h:2480
virtual VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition VPlan.h:2523
void setBackedgeValue(VPValue *V)
Update the incoming value from the loop backedge.
Definition VPlan.h:2526
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition VPlan.h:2512
A recipe representing a sequence of load -> update -> store as part of a histogram operation.
Definition VPlan.h:2195
A special type of VPBasicBlock that wraps an existing IR basic block.
Definition VPlan.h:4699
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlagsOrNone() const
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:1322
iterator_range< operand_iterator > operandsWithoutMask()
Returns an iterator range over the operands excluding the mask operand if present.
Definition VPlan.h:1576
VPInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1512
@ ResumeForEpilogue
Explicit user for values in the main VPlan, used by the epilogue vector loop.
Definition VPlan.h:1431
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
Definition VPlan.h:1423
@ ComputeReductionResult
Reduce the operands to the final reduction result using the operation specified via the operation's V...
Definition VPlan.h:1381
unsigned getOpcode() const
Definition VPlan.h:1525
VPValue * getMask() const
Returns the mask for the VPInstruction.
Definition VPlan.h:1572
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition VPlan.h:3228
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition VPlan.h:402
RecipeListTy::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition VPlan.h:547
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
VPRecipeBase * tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for a non-phi recipe R if one can be created within the given VF R...
VPHistogramRecipe * widenIfHistogram(VPInstruction *VPI)
If VPI represents a histogram operation (as determined by LoopVectorizationLegality) make that safe f...
std::optional< CompressedPtrInfo > getCompressedPtrInfo(VPInstruction *VPI)
Returns the CompressedPtrInfo for VPI if it is a compressed load or store (as determined by LoopVecto...
bool prefersVectorizedAddressing() const
Returns true if the target prefers vectorized addressing.
VPRecipeBase * tryToWidenMemory(VPInstruction *VPI, VFRange &Range)
Check if the load or store instruction VPI should widened for Range.Start and potentially masked.
bool replaceWithFinalIfReductionStore(VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder)
If VPI is a store of a reduction into an invariant address, delete it.
VPSingleDefRecipe * handleReplication(VPInstruction *VPI, VFRange &Range)
Build a replicating or single-scalar recipe for VPI.
VPWidenMemIntrinsicRecipe * widenCompressedLoadOrStore(VPInstruction *VPI)
Build a llvm.masked.expandload or llvm.masked.compressstore intrinsic for VPI.
bool isPredicatedInst(Instruction *I) const
Returns true if I needs to be predicated (i.e.
Type * getScalarType() const
Returns the scalar type of this VPRecipeValue.
Definition VPlanValue.h:349
A recipe for handling reduction phis.
Definition VPlan.h:2883
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition VPlan.h:2955
bool hasUsesOutsideReductionChain() const
Returns true, if the phi is part of a multi-use reduction.
Definition VPlan.h:2964
bool isInLoop() const
Returns true if the phi is part of an in-loop reduction.
Definition VPlan.h:2958
VPReductionPHIRecipe * cloneWithOperands(VPValue *Start, VPValue *BackedgeValue)
Definition VPlan.h:2919
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
Definition VPlan.h:2952
A recipe to represent inloop, ordered or partial reduction operations.
Definition VPlan.h:3321
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:4771
const VPBlockBase * getEntry() const
Definition VPlan.h:4815
void clearCanonicalIVNUW(VPInstruction *Increment)
Unsets NUW for the canonical IV increment Increment, for loop regions.
Definition VPlan.h:4938
VPRegionValue * getCanonicalIV()
Return the canonical induction variable of the region, null for replicating regions.
Definition VPlan.h:4891
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition VPlan.h:3488
VPSingleDefRecipe is a base class for recipes that model a sequence of one or more output IR that def...
Definition VPlan.h:605
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition VPlan.h:668
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition VPlanValue.h:396
operand_range operands()
Definition VPlanValue.h:472
void setOperand(unsigned I, VPValue *New)
Definition VPlanValue.h:445
VPValue * getOperand(unsigned N) const
Definition VPlanValue.h:437
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
Definition VPlanValue.h:50
Type * getScalarType() const
Returns the scalar type of this VPValue, dispatching based on the concrete subclass.
Definition VPlan.cpp:138
Value * getLiveInIRValue() const
Return the underlying IR value for a VPIRValue.
Definition VPlan.cpp:132
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition VPlan.cpp:119
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
Definition VPlanValue.h:75
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser *U)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
Definition VPlan.cpp:1476
void replaceAllUsesWith(VPValue *New)
Definition VPlan.cpp:1470
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition VPlan.h:1896
A recipe for handling GEP instructions.
Definition VPlan.h:2254
A recipe for widening vector memory intrinsics.
Definition VPlan.h:2073
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Definition VPlan.h:1829
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4958
bool hasVF(ElementCount VF) const
Definition VPlan.h:5181
ElementCount getSingleVF() const
Returns the single VF of the plan, asserting that the plan has exactly one VF.
Definition VPlan.h:5194
VPBasicBlock * getEntry()
Definition VPlan.h:5050
VPValue * getTripCount() const
The trip count of the original loop.
Definition VPlan.h:5122
VPSymbolicValue & getVFxUF()
Returns VF * UF of the vector loop region.
Definition VPlan.h:5153
bool hasUF(unsigned UF) const
Definition VPlan.h:5206
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition VPlan.h:5116
VPIRValue * getOrAddLiveIn(Value *V)
Gets the live-in VPIRValue for V or adds a new live-in (if none exists yet) for V.
Definition VPlan.h:5231
VPIRValue * getZero(Type *Ty)
Return a VPIRValue wrapping the null value of type Ty.
Definition VPlan.h:5257
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition VPlan.cpp:1060
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
Definition VPlan.h:5365
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
Definition VPlan.cpp:1042
LLVM_ABI_FOR_TEST bool isOuterLoop() const
Returns true if this VPlan is for an outer loop, i.e., its vector loop region contains a nested loop ...
Definition VPlan.cpp:1085
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
Definition VPlan.h:5092
VPBasicBlock * getVectorPreheader() const
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition VPlan.h:5055
bool requiresScalarEpilogue() const
Returns true if the plan requires a scalar epilogue after the vector loop.
Definition VPlan.h:5078
VPSymbolicValue & getUF()
Returns the UF of the vector loop region.
Definition VPlan.h:5150
bool hasScalarVFOnly() const
Definition VPlan.h:5199
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
Definition VPlan.h:5106
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition VPlan.cpp:937
bool hasTailFolded() const
Returns true if the vector loop region is tail-folded.
Definition VPlan.h:5071
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition VPlan.h:5112
LLVM_ABI_FOR_TEST VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
Definition VPlan.cpp:1220
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:389
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:231
constexpr bool isNonZero() const
Definition TypeSize.h:156
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:217
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr bool isZero() const
Definition TypeSize.h:154
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:224
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
CallInst * Call
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, const Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, const Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, ElementCount VFWidth, unsigned IC)
Report successful vectorization of the loop.
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
match_unless< Pattern > m_Unless(const Pattern &P)
Match if the inner matcher does NOT match.
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_Value()
Match an arbitrary value and ignore it.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
match_bind< const SCEVMulExpr > m_scev_Mul(const SCEVMulExpr *&V)
bool match(const SCEV *S, const Pattern &P)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t, SCEV::FlagNone, true > m_scev_c_Mul(const Op0_t &Op0, const Op1_t &Op1)
RecipeTy * findSingleUser(VPValue *V, const MatchT &P)
Return the sole user of V matching pattern P, casting it to RecipeTy, or nullptr.
bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start)
Match FindIV result pattern: select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),...
VPInstruction_match< VPInstruction::ExtractLastLane, Op0_t > m_ExtractLastLane(const Op0_t &Op0)
VPInstruction_match< VPInstruction::BranchOnCount > m_BranchOnCount()
auto m_VPValue()
Match an arbitrary VPValue and ignore it.
VPInstruction_match< VPInstruction::ExtractLastPart, Op0_t > m_ExtractLastPart(const Op0_t &Op0)
bool match(Val *V, const Pattern &P)
VPInstruction_match< VPInstruction::ExtractLane, Op0_t, Op1_t > m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1)
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
BranchProbability getExecutionProbability(BlockFrequency Freq)
Returns Freq as a BranchProbability, relative to the full mass.
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
bool isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE, const Loop *L)
Returns true if Addr is an address SCEV that can be passed to TTI::getAddressComputationCost,...
VPInstruction * findCanonicalIVIncrement(VPlan &Plan)
Find the canonical IV increment of Plan's vector loop region.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used by all users.
VPValue * findIncomingAliasMask(const VPlan &Plan)
Finds the incoming alias-mask within the vector preheader.
Type * getLoadStoreValueType(VPSingleDefRecipe *R)
Get the value type of a VPInstruction or VPReplicateRecipe load or store.
Definition VPlanUtils.h:252
bool doesGeneratePerAllLanes(const VPRecipeBase *R)
Returns true if R produces scalar values for all VF lanes.
VPRecipeBase * findRecipe(VPValue *Start, PredT Pred)
Search Start's users for a recipe satisfying Pred, looking through recipes with definitions.
Definition VPlanUtils.h:154
LLVM_ABI_FOR_TEST const SCEV * getSCEVExprForVPValue(const VPValue *V, PredicatedScalarEvolution &PSE, const Loop *L=nullptr)
Return the SCEV expression for V.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
SmallVector< VPBasicBlock * > vp_rpo_plain_cfg_loop_body(VPBasicBlock *Header)
Returns the VPBasicBlocks forming the loop body of a plain (pre-region) VPlan in reverse post-order s...
Definition VPlanCFG.h:262
@ Offset
Definition DWP.cpp:577
constexpr auto not_equal_to(T &&Arg)
Functor variant of std::not_equal_to that can be used as a UnaryPredicate in functional algorithms li...
Definition STLExtras.h:2196
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
Definition STLExtras.h:856
InstructionCost Cost
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
VPBuilderBase<> VPBuilder
Definition VPlan.h:66
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
Definition LCSSA.cpp:469
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2224
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Definition VPlanCFG.h:250
LLVM_ABI bool VerifySCEV
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
Definition VPlanCFG.h:285
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
Definition STLExtras.h:366
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
auto make_isa_range(RangeT &&Range)
Return a range over Range containing only elements for which isa<T> holds, casting each of them to T.
Definition STLExtras.h:567
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA, const TargetTransformInfo *TTI=nullptr)
Simplify each loop in a loop nest recursively.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
Definition CFG.h:154
std::optional< uint64_t > getMaxRuntimeElementCount(ElementCount EC, const Function &F)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
UncountableExitStyle
Different methods of handling early exits.
Definition VPlan.h:81
@ ReadOnly
No side effects to worry about, so we can process any uncountable exits in the loop and branch either...
Definition VPlan.h:85
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition Local.cpp:415
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
Definition STLExtras.h:1853
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:389
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
TargetTransformInfo TTI
@ CM_EpilogueNotAllowedLowTripLoop
@ CM_EpilogueNotNeededFoldTail
@ CM_EpilogueNotAllowedFoldTail
@ CM_EpilogueNotAllowedOptSize
@ CM_EpilogueAllowed
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
void replace(R &&Range, const T &OldValue, const T &NewValue)
Provide wrappers to std::replace which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1926
RecurKind
These are the kinds of recurrences that we support.
@ Sub
Subtraction of integers.
@ Add
Sum of integers.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
constexpr T AbsoluteDifference(U X, V Y)
Subtract two unsigned integers, X and Y, of type T and return the absolute value of the result.
Definition MathExtras.h:600
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool proveOuterLoopMemorySafety(VPlan &Plan, PredicatedScalarEvolution &PSE, AAResults &AA, const VPDominatorTree &VPDT, const Loop *OuterLoop)
Returns true if the memory accesses in outer loop plain CFG Plan can be vectorized.
auto sum_of(R &&Range, E Init=E{0})
Returns the sum of all values in Range with Init initial value.
Definition STLExtras.h:1733
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1933
const VectorizeOptions & getVectorizeOptions(const LLVMContext &C)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
constexpr bool valueOr(BoolOrDefault X, bool Default)
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
bool pred_empty(const BasicBlock *BB)
Definition CFG.h:107
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:307
LLVM_ABI std::optional< int64_t > getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy, Value *Ptr, PredicatedScalarEvolution &PSE)
If AR is an affine AddRec for Lp with a constant step, return the step in units of AccessTy's allocat...
@ Increment
Incrementally increasing token ID.
Definition AllocToken.h:26
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
Definition DWP.h:31
@ Disabled
Don't do any conversion of .debug_str_offsets tables.
Definition DWP.h:30
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
Definition VPlan.h:76
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, ElementCount VF, unsigned IC)
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:866
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan)
Verify invariants for general VPlans.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:287
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
ResumeForEpilogue markers in the main plan, used by the epilogue plan.
SmallVector< VPInstruction * > ResumeValues
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
HistogramUpdateKind UpdateKind
The update operation applied to the histogram buckets.
TargetLibraryInfo * TLI
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
ProfileSummaryInfo * PSI
LoopAccessInfoManager * LAIs
std::function< const BranchProbabilityInfo &()> GetBPI
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
ScalarEvolution * SE
AssumptionCache * AC
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
std::function< BlockFrequencyInfo &()> GetBFI
TargetTransformInfo * TTI
Storage for information about made changes.
A marker analysis to determine if extra passes should be run after loop vectorization.
static LLVM_ABI AnalysisKey Key
Parameters that control the generic loop unrolling transformation.
bool UnrollVectorizedLoop
Disable runtime unrolling by default for vectorized loops.
Holds the VFShape for a specific scalar to vector function mapping.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
ElementCount End
Struct to hold various analysis needed for cost computations.
LLVMContext & LLVMCtx
const VFSelectionContext & Config
LoopVectorizationCostModel & CM
VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan, LoopVectorizationCostModel &CM, VFSelectionContext &Config, bool ReusePrintingSlotTracker=false)
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
bool isMaskRequired(Instruction *I) const
Forwards to LoopVectorizationCostModel::isMaskRequired.
void invalidateWideningDecision(Instruction *I, ElementCount VF)
Mark the widening decision for I at VF as invalidated since a VPlan transform replaced the original r...
PredicatedScalarEvolution & PSE
bool willBeScalarized(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalarized at VF.
static bool executesAtMostOnce(const VPlan &Plan, ElementCount VF)
Returns true if the vector loop body of Plan is known to execute at most once at VF,...
TargetTransformInfo::TargetCostKind CostKind
const TargetLibraryInfo & TLI
const TargetTransformInfo & TTI
SmallPtrSet< Instruction *, 8 > SkipCostComputation
const VectorizeOptions & Opts
A pure-virtual common base class for recipes defining a single VPValue and using IR flags.
Definition VPlan.h:1089
A struct that represents some properties of the register usage of a loop.
InstructionCost spillCost(const TargetTransformInfo &TTI, TargetTransformInfo::TargetCostKind CostKind, unsigned OverrideMaxNumRegs=0) const
Calculate the estimated cost of any spills due to using more registers than the number available for ...
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
Definition VPlan.h:3972
A recipe for widening load operations, using the address to load from and an optional mask.
Definition VPlan.h:3917
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
Definition VPlan.h:4079
A recipe for widening store operations, using the stored value, the address to store to and an option...
Definition VPlan.h:4025
static void simplifyLiveInsWithSCEV(VPlan &Plan, PredicatedScalarEvolution &PSE)
Check Plan's live-ins and replace them with constants, if they can be simplified via SCEV.
static void addMinimumVectorEpilogueIterationCheck(VPlan &Plan, VPValue *MainVectorTripCount, bool RequiresScalarEpilogue, ElementCount EpilogueVF, unsigned MainLoopStep, unsigned EpilogueLoopStep, ScalarEvolution &SE)
Add a check to Plan to see if the epilogue vector loop should be executed.
static bool makeCallWideningDecisions(VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder, VPCostContext &CostCtx)
Convert call VPInstructions in Plan into widened call, vector intrinsic or replicate recipes based on...
static void expandSCEVsToVPInstructions(VPlan &Plan, ScalarEvolution &SE)
Expand VPExpandSCEVRecipes in Plan's entry block to VPInstructions.
static void materializeBroadcasts(VPlan &Plan)
Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as v...
static void createInterleaveGroups(VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, const bool &EpilogueAllowed)
static LLVM_ABI_FOR_TEST bool handleUncountableEarlyExits(VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style)
Update Plan to account for uncountable early exits by introducing appropriate branching logic in the ...
static bool simplifyKnownEVL(VPlan &Plan, ElementCount VF, PredicatedScalarEvolution &PSE)
Try to simplify VPInstruction::ExplicitVectorLength recipes when the AVL is known to be <= VF,...
static void introduceMasksAndLinearize(VPlan &Plan)
Predicate and linearize the control-flow in the only loop region of Plan.
static void materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)
Materialize UF, VF and VFxUF to be computed explicitly using VPInstructions.
static void foldTailByMasking(VPlan &Plan)
Adapts the vector loop region for tail folding by introducing a header mask and conditionally executi...
static LLVM_ABI_FOR_TEST bool tryToConvertVPInstructionsToVPRecipes(VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE, Loop *OuterLoop)
Replaces the VPInstructions in Plan with corresponding widen recipes.
static bool handleMultiUseReductions(VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop)
Try to legalize reductions with multiple in-loop uses.
static void convertToVariableLengthStep(VPlan &Plan)
Transform loops with variable-length stepping after region dissolution.
static void materializeHeaderMask(VPlan &Plan, bool UseActiveLaneMask, bool UseActiveLaneMaskForControlFlow)
Materialize the abstract header mask of the loop region into concrete recipes: an active-lane-mask if...
static void recordExecutionFrequencies(VPlan &Plan)
Add execution frequencies to each recipe in the loop body of Plan.
static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)
Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.
static std::unique_ptr< VPlan > narrowInterleaveGroups(VPlan &Plan, const TargetTransformInfo &TTI)
Try to find a single VF among Plan's VFs for which all interleave groups (with known minimum VF eleme...
static bool handleFindLastReductions(VPlan &Plan)
Check if Plan contains any FindLast reductions.
static void createInLoopReductionRecipes(VPlan &Plan, ElementCount MinVF)
Create VPReductionRecipes for in-loop reductions.
static void materializeAliasMaskCheckBlock(VPlan &Plan, ArrayRef< PointerDiffInfo > DiffChecks, ScalarEvolution &SE, bool HasBranchWeights)
Materializes the alias mask within a check block before the loop.
static LLVM_ABI_FOR_TEST bool createHeaderPhiRecipes(VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &OrigLoop, OptimizationRemarkEmitter *ORE, const VPDominatorTree &VPDT, const MapVector< PHINode *, InductionDescriptor > &Inductions, const MapVector< PHINode *, RecurrenceDescriptor > &Reductions, const MapVector< PHINode *, ConditionalInductionDescriptor > &ConditionalInductions, const SmallPtrSetImpl< const PHINode * > &FixedOrderRecurrences, const SmallPtrSetImpl< PHINode * > &InLoopReductions, bool AllowReordering)
Replace VPPhi recipes in Plan's header with corresponding VPHeaderPHIRecipe subclasses for inductions...
static void unrollByUF(VPlan &Plan, unsigned UF)
Explicitly unroll Plan by UF.
static DenseMap< const SCEV *, Value * > expandSCEVs(VPlan &Plan, ScalarEvolution &SE)
Expand remaining VPExpandSCEVRecipes in Plan's entry block using SCEVExpander.
static void convertToConcreteRecipes(VPlan &Plan)
Lower abstract recipes to concrete ones, that can be codegen'd.
static LLVM_ABI_FOR_TEST void createLoopRegions(VPlan &Plan, DebugLoc DL)
Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG...
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range, VPRecipeBuilder &RecipeBuilder, VPCostContext &CostCtx)
Convert load/store VPInstructions in Plan into widened or replicate recipes.
static VPIRBasicBlock * modelGeneratedMainLoopBlocks(VPlan &EpiPlan, VPlan &MainPlan, VPBasicBlock *MainCheck)
Model the blocks the executed MainPlan generated for the main vector loop in EpiPlan during epilogue ...
static LLVM_ABI_FOR_TEST void addMiddleCheck(VPlan &Plan)
If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle bloc...
static void expandBranchOnTwoConds(VPlan &Plan)
Expand BranchOnTwoConds instructions into explicit CFG with BranchOnCond instructions.
static void materializeVectorTripCount(VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue, VPValue *Step, std::optional< uint64_t > MaxRuntimeStep=std::nullopt)
Materialize vector trip count computations to a set of VPInstructions.
static void hoistPredicatedLoads(VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L)
Hoist predicated loads from the same address to the loop entry block, if they are guaranteed to execu...
static void expandMaskBeforeFirst(VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
Expand mask.beforefirst into an active-lane-mask of its first active lane for the VFs in Range where ...
static void attachAliasMaskToHeaderMask(VPlan &Plan)
Attaches the alias-mask to the existing header-mask.
static void optimizeFindIVReductions(VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &L)
Optimize FindLast reductions selecting IVs (or expressions of IVs) by converting them to FindIV reduc...
static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
This function converts initial recipes to the abstract recipes and clamps Range based on cost model f...
static void materializeConstantVectorTripCount(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
static void makeScalarizationDecisions(VPlan &Plan, VFRange &Range)
Make VPlan-based scalarization decision prior to delegating to the ones made by the legacy CM.
static void replaceWideCanonicalIVWithWideIV(VPlan &Plan, ScalarEvolution &SE, const TargetTransformInfo &TTI, TargetTransformInfo::TargetCostKind CostKind, ElementCount VF, unsigned UF)
Replace a VPWidenCanonicalIVRecipe if it is present in Plan, with a VPWidenIntOrFpInductionRecipe,...
static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan > buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, PredicatedScalarEvolution &PSE, LoopVersioning *LVer=nullptr, function_ref< const BranchProbabilityInfo &()> GetBPI=nullptr)
Create a base VPlan0, serving as the common starting point for all later candidates.
static void optimizeInductionLiveOutUsers(VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L)
If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them p...
static void addExplicitVectorLength(VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
Add a VPCurrentIterationPHIRecipe and related recipes to Plan and replaces all uses of the canonical ...
static void adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan, VFRange &Range)
Adjust first-order recurrence users in the middle block: create penultimate element extracts for LCSS...
static void optimizeEVLMasks(VPlan &Plan)
Optimize recipes which use an EVL-based header mask to VP intrinsics, for example:
static bool handleMaxMinNumReductions(VPlan &Plan)
Check if Plan contains any FMaxNum or FMinNum reductions.
static void foldPredicateMerge(VPlan &Plan, VPCostContext &CostCtx)
Wrap reduction operations and their vp.merge in expression recipes when the merge is expected to be f...
static void removeDeadRecipes(VPlan &Plan)
Remove dead recipes from Plan.
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock, bool AddBranchWeights)
static LLVM_ABI_FOR_TEST void handleCountableEarlyExits(VPlan &Plan)
Disconnect countable early exits from the loop.
static void sinkPredicatedStores(VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L)
Sink predicated stores to the same address with complementary predicates (P and NOT P) to an uncondit...
static bool finalizeSCEVPredicates(VPlan &Plan, PredicatedScalarEvolution &PSE, bool OptForSize, unsigned SCEVCheckThreshold, OptimizationRemarkEmitter *ORE, Loop *TheLoop)
Finalize SCEV predicates by adding induction predicates from Plan to PSE and checking constraints.
static void replicateByVF(VPlan &Plan, ElementCount VF)
Replace replicating VPReplicateRecipe, VPScalarIVStepsRecipe and VPInstruction in Plan with VF single...
static bool removeBranchOnConst(VPlan &Plan, bool OnlyLatches=false)
Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their ...
static void convertToStridedAccesses(VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &L, VPCostContext &Ctx, VFRange &Range)
Transform widen memory recipes into strided access recipes when legal and profitable.
static void addIterationCountCheckBlock(VPlan &Plan, ElementCount VF, unsigned UF, bool RequiresScalarEpilogue, Loop *OrigLoop, const uint32_t *MinItersBypassWeights, DebugLoc DL, PredicatedScalarEvolution &PSE)
Add a new check block before the vector preheader to Plan to check if the main vector loop should be ...
static void clearReductionWrapFlags(VPlan &Plan)
Clear NSW/NUW flags from reduction instructions if necessary.
static void createPartialReductions(VPlan &Plan, VPCostContext &CostCtx, VFRange &Range)
Detect and create partial reduction recipes for scaled or unordered reductions in Plan.
static bool widenCompressingPatterns(VPlan &Plan, VPRecipeBuilder &RecipeBuilder, ScalarEvolution &SE)
Widens compressing memory loads/stores.
static void cse(VPlan &Plan, const VPDominatorTree &VPDT)
Perform common-subexpression-elimination on Plan.
static void attachMemoryChecks(VPlan &Plan, const RuntimePointerChecking &RtPtrChecking, const Loop *TheLoop, ScalarEvolution &SE, ElementCount VF, unsigned UF, bool AddBranchWeights)
Generate the memory runtime checks of RtPtrChecking for TheLoop as recipes and attach the check block...
static void materializeUnpacks(VPlan &Plan)
Add explicit Unpack recipes to extract scalars from vectors as needed.
static void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, Loop *OrigLoop, const uint32_t *MinItersBypassWeights, DebugLoc DL, PredicatedScalarEvolution &PSE, VPBasicBlock *CheckBlock)
static void replaceSymbolicStrides(VPlan &Plan, PredicatedScalarEvolution &PSE, const SymbolicStrideMap &StridesMap, const VPDominatorTree &VPDT)
Replace symbolic strides from StridesMap in Plan with constants when possible.
static LLVM_ABI_FOR_TEST void optimize(VPlan &Plan)
Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations,...
static void dissolveLoopRegions(VPlan &Plan)
Replace loop regions with explicit CFG.
static void truncateToMinimalBitwidths(VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
Insert truncates and extends for any truncated recipe.
static void narrowInductionTruncates(VPlan &Plan, VFRange &Range, const TargetTransformInfo &TTI)
Replace truncates of a wide induction, or of that induction's increment, by a VPWidenIntOrFpInduction...
static void dropPoisonGeneratingRecipes(VPlan &Plan)
Drop poison flags from recipes that may generate a poison value that is used after vectorization,...
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
Optimize Plan based on BestVF and BestUF.
static void widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF, unsigned UF, const TargetTransformInfo &TTI)
Widens memory operations by a factor of UF based on a target hook.
static void convertEVLExitCond(VPlan &Plan)
Replaces the exit condition from (branch-on-cond eq CanonicalIVInc, VectorTripCount) to (branch-on-co...
static bool splitCombinedExits(VPlan &Plan, PredicatedScalarEvolution &PSE, Loop *TheLoop)
If a single exit has multiple conditions combined together, split them and create new exiting blocks.
static void combineRecipes(VPlan &Plan)
Perform instcombine-like simplifications on recipes in Plan.
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI unsigned VectorizeMemoryCheckThreshold
The maximum allowed number of runtime memory checks.
static LLVM_ABI bool HoistRuntimeChecks