LLVM 22.0.0git
LoopVectorize.cpp
Go to the documentation of this file.
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "llvm/ADT/APInt.h"
68#include "llvm/ADT/ArrayRef.h"
69#include "llvm/ADT/DenseMap.h"
71#include "llvm/ADT/Hashing.h"
72#include "llvm/ADT/MapVector.h"
73#include "llvm/ADT/STLExtras.h"
76#include "llvm/ADT/Statistic.h"
77#include "llvm/ADT/StringRef.h"
78#include "llvm/ADT/Twine.h"
79#include "llvm/ADT/TypeSwitch.h"
84#include "llvm/Analysis/CFG.h"
101#include "llvm/IR/Attributes.h"
102#include "llvm/IR/BasicBlock.h"
103#include "llvm/IR/CFG.h"
104#include "llvm/IR/Constant.h"
105#include "llvm/IR/Constants.h"
106#include "llvm/IR/DataLayout.h"
107#include "llvm/IR/DebugInfo.h"
108#include "llvm/IR/DebugLoc.h"
109#include "llvm/IR/DerivedTypes.h"
111#include "llvm/IR/Dominators.h"
112#include "llvm/IR/Function.h"
113#include "llvm/IR/IRBuilder.h"
114#include "llvm/IR/InstrTypes.h"
115#include "llvm/IR/Instruction.h"
116#include "llvm/IR/Instructions.h"
118#include "llvm/IR/Intrinsics.h"
119#include "llvm/IR/MDBuilder.h"
120#include "llvm/IR/Metadata.h"
121#include "llvm/IR/Module.h"
122#include "llvm/IR/Operator.h"
123#include "llvm/IR/PatternMatch.h"
125#include "llvm/IR/Type.h"
126#include "llvm/IR/Use.h"
127#include "llvm/IR/User.h"
128#include "llvm/IR/Value.h"
129#include "llvm/IR/Verifier.h"
130#include "llvm/Support/Casting.h"
132#include "llvm/Support/Debug.h"
147#include <algorithm>
148#include <cassert>
149#include <cstdint>
150#include <functional>
151#include <iterator>
152#include <limits>
153#include <memory>
154#include <string>
155#include <tuple>
156#include <utility>
157
158using namespace llvm;
159using namespace SCEVPatternMatch;
160
161#define LV_NAME "loop-vectorize"
162#define DEBUG_TYPE LV_NAME
163
164#ifndef NDEBUG
165const char VerboseDebug[] = DEBUG_TYPE "-verbose";
166#endif
167
168STATISTIC(LoopsVectorized, "Number of loops vectorized");
169STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
170STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
171STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
172
174 "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
175 cl::desc("Enable vectorization of epilogue loops."));
176
178 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
179 cl::desc("When epilogue vectorization is enabled, and a value greater than "
180 "1 is specified, forces the given VF for all applicable epilogue "
181 "loops."));
182
184 "epilogue-vectorization-minimum-VF", cl::Hidden,
185 cl::desc("Only loops with vectorization factor equal to or larger than "
186 "the specified value are considered for epilogue vectorization."));
187
188/// Loops with a known constant trip count below this number are vectorized only
189/// if no scalar iteration overheads are incurred.
191 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
192 cl::desc("Loops with a constant trip count that is smaller than this "
193 "value are vectorized only if no scalar iteration overheads "
194 "are incurred."));
195
197 "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
198 cl::desc("The maximum allowed number of runtime memory checks"));
199
200// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
201// that predication is preferred, and this lists all options. I.e., the
202// vectorizer will try to fold the tail-loop (epilogue) into the vector body
203// and predicate the instructions accordingly. If tail-folding fails, there are
204// different fallback strategies depending on these values:
211} // namespace PreferPredicateTy
212
214 "prefer-predicate-over-epilogue",
217 cl::desc("Tail-folding and predication preferences over creating a scalar "
218 "epilogue loop."),
220 "scalar-epilogue",
221 "Don't tail-predicate loops, create scalar epilogue"),
223 "predicate-else-scalar-epilogue",
224 "prefer tail-folding, create scalar epilogue if tail "
225 "folding fails."),
227 "predicate-dont-vectorize",
228 "prefers tail-folding, don't attempt vectorization if "
229 "tail-folding fails.")));
230
232 "force-tail-folding-style", cl::desc("Force the tail folding style"),
235 clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),
238 "Create lane mask for data only, using active.lane.mask intrinsic"),
240 "data-without-lane-mask",
241 "Create lane mask with compare/stepvector"),
243 "Create lane mask using active.lane.mask intrinsic, and use "
244 "it for both data and control flow"),
246 "data-and-control-without-rt-check",
247 "Similar to data-and-control, but remove the runtime check"),
249 "Use predicated EVL instructions for tail folding. If EVL "
250 "is unsupported, fallback to data-without-lane-mask.")));
251
253 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
254 cl::desc("Maximize bandwidth when selecting vectorization factor which "
255 "will be determined by the smallest type in loop."));
256
258 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
259 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
260
261/// An interleave-group may need masking if it resides in a block that needs
262/// predication, or in order to mask away gaps.
264 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
265 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
266
268 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
269 cl::desc("A flag that overrides the target's number of scalar registers."));
270
272 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
273 cl::desc("A flag that overrides the target's number of vector registers."));
274
276 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
277 cl::desc("A flag that overrides the target's max interleave factor for "
278 "scalar loops."));
279
281 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
282 cl::desc("A flag that overrides the target's max interleave factor for "
283 "vectorized loops."));
284
286 "force-target-instruction-cost", cl::init(0), cl::Hidden,
287 cl::desc("A flag that overrides the target's expected cost for "
288 "an instruction to a single constant value. Mostly "
289 "useful for getting consistent testing."));
290
292 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
293 cl::desc(
294 "Pretend that scalable vectors are supported, even if the target does "
295 "not support them. This flag should only be used for testing."));
296
298 "small-loop-cost", cl::init(20), cl::Hidden,
299 cl::desc(
300 "The cost of a loop that is considered 'small' by the interleaver."));
301
303 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
304 cl::desc("Enable the use of the block frequency analysis to access PGO "
305 "heuristics minimizing code growth in cold regions and being more "
306 "aggressive in hot regions."));
307
308// Runtime interleave loops for load/store throughput.
310 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
311 cl::desc(
312 "Enable runtime interleaving until load/store ports are saturated"));
313
314/// The number of stores in a loop that are allowed to need predication.
316 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
317 cl::desc("Max number of stores to be predicated behind an if."));
318
320 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
321 cl::desc("Count the induction variable only once when interleaving"));
322
324 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
325 cl::desc("Enable if predication of stores during vectorization."));
326
328 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
329 cl::desc("The maximum interleave count to use when interleaving a scalar "
330 "reduction in a nested loop."));
331
332static cl::opt<bool>
333 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
335 cl::desc("Prefer in-loop vector reductions, "
336 "overriding the targets preference."));
337
339 "force-ordered-reductions", cl::init(false), cl::Hidden,
340 cl::desc("Enable the vectorisation of loops with in-order (strict) "
341 "FP reductions"));
342
344 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
345 cl::desc(
346 "Prefer predicating a reduction operation over an after loop select."));
347
349 "enable-vplan-native-path", cl::Hidden,
350 cl::desc("Enable VPlan-native vectorization path with "
351 "support for outer loop vectorization."));
352
354 llvm::VerifyEachVPlan("vplan-verify-each",
355#ifdef EXPENSIVE_CHECKS
356 cl::init(true),
357#else
358 cl::init(false),
359#endif
361 cl::desc("Verfiy VPlans after VPlan transforms."));
362
363// This flag enables the stress testing of the VPlan H-CFG construction in the
364// VPlan-native vectorization path. It must be used in conjuction with
365// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
366// verification of the H-CFGs built.
368 "vplan-build-stress-test", cl::init(false), cl::Hidden,
369 cl::desc(
370 "Build VPlan for every supported loop nest in the function and bail "
371 "out right after the build (stress test the VPlan H-CFG construction "
372 "in the VPlan-native vectorization path)."));
373
375 "interleave-loops", cl::init(true), cl::Hidden,
376 cl::desc("Enable loop interleaving in Loop vectorization passes"));
378 "vectorize-loops", cl::init(true), cl::Hidden,
379 cl::desc("Run the Loop vectorization passes"));
380
382 "force-widen-divrem-via-safe-divisor", cl::Hidden,
383 cl::desc(
384 "Override cost based safe divisor widening for div/rem instructions"));
385
387 "vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true),
389 cl::desc("Try wider VFs if they enable the use of vector variants"));
390
392 "enable-early-exit-vectorization", cl::init(true), cl::Hidden,
393 cl::desc(
394 "Enable vectorization of early exit loops with uncountable exits."));
395
397 "vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
398 cl::desc("Discard VFs if their register pressure is too high."));
399
400// Likelyhood of bypassing the vectorized loop because there are zero trips left
401// after prolog. See `emitIterationCountCheck`.
402static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
403
404/// A helper function that returns true if the given type is irregular. The
405/// type is irregular if its allocated size doesn't equal the store size of an
406/// element of the corresponding vector type.
407static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
408 // Determine if an array of N elements of type Ty is "bitcast compatible"
409 // with a <N x Ty> vector.
410 // This is only true if there is no padding between the array elements.
411 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
412}
413
414/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
415/// ElementCount to include loops whose trip count is a function of vscale.
417 const Loop *L) {
418 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
419 return ElementCount::getFixed(ExpectedTC);
420
421 const SCEV *BTC = SE->getBackedgeTakenCount(L);
423 return ElementCount::getFixed(0);
424
425 const SCEV *ExitCount = SE->getTripCountFromExitCount(BTC, BTC->getType(), L);
426 if (isa<SCEVVScale>(ExitCount))
428
429 const APInt *Scale;
430 if (match(ExitCount, m_scev_Mul(m_scev_APInt(Scale), m_SCEVVScale())))
431 if (cast<SCEVMulExpr>(ExitCount)->hasNoUnsignedWrap())
432 if (Scale->getActiveBits() <= 32)
434
435 return ElementCount::getFixed(0);
436}
437
438/// Returns "best known" trip count, which is either a valid positive trip count
439/// or std::nullopt when an estimate cannot be made (including when the trip
440/// count would overflow), for the specified loop \p L as defined by the
441/// following procedure:
442/// 1) Returns exact trip count if it is known.
443/// 2) Returns expected trip count according to profile data if any.
444/// 3) Returns upper bound estimate if known, and if \p CanUseConstantMax.
445/// 4) Returns std::nullopt if all of the above failed.
446static std::optional<ElementCount>
448 bool CanUseConstantMax = true) {
449 // Check if exact trip count is known.
450 if (auto ExpectedTC = getSmallConstantTripCount(PSE.getSE(), L))
451 return ExpectedTC;
452
453 // Check if there is an expected trip count available from profile data.
455 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
456 return ElementCount::getFixed(*EstimatedTC);
457
458 if (!CanUseConstantMax)
459 return std::nullopt;
460
461 // Check if upper bound estimate is known.
462 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
463 return ElementCount::getFixed(ExpectedTC);
464
465 return std::nullopt;
466}
467
468namespace {
469// Forward declare GeneratedRTChecks.
470class GeneratedRTChecks;
471
472using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
473} // namespace
474
475namespace llvm {
476
478
479/// InnerLoopVectorizer vectorizes loops which contain only one basic
480/// block to a specified vectorization factor (VF).
481/// This class performs the widening of scalars into vectors, or multiple
482/// scalars. This class also implements the following features:
483/// * It inserts an epilogue loop for handling loops that don't have iteration
484/// counts that are known to be a multiple of the vectorization factor.
485/// * It handles the code generation for reduction variables.
486/// * Scalarization (implementation using scalars) of un-vectorizable
487/// instructions.
488/// InnerLoopVectorizer does not perform any vectorization-legality
489/// checks, and relies on the caller to check for the different legality
490/// aspects. The InnerLoopVectorizer relies on the
491/// LoopVectorizationLegality class to provide information about the induction
492/// and reduction variables that were found to a given vectorization factor.
494public:
498 ElementCount VecWidth, unsigned UnrollFactor,
500 ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks,
501 VPlan &Plan)
502 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
503 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
506 Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
507
508 virtual ~InnerLoopVectorizer() = default;
509
510 /// Creates a basic block for the scalar preheader. Both
511 /// EpilogueVectorizerMainLoop and EpilogueVectorizerEpilogueLoop overwrite
512 /// the method to create additional blocks and checks needed for epilogue
513 /// vectorization.
515
516 /// Fix the vectorized code, taking care of header phi's, and more.
518
519 /// Fix the non-induction PHIs in \p Plan.
521
522 /// Returns the original loop trip count.
523 Value *getTripCount() const { return TripCount; }
524
525 /// Used to set the trip count after ILV's construction and after the
526 /// preheader block has been executed. Note that this always holds the trip
527 /// count of the original loop for both main loop and epilogue vectorization.
528 void setTripCount(Value *TC) { TripCount = TC; }
529
530protected:
532
533 /// Create and return a new IR basic block for the scalar preheader whose name
534 /// is prefixed with \p Prefix.
536
537 /// Allow subclasses to override and print debug traces before/after vplan
538 /// execution, when trace information is requested.
539 virtual void printDebugTracesAtStart() {}
540 virtual void printDebugTracesAtEnd() {}
541
542 /// The original loop.
544
545 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
546 /// dynamic knowledge to simplify SCEV expressions and converts them to a
547 /// more usable form.
549
550 /// Loop Info.
552
553 /// Dominator Tree.
555
556 /// Target Transform Info.
558
559 /// Assumption Cache.
561
562 /// The vectorization SIMD factor to use. Each vector will have this many
563 /// vector elements.
565
566 /// The vectorization unroll factor to use. Each scalar is vectorized to this
567 /// many different vector instructions.
568 unsigned UF;
569
570 /// The builder that we use
572
573 // --- Vectorization state ---
574
575 /// Trip count of the original loop.
576 Value *TripCount = nullptr;
577
578 /// The profitablity analysis.
580
581 /// BFI and PSI are used to check for profile guided size optimizations.
584
585 /// Structure to hold information about generated runtime checks, responsible
586 /// for cleaning the checks, if vectorization turns out unprofitable.
587 GeneratedRTChecks &RTChecks;
588
590
591 /// The vector preheader block of \p Plan, used as target for check blocks
592 /// introduced during skeleton creation.
594};
595
596/// Encapsulate information regarding vectorization of a loop and its epilogue.
597/// This information is meant to be updated and used across two stages of
598/// epilogue vectorization.
601 unsigned MainLoopUF = 0;
603 unsigned EpilogueUF = 0;
606 Value *TripCount = nullptr;
609
611 ElementCount EVF, unsigned EUF,
613 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF),
615 assert(EUF == 1 &&
616 "A high UF for the epilogue loop is likely not beneficial.");
617 }
618};
619
620/// An extension of the inner loop vectorizer that creates a skeleton for a
621/// vectorized loop that has its epilogue (residual) also vectorized.
622/// The idea is to run the vplan on a given loop twice, firstly to setup the
623/// skeleton and vectorize the main loop, and secondly to complete the skeleton
624/// from the first step and vectorize the epilogue. This is achieved by
625/// deriving two concrete strategy classes from this base class and invoking
626/// them in succession from the loop vectorizer planner.
628public:
639
640 /// Holds and updates state information required to vectorize the main loop
641 /// and its epilogue in two separate passes. This setup helps us avoid
642 /// regenerating and recomputing runtime safety checks. It also helps us to
643 /// shorten the iteration-count-check path length for the cases where the
644 /// iteration count of the loop is so small that the main vector loop is
645 /// completely skipped.
647
648protected:
650};
651
652/// A specialized derived class of inner loop vectorizer that performs
653/// vectorization of *main* loops in the process of vectorizing loops and their
654/// epilogues.
656public:
668 /// Implements the interface for creating a vectorized skeleton using the
669 /// *main loop* strategy (i.e., the first pass of VPlan execution).
671
672protected:
673 /// Introduces a new VPIRBasicBlock for \p CheckIRBB to Plan between the
674 /// vector preheader and its predecessor, also connecting the new block to the
675 /// scalar preheader.
676 void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB);
677
678 // Create a check to see if the main vector loop should be executed
680 unsigned UF) const;
681
682 /// Emits an iteration count bypass check once for the main loop (when \p
683 /// ForEpilogue is false) and once for the epilogue loop (when \p
684 /// ForEpilogue is true).
686 bool ForEpilogue);
687 void printDebugTracesAtStart() override;
688 void printDebugTracesAtEnd() override;
689};
690
691// A specialized derived class of inner loop vectorizer that performs
692// vectorization of *epilogue* loops in the process of vectorizing loops and
693// their epilogues.
695public:
705 /// Implements the interface for creating a vectorized skeleton using the
706 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
708
709protected:
710 void printDebugTracesAtStart() override;
711 void printDebugTracesAtEnd() override;
712};
713} // end namespace llvm
714
715/// Look for a meaningful debug location on the instruction or its operands.
717 if (!I)
718 return DebugLoc::getUnknown();
719
721 if (I->getDebugLoc() != Empty)
722 return I->getDebugLoc();
723
724 for (Use &Op : I->operands()) {
725 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
726 if (OpInst->getDebugLoc() != Empty)
727 return OpInst->getDebugLoc();
728 }
729
730 return I->getDebugLoc();
731}
732
733/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
734/// is passed, the message relates to that particular instruction.
735#ifndef NDEBUG
736static void debugVectorizationMessage(const StringRef Prefix,
737 const StringRef DebugMsg,
738 Instruction *I) {
739 dbgs() << "LV: " << Prefix << DebugMsg;
740 if (I != nullptr)
741 dbgs() << " " << *I;
742 else
743 dbgs() << '.';
744 dbgs() << '\n';
745}
746#endif
747
748/// Create an analysis remark that explains why vectorization failed
749///
750/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
751/// RemarkName is the identifier for the remark. If \p I is passed it is an
752/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
753/// the location of the remark. If \p DL is passed, use it as debug location for
754/// the remark. \return the remark object that can be streamed to.
755static OptimizationRemarkAnalysis
756createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop,
757 Instruction *I, DebugLoc DL = {}) {
758 BasicBlock *CodeRegion = I ? I->getParent() : TheLoop->getHeader();
759 // If debug location is attached to the instruction, use it. Otherwise if DL
760 // was not provided, use the loop's.
761 if (I && I->getDebugLoc())
762 DL = I->getDebugLoc();
763 else if (!DL)
764 DL = TheLoop->getStartLoc();
765
766 return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
767}
768
769namespace llvm {
770
771/// Return a value for Step multiplied by VF.
773 int64_t Step) {
774 assert(Ty->isIntegerTy() && "Expected an integer step");
775 ElementCount VFxStep = VF.multiplyCoefficientBy(Step);
776 assert(isPowerOf2_64(VF.getKnownMinValue()) && "must pass power-of-2 VF");
777 if (VF.isScalable() && isPowerOf2_64(Step)) {
778 return B.CreateShl(
779 B.CreateVScale(Ty),
780 ConstantInt::get(Ty, Log2_64(VFxStep.getKnownMinValue())), "", true);
781 }
782 return B.CreateElementCount(Ty, VFxStep);
783}
784
785/// Return the runtime value for VF.
787 return B.CreateElementCount(Ty, VF);
788}
789
791 const StringRef OREMsg, const StringRef ORETag,
792 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
793 Instruction *I) {
794 LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I));
795 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
796 ORE->emit(
797 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
798 << "loop not vectorized: " << OREMsg);
799}
800
801/// Reports an informative message: print \p Msg for debugging purposes as well
802/// as an optimization remark. Uses either \p I as location of the remark, or
803/// otherwise \p TheLoop. If \p DL is passed, use it as debug location for the
804/// remark. If \p DL is passed, use it as debug location for the remark.
805static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
807 Loop *TheLoop, Instruction *I = nullptr,
808 DebugLoc DL = {}) {
810 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
811 ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop,
812 I, DL)
813 << Msg);
814}
815
816/// Report successful vectorization of the loop. In case an outer loop is
817/// vectorized, prepend "outer" to the vectorization remark.
819 VectorizationFactor VF, unsigned IC) {
821 "Vectorizing: ", TheLoop->isInnermost() ? "innermost loop" : "outer loop",
822 nullptr));
823 StringRef LoopType = TheLoop->isInnermost() ? "" : "outer ";
824 ORE->emit([&]() {
825 return OptimizationRemark(LV_NAME, "Vectorized", TheLoop->getStartLoc(),
826 TheLoop->getHeader())
827 << "vectorized " << LoopType << "loop (vectorization width: "
828 << ore::NV("VectorizationFactor", VF.Width)
829 << ", interleaved count: " << ore::NV("InterleaveCount", IC) << ")";
830 });
831}
832
833} // end namespace llvm
834
835namespace llvm {
836
837// Loop vectorization cost-model hints how the scalar epilogue loop should be
838// lowered.
840
841 // The default: allowing scalar epilogues.
843
844 // Vectorization with OptForSize: don't allow epilogues.
846
847 // A special case of vectorisation with OptForSize: loops with a very small
848 // trip count are considered for vectorization under OptForSize, thereby
849 // making sure the cost of their loop body is dominant, free of runtime
850 // guards and scalar iteration overheads.
852
853 // Loop hint predicate indicating an epilogue is undesired.
855
856 // Directive indicating we must either tail fold or not vectorize
858};
859
860/// LoopVectorizationCostModel - estimates the expected speedups due to
861/// vectorization.
862/// In many cases vectorization is not profitable. This can happen because of
863/// a number of reasons. In this class we mainly attempt to predict the
864/// expected speedup/slowdowns due to the supported instruction set. We use the
865/// TargetTransformInfo to query the different backends for the cost of
866/// different operations.
869
870public:
881 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
882 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
883 Hints(Hints), InterleaveInfo(IAI) {
884 if (TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors)
885 initializeVScaleForTuning();
887 // Query this against the original loop and save it here because the profile
888 // of the original loop header may change as the transformation happens.
889 OptForSize = llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
891 }
892
893 /// \return An upper bound for the vectorization factors (both fixed and
894 /// scalable). If the factors are 0, vectorization and interleaving should be
895 /// avoided up front.
896 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
897
898 /// \return True if runtime checks are required for vectorization, and false
899 /// otherwise.
900 bool runtimeChecksRequired();
901
902 /// Setup cost-based decisions for user vectorization factor.
903 /// \return true if the UserVF is a feasible VF to be chosen.
906 return expectedCost(UserVF).isValid();
907 }
908
909 /// \return True if maximizing vector bandwidth is enabled by the target or
910 /// user options, for the given register kind.
911 bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind);
912
913 /// \return True if register pressure should be considered for the given VF.
914 bool shouldConsiderRegPressureForVF(ElementCount VF);
915
916 /// \return The size (in bits) of the smallest and widest types in the code
917 /// that needs to be vectorized. We ignore values that remain scalar such as
918 /// 64 bit loop indices.
919 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
920
921 /// Memory access instruction may be vectorized in more than one way.
922 /// Form of instruction after vectorization depends on cost.
923 /// This function takes cost-based decisions for Load/Store instructions
924 /// and collects them in a map. This decisions map is used for building
925 /// the lists of loop-uniform and loop-scalar instructions.
926 /// The calculated cost is saved with widening decision in order to
927 /// avoid redundant calculations.
928 void setCostBasedWideningDecision(ElementCount VF);
929
930 /// A call may be vectorized in different ways depending on whether we have
931 /// vectorized variants available and whether the target supports masking.
932 /// This function analyzes all calls in the function at the supplied VF,
933 /// makes a decision based on the costs of available options, and stores that
934 /// decision in a map for use in planning and plan execution.
935 void setVectorizedCallDecision(ElementCount VF);
936
937 /// Collect values we want to ignore in the cost model.
938 void collectValuesToIgnore();
939
940 /// Collect all element types in the loop for which widening is needed.
941 void collectElementTypesForWidening();
942
943 /// Split reductions into those that happen in the loop, and those that happen
944 /// outside. In loop reductions are collected into InLoopReductions.
945 void collectInLoopReductions();
946
947 /// Returns true if we should use strict in-order reductions for the given
948 /// RdxDesc. This is true if the -enable-strict-reductions flag is passed,
949 /// the IsOrdered flag of RdxDesc is set and we do not allow reordering
950 /// of FP operations.
951 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const {
952 return !Hints->allowReordering() && RdxDesc.isOrdered();
953 }
954
955 /// \returns The smallest bitwidth each instruction can be represented with.
956 /// The vector equivalents of these instructions should be truncated to this
957 /// type.
959 return MinBWs;
960 }
961
962 /// \returns True if it is more profitable to scalarize instruction \p I for
963 /// vectorization factor \p VF.
965 assert(VF.isVector() &&
966 "Profitable to scalarize relevant only for VF > 1.");
967 assert(
968 TheLoop->isInnermost() &&
969 "cost-model should not be used for outer loops (in VPlan-native path)");
970
971 auto Scalars = InstsToScalarize.find(VF);
972 assert(Scalars != InstsToScalarize.end() &&
973 "VF not yet analyzed for scalarization profitability");
974 return Scalars->second.contains(I);
975 }
976
977 /// Returns true if \p I is known to be uniform after vectorization.
979 assert(
980 TheLoop->isInnermost() &&
981 "cost-model should not be used for outer loops (in VPlan-native path)");
982 // Pseudo probe needs to be duplicated for each unrolled iteration and
983 // vector lane so that profiled loop trip count can be accurately
984 // accumulated instead of being under counted.
986 return false;
987
988 if (VF.isScalar())
989 return true;
990
991 auto UniformsPerVF = Uniforms.find(VF);
992 assert(UniformsPerVF != Uniforms.end() &&
993 "VF not yet analyzed for uniformity");
994 return UniformsPerVF->second.count(I);
995 }
996
997 /// Returns true if \p I is known to be scalar after vectorization.
999 assert(
1000 TheLoop->isInnermost() &&
1001 "cost-model should not be used for outer loops (in VPlan-native path)");
1002 if (VF.isScalar())
1003 return true;
1004
1005 auto ScalarsPerVF = Scalars.find(VF);
1006 assert(ScalarsPerVF != Scalars.end() &&
1007 "Scalar values are not calculated for VF");
1008 return ScalarsPerVF->second.count(I);
1009 }
1010
1011 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1012 /// for vectorization factor \p VF.
1014 return VF.isVector() && MinBWs.contains(I) &&
1015 !isProfitableToScalarize(I, VF) &&
1017 }
1018
1019 /// Decision that was taken during cost calculation for memory instruction.
1022 CM_Widen, // For consecutive accesses with stride +1.
1023 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1029 };
1030
1031 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1032 /// instruction \p I and vector width \p VF.
1035 assert(VF.isVector() && "Expected VF >=2");
1036 WideningDecisions[{I, VF}] = {W, Cost};
1037 }
1038
1039 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1040 /// interleaving group \p Grp and vector width \p VF.
1044 assert(VF.isVector() && "Expected VF >=2");
1045 /// Broadcast this decicion to all instructions inside the group.
1046 /// When interleaving, the cost will only be assigned one instruction, the
1047 /// insert position. For other cases, add the appropriate fraction of the
1048 /// total cost to each instruction. This ensures accurate costs are used,
1049 /// even if the insert position instruction is not used.
1050 InstructionCost InsertPosCost = Cost;
1051 InstructionCost OtherMemberCost = 0;
1052 if (W != CM_Interleave)
1053 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
1054 ;
1055 for (unsigned Idx = 0; Idx < Grp->getFactor(); ++Idx) {
1056 if (auto *I = Grp->getMember(Idx)) {
1057 if (Grp->getInsertPos() == I)
1058 WideningDecisions[{I, VF}] = {W, InsertPosCost};
1059 else
1060 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
1061 }
1062 }
1063 }
1064
1065 /// Return the cost model decision for the given instruction \p I and vector
1066 /// width \p VF. Return CM_Unknown if this instruction did not pass
1067 /// through the cost modeling.
1069 assert(VF.isVector() && "Expected VF to be a vector VF");
1070 assert(
1071 TheLoop->isInnermost() &&
1072 "cost-model should not be used for outer loops (in VPlan-native path)");
1073
1074 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
1075 auto Itr = WideningDecisions.find(InstOnVF);
1076 if (Itr == WideningDecisions.end())
1077 return CM_Unknown;
1078 return Itr->second.first;
1079 }
1080
1081 /// Return the vectorization cost for the given instruction \p I and vector
1082 /// width \p VF.
1084 assert(VF.isVector() && "Expected VF >=2");
1085 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
1086 assert(WideningDecisions.contains(InstOnVF) &&
1087 "The cost is not calculated");
1088 return WideningDecisions[InstOnVF].second;
1089 }
1090
1098
1100 Function *Variant, Intrinsic::ID IID,
1101 std::optional<unsigned> MaskPos,
1103 assert(!VF.isScalar() && "Expected vector VF");
1104 CallWideningDecisions[{CI, VF}] = {Kind, Variant, IID, MaskPos, Cost};
1105 }
1106
1108 ElementCount VF) const {
1109 assert(!VF.isScalar() && "Expected vector VF");
1110 auto I = CallWideningDecisions.find({CI, VF});
1111 if (I == CallWideningDecisions.end())
1112 return {CM_Unknown, nullptr, Intrinsic::not_intrinsic, std::nullopt, 0};
1113 return I->second;
1114 }
1115
1116 /// Return True if instruction \p I is an optimizable truncate whose operand
1117 /// is an induction variable. Such a truncate will be removed by adding a new
1118 /// induction variable with the destination type.
1120 // If the instruction is not a truncate, return false.
1121 auto *Trunc = dyn_cast<TruncInst>(I);
1122 if (!Trunc)
1123 return false;
1124
1125 // Get the source and destination types of the truncate.
1126 Type *SrcTy = toVectorTy(Trunc->getSrcTy(), VF);
1127 Type *DestTy = toVectorTy(Trunc->getDestTy(), VF);
1128
1129 // If the truncate is free for the given types, return false. Replacing a
1130 // free truncate with an induction variable would add an induction variable
1131 // update instruction to each iteration of the loop. We exclude from this
1132 // check the primary induction variable since it will need an update
1133 // instruction regardless.
1134 Value *Op = Trunc->getOperand(0);
1135 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1136 return false;
1137
1138 // If the truncated value is not an induction variable, return false.
1139 return Legal->isInductionPhi(Op);
1140 }
1141
1142 /// Collects the instructions to scalarize for each predicated instruction in
1143 /// the loop.
1144 void collectInstsToScalarize(ElementCount VF);
1145
1146 /// Collect values that will not be widened, including Uniforms, Scalars, and
1147 /// Instructions to Scalarize for the given \p VF.
1148 /// The sets depend on CM decision for Load/Store instructions
1149 /// that may be vectorized as interleave, gather-scatter or scalarized.
1150 /// Also make a decision on what to do about call instructions in the loop
1151 /// at that VF -- scalarize, call a known vector routine, or call a
1152 /// vector intrinsic.
1154 // Do the analysis once.
1155 if (VF.isScalar() || Uniforms.contains(VF))
1156 return;
1158 collectLoopUniforms(VF);
1160 collectLoopScalars(VF);
1162 }
1163
1164 /// Returns true if the target machine supports masked store operation
1165 /// for the given \p DataType and kind of access to \p Ptr.
1166 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment,
1167 unsigned AddressSpace) const {
1168 return Legal->isConsecutivePtr(DataType, Ptr) &&
1169 TTI.isLegalMaskedStore(DataType, Alignment, AddressSpace);
1170 }
1171
1172 /// Returns true if the target machine supports masked load operation
1173 /// for the given \p DataType and kind of access to \p Ptr.
1174 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment,
1175 unsigned AddressSpace) const {
1176 return Legal->isConsecutivePtr(DataType, Ptr) &&
1177 TTI.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
1178 }
1179
1180 /// Returns true if the target machine can represent \p V as a masked gather
1181 /// or scatter operation.
1183 bool LI = isa<LoadInst>(V);
1184 bool SI = isa<StoreInst>(V);
1185 if (!LI && !SI)
1186 return false;
1187 auto *Ty = getLoadStoreType(V);
1189 if (VF.isVector())
1190 Ty = VectorType::get(Ty, VF);
1191 return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
1192 (SI && TTI.isLegalMaskedScatter(Ty, Align));
1193 }
1194
1195 /// Returns true if the target machine supports all of the reduction
1196 /// variables found for the given VF.
1198 return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool {
1199 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1200 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1201 }));
1202 }
1203
1204 /// Given costs for both strategies, return true if the scalar predication
1205 /// lowering should be used for div/rem. This incorporates an override
1206 /// option so it is not simply a cost comparison.
1208 InstructionCost SafeDivisorCost) const {
1209 switch (ForceSafeDivisor) {
1210 case cl::BOU_UNSET:
1211 return ScalarCost < SafeDivisorCost;
1212 case cl::BOU_TRUE:
1213 return false;
1214 case cl::BOU_FALSE:
1215 return true;
1216 }
1217 llvm_unreachable("impossible case value");
1218 }
1219
1220 /// Returns true if \p I is an instruction which requires predication and
1221 /// for which our chosen predication strategy is scalarization (i.e. we
1222 /// don't have an alternate strategy such as masking available).
1223 /// \p VF is the vectorization factor that will be used to vectorize \p I.
1224 bool isScalarWithPredication(Instruction *I, ElementCount VF) const;
1225
1226 /// Returns true if \p I is an instruction that needs to be predicated
1227 /// at runtime. The result is independent of the predication mechanism.
1228 /// Superset of instructions that return true for isScalarWithPredication.
1229 bool isPredicatedInst(Instruction *I) const;
1230
1231 /// Return the costs for our two available strategies for lowering a
1232 /// div/rem operation which requires speculating at least one lane.
1233 /// First result is for scalarization (will be invalid for scalable
1234 /// vectors); second is for the safe-divisor strategy.
1235 std::pair<InstructionCost, InstructionCost>
1236 getDivRemSpeculationCost(Instruction *I,
1237 ElementCount VF) const;
1238
1239 /// Returns true if \p I is a memory instruction with consecutive memory
1240 /// access that can be widened.
1241 bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF);
1242
1243 /// Returns true if \p I is a memory instruction in an interleaved-group
1244 /// of memory accesses that can be vectorized with wide vector loads/stores
1245 /// and shuffles.
1246 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
1247
1248 /// Check if \p Instr belongs to any interleaved access group.
1250 return InterleaveInfo.isInterleaved(Instr);
1251 }
1252
1253 /// Get the interleaved access group that \p Instr belongs to.
1256 return InterleaveInfo.getInterleaveGroup(Instr);
1257 }
1258
1259 /// Returns true if we're required to use a scalar epilogue for at least
1260 /// the final iteration of the original loop.
1261 bool requiresScalarEpilogue(bool IsVectorizing) const {
1262 if (!isScalarEpilogueAllowed()) {
1263 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1264 return false;
1265 }
1266 // If we might exit from anywhere but the latch and early exit vectorization
1267 // is disabled, we must run the exiting iteration in scalar form.
1268 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
1269 !(EnableEarlyExitVectorization && Legal->hasUncountableEarlyExit())) {
1270 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1271 "from latch block\n");
1272 return true;
1273 }
1274 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
1275 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
1276 "interleaved group requires scalar epilogue\n");
1277 return true;
1278 }
1279 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1280 return false;
1281 }
1282
1283 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1284 /// loop hint annotation.
1286 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1287 }
1288
1289 /// Returns the TailFoldingStyle that is best for the current loop.
1290 TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
1291 if (!ChosenTailFoldingStyle)
1293 return IVUpdateMayOverflow ? ChosenTailFoldingStyle->first
1294 : ChosenTailFoldingStyle->second;
1295 }
1296
1297 /// Selects and saves TailFoldingStyle for 2 options - if IV update may
1298 /// overflow or not.
1299 /// \param IsScalableVF true if scalable vector factors enabled.
1300 /// \param UserIC User specific interleave count.
1301 void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC) {
1302 assert(!ChosenTailFoldingStyle && "Tail folding must not be selected yet.");
1303 if (!Legal->canFoldTailByMasking()) {
1304 ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
1305 return;
1306 }
1307
1308 // Default to TTI preference, but allow command line override.
1309 ChosenTailFoldingStyle = {
1310 TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
1311 TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
1312 if (ForceTailFoldingStyle.getNumOccurrences())
1313 ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
1314 ForceTailFoldingStyle.getValue()};
1315
1316 if (ChosenTailFoldingStyle->first != TailFoldingStyle::DataWithEVL &&
1317 ChosenTailFoldingStyle->second != TailFoldingStyle::DataWithEVL)
1318 return;
1319 // Override EVL styles if needed.
1320 // FIXME: Investigate opportunity for fixed vector factor.
1321 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1322 TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
1323 if (EVLIsLegal)
1324 return;
1325 // If for some reason EVL mode is unsupported, fallback to a scalar epilogue
1326 // if it's allowed, or DataWithoutLaneMask otherwise.
1327 if (ScalarEpilogueStatus == CM_ScalarEpilogueAllowed ||
1328 ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate)
1329 ChosenTailFoldingStyle = {TailFoldingStyle::None, TailFoldingStyle::None};
1330 else
1331 ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
1333
1334 LLVM_DEBUG(
1335 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
1336 "not try to generate VP Intrinsics "
1337 << (UserIC > 1
1338 ? "since interleave count specified is greater than 1.\n"
1339 : "due to non-interleaving reasons.\n"));
1340 }
1341
1342 /// Returns true if all loop blocks should be masked to fold tail loop.
1343 bool foldTailByMasking() const {
1344 // TODO: check if it is possible to check for None style independent of
1345 // IVUpdateMayOverflow flag in getTailFoldingStyle.
1347 }
1348
1349 /// Return maximum safe number of elements to be processed per vector
1350 /// iteration, which do not prevent store-load forwarding and are safe with
1351 /// regard to the memory dependencies. Required for EVL-based VPlans to
1352 /// correctly calculate AVL (application vector length) as min(remaining AVL,
1353 /// MaxSafeElements).
1354 /// TODO: need to consider adjusting cost model to use this value as a
1355 /// vectorization factor for EVL-based vectorization.
1356 std::optional<unsigned> getMaxSafeElements() const { return MaxSafeElements; }
1357
1358 /// Returns true if the instructions in this block requires predication
1359 /// for any reason, e.g. because tail folding now requires a predicate
1360 /// or because the block in the original loop was predicated.
1362 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1363 }
1364
1365 /// Returns true if VP intrinsics with explicit vector length support should
1366 /// be generated in the tail folded loop.
1370
1371 /// Returns true if the Phi is part of an inloop reduction.
1372 bool isInLoopReduction(PHINode *Phi) const {
1373 return InLoopReductions.contains(Phi);
1374 }
1375
1376 /// Returns true if the predicated reduction select should be used to set the
1377 /// incoming value for the reduction phi.
1379 // Force to use predicated reduction select since the EVL of the
1380 // second-to-last iteration might not be VF*UF.
1381 if (foldTailWithEVL())
1382 return true;
1384 TTI.preferPredicatedReductionSelect();
1385 }
1386
1387 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1388 /// with factor VF. Return the cost of the instruction, including
1389 /// scalarization overhead if it's needed.
1390 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1391
1392 /// Estimate cost of a call instruction CI if it were vectorized with factor
1393 /// VF. Return the cost of the instruction, including scalarization overhead
1394 /// if it's needed.
1395 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
1396
1397 /// Invalidates decisions already taken by the cost model.
1399 WideningDecisions.clear();
1400 CallWideningDecisions.clear();
1401 Uniforms.clear();
1402 Scalars.clear();
1403 }
1404
1405 /// Returns the expected execution cost. The unit of the cost does
1406 /// not matter because we use the 'cost' units to compare different
1407 /// vector widths. The cost that is returned is *not* normalized by
1408 /// the factor width.
1409 InstructionCost expectedCost(ElementCount VF);
1410
1411 bool hasPredStores() const { return NumPredStores > 0; }
1412
1413 /// Returns true if epilogue vectorization is considered profitable, and
1414 /// false otherwise.
1415 /// \p VF is the vectorization factor chosen for the original loop.
1416 /// \p Multiplier is an aditional scaling factor applied to VF before
1417 /// comparing to EpilogueVectorizationMinVF.
1418 bool isEpilogueVectorizationProfitable(const ElementCount VF,
1419 const unsigned IC) const;
1420
1421 /// Returns the execution time cost of an instruction for a given vector
1422 /// width. Vector width of one means scalar.
1423 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
1424
1425 /// Return the cost of instructions in an inloop reduction pattern, if I is
1426 /// part of that pattern.
1427 std::optional<InstructionCost> getReductionPatternCost(Instruction *I,
1428 ElementCount VF,
1429 Type *VectorTy) const;
1430
1431 /// Returns true if \p Op should be considered invariant and if it is
1432 /// trivially hoistable.
1433 bool shouldConsiderInvariant(Value *Op);
1434
1435 /// Return the value of vscale used for tuning the cost model.
1436 std::optional<unsigned> getVScaleForTuning() const { return VScaleForTuning; }
1437
1438private:
1439 unsigned NumPredStores = 0;
1440
1441 /// Used to store the value of vscale used for tuning the cost model. It is
1442 /// initialized during object construction.
1443 std::optional<unsigned> VScaleForTuning;
1444
1445 /// Initializes the value of vscale used for tuning the cost model. If
1446 /// vscale_range.min == vscale_range.max then return vscale_range.max, else
1447 /// return the value returned by the corresponding TTI method.
1448 void initializeVScaleForTuning() {
1449 const Function *Fn = TheLoop->getHeader()->getParent();
1450 if (Fn->hasFnAttribute(Attribute::VScaleRange)) {
1451 auto Attr = Fn->getFnAttribute(Attribute::VScaleRange);
1452 auto Min = Attr.getVScaleRangeMin();
1453 auto Max = Attr.getVScaleRangeMax();
1454 if (Max && Min == Max) {
1455 VScaleForTuning = Max;
1456 return;
1457 }
1458 }
1459
1460 VScaleForTuning = TTI.getVScaleForTuning();
1461 }
1462
1463 /// \return An upper bound for the vectorization factors for both
1464 /// fixed and scalable vectorization, where the minimum-known number of
1465 /// elements is a power-of-2 larger than zero. If scalable vectorization is
1466 /// disabled or unsupported, then the scalable part will be equal to
1467 /// ElementCount::getScalable(0).
1468 FixedScalableVFPair computeFeasibleMaxVF(unsigned MaxTripCount,
1469 ElementCount UserVF,
1470 bool FoldTailByMasking);
1471
1472 /// If \p VF > MaxTripcount, clamps it to the next lower VF that is <=
1473 /// MaxTripCount.
1474 ElementCount clampVFByMaxTripCount(ElementCount VF, unsigned MaxTripCount,
1475 bool FoldTailByMasking) const;
1476
1477 /// \return the maximized element count based on the targets vector
1478 /// registers and the loop trip-count, but limited to a maximum safe VF.
1479 /// This is a helper function of computeFeasibleMaxVF.
1480 ElementCount getMaximizedVFForTarget(unsigned MaxTripCount,
1481 unsigned SmallestType,
1482 unsigned WidestType,
1483 ElementCount MaxSafeVF,
1484 bool FoldTailByMasking);
1485
1486 /// Checks if scalable vectorization is supported and enabled. Caches the
1487 /// result to avoid repeated debug dumps for repeated queries.
1488 bool isScalableVectorizationAllowed();
1489
1490 /// \return the maximum legal scalable VF, based on the safe max number
1491 /// of elements.
1492 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
1493
1494 /// Calculate vectorization cost of memory instruction \p I.
1495 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1496
1497 /// The cost computation for scalarized memory instruction.
1498 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1499
1500 /// The cost computation for interleaving group of memory instructions.
1501 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1502
1503 /// The cost computation for Gather/Scatter instruction.
1504 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1505
1506 /// The cost computation for widening instruction \p I with consecutive
1507 /// memory access.
1508 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1509
1510 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1511 /// Load: scalar load + broadcast.
1512 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1513 /// element)
1514 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1515
1516 /// Estimate the overhead of scalarizing an instruction. This is a
1517 /// convenience wrapper for the type-based getScalarizationOverhead API.
1519 ElementCount VF) const;
1520
1521 /// Returns true if an artificially high cost for emulated masked memrefs
1522 /// should be used.
1523 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
1524
1525 /// Map of scalar integer values to the smallest bitwidth they can be legally
1526 /// represented as. The vector equivalents of these values should be truncated
1527 /// to this type.
1528 MapVector<Instruction *, uint64_t> MinBWs;
1529
1530 /// A type representing the costs for instructions if they were to be
1531 /// scalarized rather than vectorized. The entries are Instruction-Cost
1532 /// pairs.
1533 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1534
1535 /// A set containing all BasicBlocks that are known to present after
1536 /// vectorization as a predicated block.
1537 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1538 PredicatedBBsAfterVectorization;
1539
1540 /// Records whether it is allowed to have the original scalar loop execute at
1541 /// least once. This may be needed as a fallback loop in case runtime
1542 /// aliasing/dependence checks fail, or to handle the tail/remainder
1543 /// iterations when the trip count is unknown or doesn't divide by the VF,
1544 /// or as a peel-loop to handle gaps in interleave-groups.
1545 /// Under optsize and when the trip count is very small we don't allow any
1546 /// iterations to execute in the scalar loop.
1547 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1548
1549 /// Control finally chosen tail folding style. The first element is used if
1550 /// the IV update may overflow, the second element - if it does not.
1551 std::optional<std::pair<TailFoldingStyle, TailFoldingStyle>>
1552 ChosenTailFoldingStyle;
1553
1554 /// true if scalable vectorization is supported and enabled.
1555 std::optional<bool> IsScalableVectorizationAllowed;
1556
1557 /// Maximum safe number of elements to be processed per vector iteration,
1558 /// which do not prevent store-load forwarding and are safe with regard to the
1559 /// memory dependencies. Required for EVL-based veectorization, where this
1560 /// value is used as the upper bound of the safe AVL.
1561 std::optional<unsigned> MaxSafeElements;
1562
1563 /// A map holding scalar costs for different vectorization factors. The
1564 /// presence of a cost for an instruction in the mapping indicates that the
1565 /// instruction will be scalarized when vectorizing with the associated
1566 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1567 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1568
1569 /// Holds the instructions known to be uniform after vectorization.
1570 /// The data is collected per VF.
1571 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1572
1573 /// Holds the instructions known to be scalar after vectorization.
1574 /// The data is collected per VF.
1575 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1576
1577 /// Holds the instructions (address computations) that are forced to be
1578 /// scalarized.
1579 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1580
1581 /// PHINodes of the reductions that should be expanded in-loop.
1582 SmallPtrSet<PHINode *, 4> InLoopReductions;
1583
1584 /// A Map of inloop reduction operations and their immediate chain operand.
1585 /// FIXME: This can be removed once reductions can be costed correctly in
1586 /// VPlan. This was added to allow quick lookup of the inloop operations.
1587 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1588
1589 /// Returns the expected difference in cost from scalarizing the expression
1590 /// feeding a predicated instruction \p PredInst. The instructions to
1591 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1592 /// non-negative return value implies the expression will be scalarized.
1593 /// Currently, only single-use chains are considered for scalarization.
1594 InstructionCost computePredInstDiscount(Instruction *PredInst,
1595 ScalarCostsTy &ScalarCosts,
1596 ElementCount VF);
1597
1598 /// Collect the instructions that are uniform after vectorization. An
1599 /// instruction is uniform if we represent it with a single scalar value in
1600 /// the vectorized loop corresponding to each vector iteration. Examples of
1601 /// uniform instructions include pointer operands of consecutive or
1602 /// interleaved memory accesses. Note that although uniformity implies an
1603 /// instruction will be scalar, the reverse is not true. In general, a
1604 /// scalarized instruction will be represented by VF scalar values in the
1605 /// vectorized loop, each corresponding to an iteration of the original
1606 /// scalar loop.
1607 void collectLoopUniforms(ElementCount VF);
1608
1609 /// Collect the instructions that are scalar after vectorization. An
1610 /// instruction is scalar if it is known to be uniform or will be scalarized
1611 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1612 /// to the list if they are used by a load/store instruction that is marked as
1613 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1614 /// VF values in the vectorized loop, each corresponding to an iteration of
1615 /// the original scalar loop.
1616 void collectLoopScalars(ElementCount VF);
1617
1618 /// Keeps cost model vectorization decision and cost for instructions.
1619 /// Right now it is used for memory instructions only.
1620 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1621 std::pair<InstWidening, InstructionCost>>;
1622
1623 DecisionList WideningDecisions;
1624
1625 using CallDecisionList =
1626 DenseMap<std::pair<CallInst *, ElementCount>, CallWideningDecision>;
1627
1628 CallDecisionList CallWideningDecisions;
1629
1630 /// Returns true if \p V is expected to be vectorized and it needs to be
1631 /// extracted.
1632 bool needsExtract(Value *V, ElementCount VF) const {
1634 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1635 TheLoop->isLoopInvariant(I) ||
1636 getWideningDecision(I, VF) == CM_Scalarize ||
1637 (isa<CallInst>(I) &&
1638 getCallWideningDecision(cast<CallInst>(I), VF).Kind == CM_Scalarize))
1639 return false;
1640
1641 // Assume we can vectorize V (and hence we need extraction) if the
1642 // scalars are not computed yet. This can happen, because it is called
1643 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1644 // the scalars are collected. That should be a safe assumption in most
1645 // cases, because we check if the operands have vectorizable types
1646 // beforehand in LoopVectorizationLegality.
1647 return !Scalars.contains(VF) || !isScalarAfterVectorization(I, VF);
1648 };
1649
1650 /// Returns a range containing only operands needing to be extracted.
1651 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1652 ElementCount VF) const {
1653
1654 SmallPtrSet<const Value *, 4> UniqueOperands;
1656 for (Value *Op : Ops) {
1657 if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second ||
1658 !needsExtract(Op, VF))
1659 continue;
1660 Res.push_back(Op);
1661 }
1662 return Res;
1663 }
1664
1665public:
1666 /// The loop that we evaluate.
1668
1669 /// Predicated scalar evolution analysis.
1671
1672 /// Loop Info analysis.
1674
1675 /// Vectorization legality.
1677
1678 /// Vector target information.
1680
1681 /// Target Library Info.
1683
1684 /// Demanded bits analysis.
1686
1687 /// Assumption cache.
1689
1690 /// Interface to emit optimization remarks.
1692
1694
1695 /// Loop Vectorize Hint.
1697
1698 /// The interleave access information contains groups of interleaved accesses
1699 /// with the same stride and close to each other.
1701
1702 /// Values to ignore in the cost model.
1704
1705 /// Values to ignore in the cost model when VF > 1.
1707
1708 /// All element types found in the loop.
1710
1711 /// The kind of cost that we are calculating
1713
1714 /// Whether this loop should be optimized for size based on function attribute
1715 /// or profile information.
1717
1718 /// The highest VF possible for this loop, without using MaxBandwidth.
1720};
1721} // end namespace llvm
1722
1723namespace {
1724/// Helper struct to manage generating runtime checks for vectorization.
1725///
1726/// The runtime checks are created up-front in temporary blocks to allow better
1727/// estimating the cost and un-linked from the existing IR. After deciding to
1728/// vectorize, the checks are moved back. If deciding not to vectorize, the
1729/// temporary blocks are completely removed.
1730class GeneratedRTChecks {
1731 /// Basic block which contains the generated SCEV checks, if any.
1732 BasicBlock *SCEVCheckBlock = nullptr;
1733
1734 /// The value representing the result of the generated SCEV checks. If it is
1735 /// nullptr no SCEV checks have been generated.
1736 Value *SCEVCheckCond = nullptr;
1737
1738 /// Basic block which contains the generated memory runtime checks, if any.
1739 BasicBlock *MemCheckBlock = nullptr;
1740
1741 /// The value representing the result of the generated memory runtime checks.
1742 /// If it is nullptr no memory runtime checks have been generated.
1743 Value *MemRuntimeCheckCond = nullptr;
1744
1745 DominatorTree *DT;
1746 LoopInfo *LI;
1748
1749 SCEVExpander SCEVExp;
1750 SCEVExpander MemCheckExp;
1751
1752 bool CostTooHigh = false;
1753
1754 Loop *OuterLoop = nullptr;
1755
1757
1758 /// The kind of cost that we are calculating
1760
1761public:
1762 GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
1765 : DT(DT), LI(LI), TTI(TTI),
1766 SCEVExp(*PSE.getSE(), DL, "scev.check", /*PreserveLCSSA=*/false),
1767 MemCheckExp(*PSE.getSE(), DL, "scev.check", /*PreserveLCSSA=*/false),
1768 PSE(PSE), CostKind(CostKind) {}
1769
1770 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1771 /// accurately estimate the cost of the runtime checks. The blocks are
1772 /// un-linked from the IR and are added back during vector code generation. If
1773 /// there is no vector code generation, the check blocks are removed
1774 /// completely.
1775 void create(Loop *L, const LoopAccessInfo &LAI,
1776 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC) {
1777
1778 // Hard cutoff to limit compile-time increase in case a very large number of
1779 // runtime checks needs to be generated.
1780 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1781 // profile info.
1782 CostTooHigh =
1784 if (CostTooHigh)
1785 return;
1786
1787 BasicBlock *LoopHeader = L->getHeader();
1788 BasicBlock *Preheader = L->getLoopPreheader();
1789
1790 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1791 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1792 // may be used by SCEVExpander. The blocks will be un-linked from their
1793 // predecessors and removed from LI & DT at the end of the function.
1794 if (!UnionPred.isAlwaysTrue()) {
1795 SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1796 nullptr, "vector.scevcheck");
1797
1798 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1799 &UnionPred, SCEVCheckBlock->getTerminator());
1800 if (isa<Constant>(SCEVCheckCond)) {
1801 // Clean up directly after expanding the predicate to a constant, to
1802 // avoid further expansions re-using anything left over from SCEVExp.
1803 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1804 SCEVCleaner.cleanup();
1805 }
1806 }
1807
1808 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1809 if (RtPtrChecking.Need) {
1810 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1811 MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
1812 "vector.memcheck");
1813
1814 auto DiffChecks = RtPtrChecking.getDiffChecks();
1815 if (DiffChecks) {
1816 Value *RuntimeVF = nullptr;
1817 MemRuntimeCheckCond = addDiffRuntimeChecks(
1818 MemCheckBlock->getTerminator(), *DiffChecks, MemCheckExp,
1819 [VF, &RuntimeVF](IRBuilderBase &B, unsigned Bits) {
1820 if (!RuntimeVF)
1821 RuntimeVF = getRuntimeVF(B, B.getIntNTy(Bits), VF);
1822 return RuntimeVF;
1823 },
1824 IC);
1825 } else {
1826 MemRuntimeCheckCond = addRuntimeChecks(
1827 MemCheckBlock->getTerminator(), L, RtPtrChecking.getChecks(),
1829 }
1830 assert(MemRuntimeCheckCond &&
1831 "no RT checks generated although RtPtrChecking "
1832 "claimed checks are required");
1833 }
1834
1835 SCEVExp.eraseDeadInstructions(SCEVCheckCond);
1836
1837 if (!MemCheckBlock && !SCEVCheckBlock)
1838 return;
1839
1840 // Unhook the temporary block with the checks, update various places
1841 // accordingly.
1842 if (SCEVCheckBlock)
1843 SCEVCheckBlock->replaceAllUsesWith(Preheader);
1844 if (MemCheckBlock)
1845 MemCheckBlock->replaceAllUsesWith(Preheader);
1846
1847 if (SCEVCheckBlock) {
1848 SCEVCheckBlock->getTerminator()->moveBefore(
1849 Preheader->getTerminator()->getIterator());
1850 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1851 UI->setDebugLoc(DebugLoc::getTemporary());
1852 Preheader->getTerminator()->eraseFromParent();
1853 }
1854 if (MemCheckBlock) {
1855 MemCheckBlock->getTerminator()->moveBefore(
1856 Preheader->getTerminator()->getIterator());
1857 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1858 UI->setDebugLoc(DebugLoc::getTemporary());
1859 Preheader->getTerminator()->eraseFromParent();
1860 }
1861
1862 DT->changeImmediateDominator(LoopHeader, Preheader);
1863 if (MemCheckBlock) {
1864 DT->eraseNode(MemCheckBlock);
1865 LI->removeBlock(MemCheckBlock);
1866 }
1867 if (SCEVCheckBlock) {
1868 DT->eraseNode(SCEVCheckBlock);
1869 LI->removeBlock(SCEVCheckBlock);
1870 }
1871
1872 // Outer loop is used as part of the later cost calculations.
1873 OuterLoop = L->getParentLoop();
1874 }
1875
1877 if (SCEVCheckBlock || MemCheckBlock)
1878 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1879
1880 if (CostTooHigh) {
1882 Cost.setInvalid();
1883 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1884 return Cost;
1885 }
1886
1887 InstructionCost RTCheckCost = 0;
1888 if (SCEVCheckBlock)
1889 for (Instruction &I : *SCEVCheckBlock) {
1890 if (SCEVCheckBlock->getTerminator() == &I)
1891 continue;
1893 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1894 RTCheckCost += C;
1895 }
1896 if (MemCheckBlock) {
1897 InstructionCost MemCheckCost = 0;
1898 for (Instruction &I : *MemCheckBlock) {
1899 if (MemCheckBlock->getTerminator() == &I)
1900 continue;
1902 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1903 MemCheckCost += C;
1904 }
1905
1906 // If the runtime memory checks are being created inside an outer loop
1907 // we should find out if these checks are outer loop invariant. If so,
1908 // the checks will likely be hoisted out and so the effective cost will
1909 // reduce according to the outer loop trip count.
1910 if (OuterLoop) {
1911 ScalarEvolution *SE = MemCheckExp.getSE();
1912 // TODO: If profitable, we could refine this further by analysing every
1913 // individual memory check, since there could be a mixture of loop
1914 // variant and invariant checks that mean the final condition is
1915 // variant.
1916 const SCEV *Cond = SE->getSCEV(MemRuntimeCheckCond);
1917 if (SE->isLoopInvariant(Cond, OuterLoop)) {
1918 // It seems reasonable to assume that we can reduce the effective
1919 // cost of the checks even when we know nothing about the trip
1920 // count. Assume that the outer loop executes at least twice.
1921 unsigned BestTripCount = 2;
1922
1923 // Get the best known TC estimate.
1924 if (auto EstimatedTC = getSmallBestKnownTC(
1925 PSE, OuterLoop, /* CanUseConstantMax = */ false))
1926 if (EstimatedTC->isFixed())
1927 BestTripCount = EstimatedTC->getFixedValue();
1928
1929 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1930
1931 // Let's ensure the cost is always at least 1.
1932 NewMemCheckCost = std::max(NewMemCheckCost.getValue(),
1933 (InstructionCost::CostType)1);
1934
1935 if (BestTripCount > 1)
1937 << "We expect runtime memory checks to be hoisted "
1938 << "out of the outer loop. Cost reduced from "
1939 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1940
1941 MemCheckCost = NewMemCheckCost;
1942 }
1943 }
1944
1945 RTCheckCost += MemCheckCost;
1946 }
1947
1948 if (SCEVCheckBlock || MemCheckBlock)
1949 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1950 << "\n");
1951
1952 return RTCheckCost;
1953 }
1954
1955 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1956 /// unused.
1957 ~GeneratedRTChecks() {
1958 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1959 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1960 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(SCEVCheckBlock);
1961 bool MemChecksUsed = !MemCheckBlock || !pred_empty(MemCheckBlock);
1962 if (SCEVChecksUsed)
1963 SCEVCleaner.markResultUsed();
1964
1965 if (MemChecksUsed) {
1966 MemCheckCleaner.markResultUsed();
1967 } else {
1968 auto &SE = *MemCheckExp.getSE();
1969 // Memory runtime check generation creates compares that use expanded
1970 // values. Remove them before running the SCEVExpanderCleaners.
1971 for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
1972 if (MemCheckExp.isInsertedInstruction(&I))
1973 continue;
1974 SE.forgetValue(&I);
1975 I.eraseFromParent();
1976 }
1977 }
1978 MemCheckCleaner.cleanup();
1979 SCEVCleaner.cleanup();
1980
1981 if (!SCEVChecksUsed)
1982 SCEVCheckBlock->eraseFromParent();
1983 if (!MemChecksUsed)
1984 MemCheckBlock->eraseFromParent();
1985 }
1986
1987 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1988 /// outside VPlan.
1989 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1990 using namespace llvm::PatternMatch;
1991 if (!SCEVCheckCond || match(SCEVCheckCond, m_ZeroInt()))
1992 return {nullptr, nullptr};
1993
1994 return {SCEVCheckCond, SCEVCheckBlock};
1995 }
1996
1997 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1998 /// outside VPlan.
1999 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
2000 using namespace llvm::PatternMatch;
2001 if (MemRuntimeCheckCond && match(MemRuntimeCheckCond, m_ZeroInt()))
2002 return {nullptr, nullptr};
2003 return {MemRuntimeCheckCond, MemCheckBlock};
2004 }
2005
2006 /// Return true if any runtime checks have been added
2007 bool hasChecks() const {
2008 return getSCEVChecks().first || getMemRuntimeChecks().first;
2009 }
2010};
2011} // namespace
2012
2018
2023
2024// Return true if \p OuterLp is an outer loop annotated with hints for explicit
2025// vectorization. The loop needs to be annotated with #pragma omp simd
2026// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
2027// vector length information is not provided, vectorization is not considered
2028// explicit. Interleave hints are not allowed either. These limitations will be
2029// relaxed in the future.
2030// Please, note that we are currently forced to abuse the pragma 'clang
2031// vectorize' semantics. This pragma provides *auto-vectorization hints*
2032// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
2033// provides *explicit vectorization hints* (LV can bypass legal checks and
2034// assume that vectorization is legal). However, both hints are implemented
2035// using the same metadata (llvm.loop.vectorize, processed by
2036// LoopVectorizeHints). This will be fixed in the future when the native IR
2037// representation for pragma 'omp simd' is introduced.
2038static bool isExplicitVecOuterLoop(Loop *OuterLp,
2040 assert(!OuterLp->isInnermost() && "This is not an outer loop");
2041 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
2042
2043 // Only outer loops with an explicit vectorization hint are supported.
2044 // Unannotated outer loops are ignored.
2046 return false;
2047
2048 Function *Fn = OuterLp->getHeader()->getParent();
2049 if (!Hints.allowVectorization(Fn, OuterLp,
2050 true /*VectorizeOnlyWhenForced*/)) {
2051 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
2052 return false;
2053 }
2054
2055 if (Hints.getInterleave() > 1) {
2056 // TODO: Interleave support is future work.
2057 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
2058 "outer loops.\n");
2059 Hints.emitRemarkWithHints();
2060 return false;
2061 }
2062
2063 return true;
2064}
2065
2069 // Collect inner loops and outer loops without irreducible control flow. For
2070 // now, only collect outer loops that have explicit vectorization hints. If we
2071 // are stress testing the VPlan H-CFG construction, we collect the outermost
2072 // loop of every loop nest.
2073 if (L.isInnermost() || VPlanBuildStressTest ||
2075 LoopBlocksRPO RPOT(&L);
2076 RPOT.perform(LI);
2078 V.push_back(&L);
2079 // TODO: Collect inner loops inside marked outer loops in case
2080 // vectorization fails for the outer loop. Do not invoke
2081 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
2082 // already known to be reducible. We can use an inherited attribute for
2083 // that.
2084 return;
2085 }
2086 }
2087 for (Loop *InnerL : L)
2088 collectSupportedLoops(*InnerL, LI, ORE, V);
2089}
2090
2091//===----------------------------------------------------------------------===//
2092// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2093// LoopVectorizationCostModel and LoopVectorizationPlanner.
2094//===----------------------------------------------------------------------===//
2095
2096/// Compute the transformed value of Index at offset StartValue using step
2097/// StepValue.
2098/// For integer induction, returns StartValue + Index * StepValue.
2099/// For pointer induction, returns StartValue[Index * StepValue].
2100/// FIXME: The newly created binary instructions should contain nsw/nuw
2101/// flags, which can be found from the original scalar operations.
2102static Value *
2104 Value *Step,
2106 const BinaryOperator *InductionBinOp) {
2107 using namespace llvm::PatternMatch;
2108 Type *StepTy = Step->getType();
2109 Value *CastedIndex = StepTy->isIntegerTy()
2110 ? B.CreateSExtOrTrunc(Index, StepTy)
2111 : B.CreateCast(Instruction::SIToFP, Index, StepTy);
2112 if (CastedIndex != Index) {
2113 CastedIndex->setName(CastedIndex->getName() + ".cast");
2114 Index = CastedIndex;
2115 }
2116
2117 // Note: the IR at this point is broken. We cannot use SE to create any new
2118 // SCEV and then expand it, hoping that SCEV's simplification will give us
2119 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
2120 // lead to various SCEV crashes. So all we can do is to use builder and rely
2121 // on InstCombine for future simplifications. Here we handle some trivial
2122 // cases only.
2123 auto CreateAdd = [&B](Value *X, Value *Y) {
2124 assert(X->getType() == Y->getType() && "Types don't match!");
2125 if (match(X, m_ZeroInt()))
2126 return Y;
2127 if (match(Y, m_ZeroInt()))
2128 return X;
2129 return B.CreateAdd(X, Y);
2130 };
2131
2132 // We allow X to be a vector type, in which case Y will potentially be
2133 // splatted into a vector with the same element count.
2134 auto CreateMul = [&B](Value *X, Value *Y) {
2135 assert(X->getType()->getScalarType() == Y->getType() &&
2136 "Types don't match!");
2137 if (match(X, m_One()))
2138 return Y;
2139 if (match(Y, m_One()))
2140 return X;
2141 VectorType *XVTy = dyn_cast<VectorType>(X->getType());
2142 if (XVTy && !isa<VectorType>(Y->getType()))
2143 Y = B.CreateVectorSplat(XVTy->getElementCount(), Y);
2144 return B.CreateMul(X, Y);
2145 };
2146
2147 switch (InductionKind) {
2149 assert(!isa<VectorType>(Index->getType()) &&
2150 "Vector indices not supported for integer inductions yet");
2151 assert(Index->getType() == StartValue->getType() &&
2152 "Index type does not match StartValue type");
2153 if (isa<ConstantInt>(Step) && cast<ConstantInt>(Step)->isMinusOne())
2154 return B.CreateSub(StartValue, Index);
2155 auto *Offset = CreateMul(Index, Step);
2156 return CreateAdd(StartValue, Offset);
2157 }
2159 return B.CreatePtrAdd(StartValue, CreateMul(Index, Step));
2161 assert(!isa<VectorType>(Index->getType()) &&
2162 "Vector indices not supported for FP inductions yet");
2163 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value");
2164 assert(InductionBinOp &&
2165 (InductionBinOp->getOpcode() == Instruction::FAdd ||
2166 InductionBinOp->getOpcode() == Instruction::FSub) &&
2167 "Original bin op should be defined for FP induction");
2168
2169 Value *MulExp = B.CreateFMul(Step, Index);
2170 return B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
2171 "induction");
2172 }
2174 return nullptr;
2175 }
2176 llvm_unreachable("invalid enum");
2177}
2178
2179static std::optional<unsigned> getMaxVScale(const Function &F,
2180 const TargetTransformInfo &TTI) {
2181 if (std::optional<unsigned> MaxVScale = TTI.getMaxVScale())
2182 return MaxVScale;
2183
2184 if (F.hasFnAttribute(Attribute::VScaleRange))
2185 return F.getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
2186
2187 return std::nullopt;
2188}
2189
2190/// For the given VF and UF and maximum trip count computed for the loop, return
2191/// whether the induction variable might overflow in the vectorized loop. If not,
2192/// then we know a runtime overflow check always evaluates to false and can be
2193/// removed.
2195 const LoopVectorizationCostModel *Cost,
2196 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
2197 // Always be conservative if we don't know the exact unroll factor.
2198 unsigned MaxUF = UF ? *UF : Cost->TTI.getMaxInterleaveFactor(VF);
2199
2200 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
2201 APInt MaxUIntTripCount = IdxTy->getMask();
2202
2203 // We know the runtime overflow check is known false iff the (max) trip-count
2204 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
2205 // the vector loop induction variable.
2206 if (unsigned TC = Cost->PSE.getSmallConstantMaxTripCount()) {
2207 uint64_t MaxVF = VF.getKnownMinValue();
2208 if (VF.isScalable()) {
2209 std::optional<unsigned> MaxVScale =
2210 getMaxVScale(*Cost->TheFunction, Cost->TTI);
2211 if (!MaxVScale)
2212 return false;
2213 MaxVF *= *MaxVScale;
2214 }
2215
2216 return (MaxUIntTripCount - TC).ugt(MaxVF * MaxUF);
2217 }
2218
2219 return false;
2220}
2221
2222// Return whether we allow using masked interleave-groups (for dealing with
2223// strided loads/stores that reside in predicated blocks, or for dealing
2224// with gaps).
2226 // If an override option has been passed in for interleaved accesses, use it.
2227 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2229
2230 return TTI.enableMaskedInterleavedAccessVectorization();
2231}
2232
2234 BasicBlock *CheckIRBB) {
2235 // Note: The block with the minimum trip-count check is already connected
2236 // during earlier VPlan construction.
2237 VPBlockBase *ScalarPH = Plan.getScalarPreheader();
2238 VPBlockBase *PreVectorPH = VectorPHVPBB->getSinglePredecessor();
2239 assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
2240 assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
2241 VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
2242 VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPBB, CheckVPIRBB);
2243 PreVectorPH = CheckVPIRBB;
2244 VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
2245 PreVectorPH->swapSuccessors();
2246
2247 // We just connected a new block to the scalar preheader. Update all
2248 // VPPhis by adding an incoming value for it, replicating the last value.
2249 unsigned NumPredecessors = ScalarPH->getNumPredecessors();
2250 for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
2251 assert(isa<VPPhi>(&R) && "Phi expected to be VPPhi");
2252 assert(cast<VPPhi>(&R)->getNumIncoming() == NumPredecessors - 1 &&
2253 "must have incoming values for all operands");
2254 R.addOperand(R.getOperand(NumPredecessors - 2));
2255 }
2256}
2257
2259 BasicBlock *VectorPH, ElementCount VF, unsigned UF) const {
2260 // Generate code to check if the loop's trip count is less than VF * UF, or
2261 // equal to it in case a scalar epilogue is required; this implies that the
2262 // vector trip count is zero. This check also covers the case where adding one
2263 // to the backedge-taken count overflowed leading to an incorrect trip count
2264 // of zero. In this case we will also jump to the scalar loop.
2265 auto P = Cost->requiresScalarEpilogue(VF.isVector()) ? ICmpInst::ICMP_ULE
2267
2268 // Reuse existing vector loop preheader for TC checks.
2269 // Note that new preheader block is generated for vector loop.
2270 BasicBlock *const TCCheckBlock = VectorPH;
2272 TCCheckBlock->getContext(),
2273 InstSimplifyFolder(TCCheckBlock->getDataLayout()));
2274 Builder.SetInsertPoint(TCCheckBlock->getTerminator());
2275
2276 // If tail is to be folded, vector loop takes care of all iterations.
2278 Type *CountTy = Count->getType();
2279 Value *CheckMinIters = Builder.getFalse();
2280 auto CreateStep = [&]() -> Value * {
2281 // Create step with max(MinProTripCount, UF * VF).
2282 if (UF * VF.getKnownMinValue() >= MinProfitableTripCount.getKnownMinValue())
2283 return createStepForVF(Builder, CountTy, VF, UF);
2284
2285 Value *MinProfTC =
2286 Builder.CreateElementCount(CountTy, MinProfitableTripCount);
2287 if (!VF.isScalable())
2288 return MinProfTC;
2289 return Builder.CreateBinaryIntrinsic(
2290 Intrinsic::umax, MinProfTC, createStepForVF(Builder, CountTy, VF, UF));
2291 };
2292
2293 TailFoldingStyle Style = Cost->getTailFoldingStyle();
2294 if (Style == TailFoldingStyle::None) {
2295 Value *Step = CreateStep();
2296 ScalarEvolution &SE = *PSE.getSE();
2297 // TODO: Emit unconditional branch to vector preheader instead of
2298 // conditional branch with known condition.
2299 const SCEV *TripCountSCEV = SE.applyLoopGuards(SE.getSCEV(Count), OrigLoop);
2300 // Check if the trip count is < the step.
2301 if (SE.isKnownPredicate(P, TripCountSCEV, SE.getSCEV(Step))) {
2302 // TODO: Ensure step is at most the trip count when determining max VF and
2303 // UF, w/o tail folding.
2304 CheckMinIters = Builder.getTrue();
2306 TripCountSCEV, SE.getSCEV(Step))) {
2307 // Generate the minimum iteration check only if we cannot prove the
2308 // check is known to be true, or known to be false.
2309 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
2310 } // else step known to be < trip count, use CheckMinIters preset to false.
2311 } else if (VF.isScalable() && !TTI->isVScaleKnownToBeAPowerOfTwo() &&
2314 // vscale is not necessarily a power-of-2, which means we cannot guarantee
2315 // an overflow to zero when updating induction variables and so an
2316 // additional overflow check is required before entering the vector loop.
2317
2318 // Get the maximum unsigned value for the type.
2319 Value *MaxUIntTripCount =
2320 ConstantInt::get(CountTy, cast<IntegerType>(CountTy)->getMask());
2321 Value *LHS = Builder.CreateSub(MaxUIntTripCount, Count);
2322
2323 // Don't execute the vector loop if (UMax - n) < (VF * UF).
2324 CheckMinIters = Builder.CreateICmp(ICmpInst::ICMP_ULT, LHS, CreateStep());
2325 }
2326 return CheckMinIters;
2327}
2328
2329/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
2330/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
2331/// predecessors and successors of VPBB, if any, are rewired to the new
2332/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
2334 BasicBlock *IRBB,
2335 VPlan *Plan = nullptr) {
2336 if (!Plan)
2337 Plan = VPBB->getPlan();
2338 VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
2339 auto IP = IRVPBB->begin();
2340 for (auto &R : make_early_inc_range(VPBB->phis()))
2341 R.moveBefore(*IRVPBB, IP);
2342
2343 for (auto &R :
2345 R.moveBefore(*IRVPBB, IRVPBB->end());
2346
2347 VPBlockUtils::reassociateBlocks(VPBB, IRVPBB);
2348 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
2349 return IRVPBB;
2350}
2351
2353 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
2354 assert(VectorPH && "Invalid loop structure");
2355 assert((OrigLoop->getUniqueLatchExitBlock() ||
2356 Cost->requiresScalarEpilogue(VF.isVector())) &&
2357 "loops not exiting via the latch without required epilogue?");
2358
2359 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
2360 // wrapping the newly created scalar preheader here at the moment, because the
2361 // Plan's scalar preheader may be unreachable at this point. Instead it is
2362 // replaced in executePlan.
2363 return SplitBlock(VectorPH, VectorPH->getTerminator(), DT, LI, nullptr,
2364 Twine(Prefix) + "scalar.ph");
2365}
2366
2367/// Return the expanded step for \p ID using \p ExpandedSCEVs to look up SCEV
2368/// expansion results.
2370 const SCEV2ValueTy &ExpandedSCEVs) {
2371 const SCEV *Step = ID.getStep();
2372 if (auto *C = dyn_cast<SCEVConstant>(Step))
2373 return C->getValue();
2374 if (auto *U = dyn_cast<SCEVUnknown>(Step))
2375 return U->getValue();
2376 Value *V = ExpandedSCEVs.lookup(Step);
2377 assert(V && "SCEV must be expanded at this point");
2378 return V;
2379}
2380
2381/// Knowing that loop \p L executes a single vector iteration, add instructions
2382/// that will get simplified and thus should not have any cost to \p
2383/// InstsToIgnore.
2386 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
2387 auto *Cmp = L->getLatchCmpInst();
2388 if (Cmp)
2389 InstsToIgnore.insert(Cmp);
2390 for (const auto &KV : IL) {
2391 // Extract the key by hand so that it can be used in the lambda below. Note
2392 // that captured structured bindings are a C++20 extension.
2393 const PHINode *IV = KV.first;
2394
2395 // Get next iteration value of the induction variable.
2396 Instruction *IVInst =
2397 cast<Instruction>(IV->getIncomingValueForBlock(L->getLoopLatch()));
2398 if (all_of(IVInst->users(),
2399 [&](const User *U) { return U == IV || U == Cmp; }))
2400 InstsToIgnore.insert(IVInst);
2401 }
2402}
2403
2405 // Create a new IR basic block for the scalar preheader.
2406 BasicBlock *ScalarPH = createScalarPreheader("");
2407 return ScalarPH->getSinglePredecessor();
2408}
2409
2410namespace {
2411
2412struct CSEDenseMapInfo {
2413 static bool canHandle(const Instruction *I) {
2416 }
2417
2418 static inline Instruction *getEmptyKey() {
2420 }
2421
2422 static inline Instruction *getTombstoneKey() {
2423 return DenseMapInfo<Instruction *>::getTombstoneKey();
2424 }
2425
2426 static unsigned getHashValue(const Instruction *I) {
2427 assert(canHandle(I) && "Unknown instruction!");
2428 return hash_combine(I->getOpcode(),
2429 hash_combine_range(I->operand_values()));
2430 }
2431
2432 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
2433 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
2434 LHS == getTombstoneKey() || RHS == getTombstoneKey())
2435 return LHS == RHS;
2436 return LHS->isIdenticalTo(RHS);
2437 }
2438};
2439
2440} // end anonymous namespace
2441
2442/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
2443/// removal, in favor of the VPlan-based one.
2444static void legacyCSE(BasicBlock *BB) {
2445 // Perform simple cse.
2447 for (Instruction &In : llvm::make_early_inc_range(*BB)) {
2448 if (!CSEDenseMapInfo::canHandle(&In))
2449 continue;
2450
2451 // Check if we can replace this instruction with any of the
2452 // visited instructions.
2453 if (Instruction *V = CSEMap.lookup(&In)) {
2454 In.replaceAllUsesWith(V);
2455 In.eraseFromParent();
2456 continue;
2457 }
2458
2459 CSEMap[&In] = &In;
2460 }
2461}
2462
2463/// This function attempts to return a value that represents the ElementCount
2464/// at runtime. For fixed-width VFs we know this precisely at compile
2465/// time, but for scalable VFs we calculate it based on an estimate of the
2466/// vscale value.
2468 std::optional<unsigned> VScale) {
2469 unsigned EstimatedVF = VF.getKnownMinValue();
2470 if (VF.isScalable())
2471 if (VScale)
2472 EstimatedVF *= *VScale;
2473 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
2474 return EstimatedVF;
2475}
2476
2479 ElementCount VF) const {
2480 // We only need to calculate a cost if the VF is scalar; for actual vectors
2481 // we should already have a pre-calculated cost at each VF.
2482 if (!VF.isScalar())
2483 return getCallWideningDecision(CI, VF).Cost;
2484
2485 Type *RetTy = CI->getType();
2487 if (auto RedCost = getReductionPatternCost(CI, VF, RetTy))
2488 return *RedCost;
2489
2491 for (auto &ArgOp : CI->args())
2492 Tys.push_back(ArgOp->getType());
2493
2494 InstructionCost ScalarCallCost =
2495 TTI.getCallInstrCost(CI->getCalledFunction(), RetTy, Tys, CostKind);
2496
2497 // If this is an intrinsic we may have a lower cost for it.
2500 return std::min(ScalarCallCost, IntrinsicCost);
2501 }
2502 return ScalarCallCost;
2503}
2504
2506 if (VF.isScalar() || !canVectorizeTy(Ty))
2507 return Ty;
2508 return toVectorizedTy(Ty, VF);
2509}
2510
2513 ElementCount VF) const {
2515 assert(ID && "Expected intrinsic call!");
2516 Type *RetTy = maybeVectorizeType(CI->getType(), VF);
2517 FastMathFlags FMF;
2518 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2519 FMF = FPMO->getFastMathFlags();
2520
2523 SmallVector<Type *> ParamTys;
2524 std::transform(FTy->param_begin(), FTy->param_end(),
2525 std::back_inserter(ParamTys),
2526 [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
2527
2528 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2531 return TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
2532}
2533
2535 // Fix widened non-induction PHIs by setting up the PHI operands.
2536 fixNonInductionPHIs(State);
2537
2538 // Don't apply optimizations below when no (vector) loop remains, as they all
2539 // require one at the moment.
2540 VPBasicBlock *HeaderVPBB =
2541 vputils::getFirstLoopHeader(*State.Plan, State.VPDT);
2542 if (!HeaderVPBB)
2543 return;
2544
2545 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2546
2547 // Remove redundant induction instructions.
2548 legacyCSE(HeaderBB);
2549}
2550
2552 auto Iter = vp_depth_first_shallow(Plan.getEntry());
2554 for (VPRecipeBase &P : VPBB->phis()) {
2556 if (!VPPhi)
2557 continue;
2558 PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
2559 // Make sure the builder has a valid insert point.
2560 Builder.SetInsertPoint(NewPhi);
2561 for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
2562 NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
2563 }
2564 }
2565}
2566
2567void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
2568 // We should not collect Scalars more than once per VF. Right now, this
2569 // function is called from collectUniformsAndScalars(), which already does
2570 // this check. Collecting Scalars for VF=1 does not make any sense.
2571 assert(VF.isVector() && !Scalars.contains(VF) &&
2572 "This function should not be visited twice for the same VF");
2573
2574 // This avoids any chances of creating a REPLICATE recipe during planning
2575 // since that would result in generation of scalarized code during execution,
2576 // which is not supported for scalable vectors.
2577 if (VF.isScalable()) {
2578 Scalars[VF].insert_range(Uniforms[VF]);
2579 return;
2580 }
2581
2583
2584 // These sets are used to seed the analysis with pointers used by memory
2585 // accesses that will remain scalar.
2587 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
2588 auto *Latch = TheLoop->getLoopLatch();
2589
2590 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
2591 // The pointer operands of loads and stores will be scalar as long as the
2592 // memory access is not a gather or scatter operation. The value operand of a
2593 // store will remain scalar if the store is scalarized.
2594 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
2595 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
2596 assert(WideningDecision != CM_Unknown &&
2597 "Widening decision should be ready at this moment");
2598 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
2599 if (Ptr == Store->getValueOperand())
2600 return WideningDecision == CM_Scalarize;
2601 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
2602 "Ptr is neither a value or pointer operand");
2603 return WideningDecision != CM_GatherScatter;
2604 };
2605
2606 // A helper that returns true if the given value is a getelementptr
2607 // instruction contained in the loop.
2608 auto IsLoopVaryingGEP = [&](Value *V) {
2609 return isa<GetElementPtrInst>(V) && !TheLoop->isLoopInvariant(V);
2610 };
2611
2612 // A helper that evaluates a memory access's use of a pointer. If the use will
2613 // be a scalar use and the pointer is only used by memory accesses, we place
2614 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
2615 // PossibleNonScalarPtrs.
2616 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
2617 // We only care about bitcast and getelementptr instructions contained in
2618 // the loop.
2619 if (!IsLoopVaryingGEP(Ptr))
2620 return;
2621
2622 // If the pointer has already been identified as scalar (e.g., if it was
2623 // also identified as uniform), there's nothing to do.
2624 auto *I = cast<Instruction>(Ptr);
2625 if (Worklist.count(I))
2626 return;
2627
2628 // If the use of the pointer will be a scalar use, and all users of the
2629 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
2630 // place the pointer in PossibleNonScalarPtrs.
2631 if (IsScalarUse(MemAccess, Ptr) &&
2633 ScalarPtrs.insert(I);
2634 else
2635 PossibleNonScalarPtrs.insert(I);
2636 };
2637
2638 // We seed the scalars analysis with three classes of instructions: (1)
2639 // instructions marked uniform-after-vectorization and (2) bitcast,
2640 // getelementptr and (pointer) phi instructions used by memory accesses
2641 // requiring a scalar use.
2642 //
2643 // (1) Add to the worklist all instructions that have been identified as
2644 // uniform-after-vectorization.
2645 Worklist.insert_range(Uniforms[VF]);
2646
2647 // (2) Add to the worklist all bitcast and getelementptr instructions used by
2648 // memory accesses requiring a scalar use. The pointer operands of loads and
2649 // stores will be scalar unless the operation is a gather or scatter.
2650 // The value operand of a store will remain scalar if the store is scalarized.
2651 for (auto *BB : TheLoop->blocks())
2652 for (auto &I : *BB) {
2653 if (auto *Load = dyn_cast<LoadInst>(&I)) {
2654 EvaluatePtrUse(Load, Load->getPointerOperand());
2655 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
2656 EvaluatePtrUse(Store, Store->getPointerOperand());
2657 EvaluatePtrUse(Store, Store->getValueOperand());
2658 }
2659 }
2660 for (auto *I : ScalarPtrs)
2661 if (!PossibleNonScalarPtrs.count(I)) {
2662 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
2663 Worklist.insert(I);
2664 }
2665
2666 // Insert the forced scalars.
2667 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
2668 // induction variable when the PHI user is scalarized.
2669 auto ForcedScalar = ForcedScalars.find(VF);
2670 if (ForcedScalar != ForcedScalars.end())
2671 for (auto *I : ForcedScalar->second) {
2672 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
2673 Worklist.insert(I);
2674 }
2675
2676 // Expand the worklist by looking through any bitcasts and getelementptr
2677 // instructions we've already identified as scalar. This is similar to the
2678 // expansion step in collectLoopUniforms(); however, here we're only
2679 // expanding to include additional bitcasts and getelementptr instructions.
2680 unsigned Idx = 0;
2681 while (Idx != Worklist.size()) {
2682 Instruction *Dst = Worklist[Idx++];
2683 if (!IsLoopVaryingGEP(Dst->getOperand(0)))
2684 continue;
2685 auto *Src = cast<Instruction>(Dst->getOperand(0));
2686 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
2687 auto *J = cast<Instruction>(U);
2688 return !TheLoop->contains(J) || Worklist.count(J) ||
2689 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
2690 IsScalarUse(J, Src));
2691 })) {
2692 Worklist.insert(Src);
2693 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2694 }
2695 }
2696
2697 // An induction variable will remain scalar if all users of the induction
2698 // variable and induction variable update remain scalar.
2699 for (const auto &Induction : Legal->getInductionVars()) {
2700 auto *Ind = Induction.first;
2701 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
2702
2703 // If tail-folding is applied, the primary induction variable will be used
2704 // to feed a vector compare.
2705 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2706 continue;
2707
2708 // Returns true if \p Indvar is a pointer induction that is used directly by
2709 // load/store instruction \p I.
2710 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2711 Instruction *I) {
2712 return Induction.second.getKind() ==
2715 Indvar == getLoadStorePointerOperand(I) && IsScalarUse(I, Indvar);
2716 };
2717
2718 // Determine if all users of the induction variable are scalar after
2719 // vectorization.
2720 bool ScalarInd = all_of(Ind->users(), [&](User *U) -> bool {
2721 auto *I = cast<Instruction>(U);
2722 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
2723 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2724 });
2725 if (!ScalarInd)
2726 continue;
2727
2728 // If the induction variable update is a fixed-order recurrence, neither the
2729 // induction variable or its update should be marked scalar after
2730 // vectorization.
2731 auto *IndUpdatePhi = dyn_cast<PHINode>(IndUpdate);
2732 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(IndUpdatePhi))
2733 continue;
2734
2735 // Determine if all users of the induction variable update instruction are
2736 // scalar after vectorization.
2737 bool ScalarIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
2738 auto *I = cast<Instruction>(U);
2739 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
2740 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2741 });
2742 if (!ScalarIndUpdate)
2743 continue;
2744
2745 // The induction variable and its update instruction will remain scalar.
2746 Worklist.insert(Ind);
2747 Worklist.insert(IndUpdate);
2748 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2749 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2750 << "\n");
2751 }
2752
2753 Scalars[VF].insert_range(Worklist);
2754}
2755
2757 Instruction *I, ElementCount VF) const {
2758 if (!isPredicatedInst(I))
2759 return false;
2760
2761 // Do we have a non-scalar lowering for this predicated
2762 // instruction? No - it is scalar with predication.
2763 switch(I->getOpcode()) {
2764 default:
2765 return true;
2766 case Instruction::Call:
2767 if (VF.isScalar())
2768 return true;
2770 case Instruction::Load:
2771 case Instruction::Store: {
2773 auto *Ty = getLoadStoreType(I);
2774 unsigned AS = getLoadStoreAddressSpace(I);
2775 Type *VTy = Ty;
2776 if (VF.isVector())
2777 VTy = VectorType::get(Ty, VF);
2778 const Align Alignment = getLoadStoreAlignment(I);
2779 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment, AS) ||
2780 TTI.isLegalMaskedGather(VTy, Alignment))
2781 : !(isLegalMaskedStore(Ty, Ptr, Alignment, AS) ||
2782 TTI.isLegalMaskedScatter(VTy, Alignment));
2783 }
2784 case Instruction::UDiv:
2785 case Instruction::SDiv:
2786 case Instruction::SRem:
2787 case Instruction::URem: {
2788 // We have the option to use the safe-divisor idiom to avoid predication.
2789 // The cost based decision here will always select safe-divisor for
2790 // scalable vectors as scalarization isn't legal.
2791 const auto [ScalarCost, SafeDivisorCost] = getDivRemSpeculationCost(I, VF);
2792 return isDivRemScalarWithPredication(ScalarCost, SafeDivisorCost);
2793 }
2794 }
2795}
2796
2797// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2799 // TODO: We can use the loop-preheader as context point here and get
2800 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2802 (isa<LoadInst, StoreInst, CallInst>(I) && !Legal->isMaskRequired(I)) ||
2804 return false;
2805
2806 // If the instruction was executed conditionally in the original scalar loop,
2807 // predication is needed with a mask whose lanes are all possibly inactive.
2808 if (Legal->blockNeedsPredication(I->getParent()))
2809 return true;
2810
2811 // If we're not folding the tail by masking, predication is unnecessary.
2812 if (!foldTailByMasking())
2813 return false;
2814
2815 // All that remain are instructions with side-effects originally executed in
2816 // the loop unconditionally, but now execute under a tail-fold mask (only)
2817 // having at least one active lane (the first). If the side-effects of the
2818 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2819 // - it will cause the same side-effects as when masked.
2820 switch(I->getOpcode()) {
2821 default:
2823 "instruction should have been considered by earlier checks");
2824 case Instruction::Call:
2825 // Side-effects of a Call are assumed to be non-invariant, needing a
2826 // (fold-tail) mask.
2827 assert(Legal->isMaskRequired(I) &&
2828 "should have returned earlier for calls not needing a mask");
2829 return true;
2830 case Instruction::Load:
2831 // If the address is loop invariant no predication is needed.
2832 return !Legal->isInvariant(getLoadStorePointerOperand(I));
2833 case Instruction::Store: {
2834 // For stores, we need to prove both speculation safety (which follows from
2835 // the same argument as loads), but also must prove the value being stored
2836 // is correct. The easiest form of the later is to require that all values
2837 // stored are the same.
2838 return !(Legal->isInvariant(getLoadStorePointerOperand(I)) &&
2839 TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand()));
2840 }
2841 case Instruction::UDiv:
2842 case Instruction::SDiv:
2843 case Instruction::SRem:
2844 case Instruction::URem:
2845 // If the divisor is loop-invariant no predication is needed.
2846 return !Legal->isInvariant(I->getOperand(1));
2847 }
2848}
2849
2850std::pair<InstructionCost, InstructionCost>
2852 ElementCount VF) const {
2853 assert(I->getOpcode() == Instruction::UDiv ||
2854 I->getOpcode() == Instruction::SDiv ||
2855 I->getOpcode() == Instruction::SRem ||
2856 I->getOpcode() == Instruction::URem);
2858
2859 // Scalarization isn't legal for scalable vector types
2860 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2861 if (!VF.isScalable()) {
2862 // Get the scalarization cost and scale this amount by the probability of
2863 // executing the predicated block. If the instruction is not predicated,
2864 // we fall through to the next case.
2865 ScalarizationCost = 0;
2866
2867 // These instructions have a non-void type, so account for the phi nodes
2868 // that we will create. This cost is likely to be zero. The phi node
2869 // cost, if any, should be scaled by the block probability because it
2870 // models a copy at the end of each predicated block.
2871 ScalarizationCost +=
2872 VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
2873
2874 // The cost of the non-predicated instruction.
2875 ScalarizationCost +=
2876 VF.getFixedValue() *
2877 TTI.getArithmeticInstrCost(I->getOpcode(), I->getType(), CostKind);
2878
2879 // The cost of insertelement and extractelement instructions needed for
2880 // scalarization.
2881 ScalarizationCost += getScalarizationOverhead(I, VF);
2882
2883 // Scale the cost by the probability of executing the predicated blocks.
2884 // This assumes the predicated block for each vector lane is equally
2885 // likely.
2886 ScalarizationCost = ScalarizationCost / getPredBlockCostDivisor(CostKind);
2887 }
2888
2889 InstructionCost SafeDivisorCost = 0;
2890 auto *VecTy = toVectorTy(I->getType(), VF);
2891 // The cost of the select guard to ensure all lanes are well defined
2892 // after we speculate above any internal control flow.
2893 SafeDivisorCost +=
2894 TTI.getCmpSelInstrCost(Instruction::Select, VecTy,
2895 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
2897
2898 SmallVector<const Value *, 4> Operands(I->operand_values());
2899 SafeDivisorCost += TTI.getArithmeticInstrCost(
2900 I->getOpcode(), VecTy, CostKind,
2901 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2902 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
2903 Operands, I);
2904 return {ScalarizationCost, SafeDivisorCost};
2905}
2906
2908 Instruction *I, ElementCount VF) const {
2909 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2911 "Decision should not be set yet.");
2912 auto *Group = getInterleavedAccessGroup(I);
2913 assert(Group && "Must have a group.");
2914 unsigned InterleaveFactor = Group->getFactor();
2915
2916 // If the instruction's allocated size doesn't equal its type size, it
2917 // requires padding and will be scalarized.
2918 auto &DL = I->getDataLayout();
2919 auto *ScalarTy = getLoadStoreType(I);
2920 if (hasIrregularType(ScalarTy, DL))
2921 return false;
2922
2923 // For scalable vectors, the interleave factors must be <= 8 since we require
2924 // the (de)interleaveN intrinsics instead of shufflevectors.
2925 if (VF.isScalable() && InterleaveFactor > 8)
2926 return false;
2927
2928 // If the group involves a non-integral pointer, we may not be able to
2929 // losslessly cast all values to a common type.
2930 bool ScalarNI = DL.isNonIntegralPointerType(ScalarTy);
2931 for (unsigned Idx = 0; Idx < InterleaveFactor; Idx++) {
2932 Instruction *Member = Group->getMember(Idx);
2933 if (!Member)
2934 continue;
2935 auto *MemberTy = getLoadStoreType(Member);
2936 bool MemberNI = DL.isNonIntegralPointerType(MemberTy);
2937 // Don't coerce non-integral pointers to integers or vice versa.
2938 if (MemberNI != ScalarNI)
2939 // TODO: Consider adding special nullptr value case here
2940 return false;
2941 if (MemberNI && ScalarNI &&
2942 ScalarTy->getPointerAddressSpace() !=
2943 MemberTy->getPointerAddressSpace())
2944 return false;
2945 }
2946
2947 // Check if masking is required.
2948 // A Group may need masking for one of two reasons: it resides in a block that
2949 // needs predication, or it was decided to use masking to deal with gaps
2950 // (either a gap at the end of a load-access that may result in a speculative
2951 // load, or any gaps in a store-access).
2952 bool PredicatedAccessRequiresMasking =
2953 blockNeedsPredicationForAnyReason(I->getParent()) &&
2954 Legal->isMaskRequired(I);
2955 bool LoadAccessWithGapsRequiresEpilogMasking =
2956 isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
2958 bool StoreAccessWithGapsRequiresMasking =
2959 isa<StoreInst>(I) && !Group->isFull();
2960 if (!PredicatedAccessRequiresMasking &&
2961 !LoadAccessWithGapsRequiresEpilogMasking &&
2962 !StoreAccessWithGapsRequiresMasking)
2963 return true;
2964
2965 // If masked interleaving is required, we expect that the user/target had
2966 // enabled it, because otherwise it either wouldn't have been created or
2967 // it should have been invalidated by the CostModel.
2969 "Masked interleave-groups for predicated accesses are not enabled.");
2970
2971 if (Group->isReverse())
2972 return false;
2973
2974 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2975 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2976 StoreAccessWithGapsRequiresMasking;
2977 if (VF.isScalable() && NeedsMaskForGaps)
2978 return false;
2979
2980 auto *Ty = getLoadStoreType(I);
2981 const Align Alignment = getLoadStoreAlignment(I);
2982 unsigned AS = getLoadStoreAddressSpace(I);
2983 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
2984 : TTI.isLegalMaskedStore(Ty, Alignment, AS);
2985}
2986
2988 Instruction *I, ElementCount VF) {
2989 // Get and ensure we have a valid memory instruction.
2990 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2991
2993 auto *ScalarTy = getLoadStoreType(I);
2994
2995 // In order to be widened, the pointer should be consecutive, first of all.
2996 if (!Legal->isConsecutivePtr(ScalarTy, Ptr))
2997 return false;
2998
2999 // If the instruction is a store located in a predicated block, it will be
3000 // scalarized.
3001 if (isScalarWithPredication(I, VF))
3002 return false;
3003
3004 // If the instruction's allocated size doesn't equal it's type size, it
3005 // requires padding and will be scalarized.
3006 auto &DL = I->getDataLayout();
3007 if (hasIrregularType(ScalarTy, DL))
3008 return false;
3009
3010 return true;
3011}
3012
3013void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
3014 // We should not collect Uniforms more than once per VF. Right now,
3015 // this function is called from collectUniformsAndScalars(), which
3016 // already does this check. Collecting Uniforms for VF=1 does not make any
3017 // sense.
3018
3019 assert(VF.isVector() && !Uniforms.contains(VF) &&
3020 "This function should not be visited twice for the same VF");
3021
3022 // Visit the list of Uniforms. If we find no uniform value, we won't
3023 // analyze again. Uniforms.count(VF) will return 1.
3024 Uniforms[VF].clear();
3025
3026 // Now we know that the loop is vectorizable!
3027 // Collect instructions inside the loop that will remain uniform after
3028 // vectorization.
3029
3030 // Global values, params and instructions outside of current loop are out of
3031 // scope.
3032 auto IsOutOfScope = [&](Value *V) -> bool {
3034 return (!I || !TheLoop->contains(I));
3035 };
3036
3037 // Worklist containing uniform instructions demanding lane 0.
3038 SetVector<Instruction *> Worklist;
3039
3040 // Add uniform instructions demanding lane 0 to the worklist. Instructions
3041 // that require predication must not be considered uniform after
3042 // vectorization, because that would create an erroneous replicating region
3043 // where only a single instance out of VF should be formed.
3044 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
3045 if (IsOutOfScope(I)) {
3046 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
3047 << *I << "\n");
3048 return;
3049 }
3050 if (isPredicatedInst(I)) {
3051 LLVM_DEBUG(
3052 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
3053 << "\n");
3054 return;
3055 }
3056 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
3057 Worklist.insert(I);
3058 };
3059
3060 // Start with the conditional branches exiting the loop. If the branch
3061 // condition is an instruction contained in the loop that is only used by the
3062 // branch, it is uniform. Note conditions from uncountable early exits are not
3063 // uniform.
3065 TheLoop->getExitingBlocks(Exiting);
3066 for (BasicBlock *E : Exiting) {
3067 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
3068 continue;
3069 auto *Cmp = dyn_cast<Instruction>(E->getTerminator()->getOperand(0));
3070 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
3071 AddToWorklistIfAllowed(Cmp);
3072 }
3073
3074 auto PrevVF = VF.divideCoefficientBy(2);
3075 // Return true if all lanes perform the same memory operation, and we can
3076 // thus choose to execute only one.
3077 auto IsUniformMemOpUse = [&](Instruction *I) {
3078 // If the value was already known to not be uniform for the previous
3079 // (smaller VF), it cannot be uniform for the larger VF.
3080 if (PrevVF.isVector()) {
3081 auto Iter = Uniforms.find(PrevVF);
3082 if (Iter != Uniforms.end() && !Iter->second.contains(I))
3083 return false;
3084 }
3085 if (!Legal->isUniformMemOp(*I, VF))
3086 return false;
3087 if (isa<LoadInst>(I))
3088 // Loading the same address always produces the same result - at least
3089 // assuming aliasing and ordering which have already been checked.
3090 return true;
3091 // Storing the same value on every iteration.
3092 return TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand());
3093 };
3094
3095 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
3096 InstWidening WideningDecision = getWideningDecision(I, VF);
3097 assert(WideningDecision != CM_Unknown &&
3098 "Widening decision should be ready at this moment");
3099
3100 if (IsUniformMemOpUse(I))
3101 return true;
3102
3103 return (WideningDecision == CM_Widen ||
3104 WideningDecision == CM_Widen_Reverse ||
3105 WideningDecision == CM_Interleave);
3106 };
3107
3108 // Returns true if Ptr is the pointer operand of a memory access instruction
3109 // I, I is known to not require scalarization, and the pointer is not also
3110 // stored.
3111 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
3112 if (isa<StoreInst>(I) && I->getOperand(0) == Ptr)
3113 return false;
3114 return getLoadStorePointerOperand(I) == Ptr &&
3115 (IsUniformDecision(I, VF) || Legal->isInvariant(Ptr));
3116 };
3117
3118 // Holds a list of values which are known to have at least one uniform use.
3119 // Note that there may be other uses which aren't uniform. A "uniform use"
3120 // here is something which only demands lane 0 of the unrolled iterations;
3121 // it does not imply that all lanes produce the same value (e.g. this is not
3122 // the usual meaning of uniform)
3123 SetVector<Value *> HasUniformUse;
3124
3125 // Scan the loop for instructions which are either a) known to have only
3126 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
3127 for (auto *BB : TheLoop->blocks())
3128 for (auto &I : *BB) {
3129 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I)) {
3130 switch (II->getIntrinsicID()) {
3131 case Intrinsic::sideeffect:
3132 case Intrinsic::experimental_noalias_scope_decl:
3133 case Intrinsic::assume:
3134 case Intrinsic::lifetime_start:
3135 case Intrinsic::lifetime_end:
3136 if (TheLoop->hasLoopInvariantOperands(&I))
3137 AddToWorklistIfAllowed(&I);
3138 break;
3139 default:
3140 break;
3141 }
3142 }
3143
3144 if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
3145 if (IsOutOfScope(EVI->getAggregateOperand())) {
3146 AddToWorklistIfAllowed(EVI);
3147 continue;
3148 }
3149 // Only ExtractValue instructions where the aggregate value comes from a
3150 // call are allowed to be non-uniform.
3151 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
3152 "Expected aggregate value to be call return value");
3153 }
3154
3155 // If there's no pointer operand, there's nothing to do.
3157 if (!Ptr)
3158 continue;
3159
3160 // If the pointer can be proven to be uniform, always add it to the
3161 // worklist.
3162 if (isa<Instruction>(Ptr) && Legal->isUniform(Ptr, VF))
3163 AddToWorklistIfAllowed(cast<Instruction>(Ptr));
3164
3165 if (IsUniformMemOpUse(&I))
3166 AddToWorklistIfAllowed(&I);
3167
3168 if (IsVectorizedMemAccessUse(&I, Ptr))
3169 HasUniformUse.insert(Ptr);
3170 }
3171
3172 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
3173 // demanding) users. Since loops are assumed to be in LCSSA form, this
3174 // disallows uses outside the loop as well.
3175 for (auto *V : HasUniformUse) {
3176 if (IsOutOfScope(V))
3177 continue;
3178 auto *I = cast<Instruction>(V);
3179 bool UsersAreMemAccesses = all_of(I->users(), [&](User *U) -> bool {
3180 auto *UI = cast<Instruction>(U);
3181 return TheLoop->contains(UI) && IsVectorizedMemAccessUse(UI, V);
3182 });
3183 if (UsersAreMemAccesses)
3184 AddToWorklistIfAllowed(I);
3185 }
3186
3187 // Expand Worklist in topological order: whenever a new instruction
3188 // is added , its users should be already inside Worklist. It ensures
3189 // a uniform instruction will only be used by uniform instructions.
3190 unsigned Idx = 0;
3191 while (Idx != Worklist.size()) {
3192 Instruction *I = Worklist[Idx++];
3193
3194 for (auto *OV : I->operand_values()) {
3195 // isOutOfScope operands cannot be uniform instructions.
3196 if (IsOutOfScope(OV))
3197 continue;
3198 // First order recurrence Phi's should typically be considered
3199 // non-uniform.
3200 auto *OP = dyn_cast<PHINode>(OV);
3201 if (OP && Legal->isFixedOrderRecurrence(OP))
3202 continue;
3203 // If all the users of the operand are uniform, then add the
3204 // operand into the uniform worklist.
3205 auto *OI = cast<Instruction>(OV);
3206 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
3207 auto *J = cast<Instruction>(U);
3208 return Worklist.count(J) || IsVectorizedMemAccessUse(J, OI);
3209 }))
3210 AddToWorklistIfAllowed(OI);
3211 }
3212 }
3213
3214 // For an instruction to be added into Worklist above, all its users inside
3215 // the loop should also be in Worklist. However, this condition cannot be
3216 // true for phi nodes that form a cyclic dependence. We must process phi
3217 // nodes separately. An induction variable will remain uniform if all users
3218 // of the induction variable and induction variable update remain uniform.
3219 // The code below handles both pointer and non-pointer induction variables.
3220 BasicBlock *Latch = TheLoop->getLoopLatch();
3221 for (const auto &Induction : Legal->getInductionVars()) {
3222 auto *Ind = Induction.first;
3223 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
3224
3225 // Determine if all users of the induction variable are uniform after
3226 // vectorization.
3227 bool UniformInd = all_of(Ind->users(), [&](User *U) -> bool {
3228 auto *I = cast<Instruction>(U);
3229 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
3230 IsVectorizedMemAccessUse(I, Ind);
3231 });
3232 if (!UniformInd)
3233 continue;
3234
3235 // Determine if all users of the induction variable update instruction are
3236 // uniform after vectorization.
3237 bool UniformIndUpdate = all_of(IndUpdate->users(), [&](User *U) -> bool {
3238 auto *I = cast<Instruction>(U);
3239 return I == Ind || Worklist.count(I) ||
3240 IsVectorizedMemAccessUse(I, IndUpdate);
3241 });
3242 if (!UniformIndUpdate)
3243 continue;
3244
3245 // The induction variable and its update instruction will remain uniform.
3246 AddToWorklistIfAllowed(Ind);
3247 AddToWorklistIfAllowed(IndUpdate);
3248 }
3249
3250 Uniforms[VF].insert_range(Worklist);
3251}
3252
3254 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n");
3255
3256 if (Legal->getRuntimePointerChecking()->Need) {
3257 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
3258 "runtime pointer checks needed. Enable vectorization of this "
3259 "loop with '#pragma clang loop vectorize(enable)' when "
3260 "compiling with -Os/-Oz",
3261 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3262 return true;
3263 }
3264
3265 if (!PSE.getPredicate().isAlwaysTrue()) {
3266 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
3267 "runtime SCEV checks needed. Enable vectorization of this "
3268 "loop with '#pragma clang loop vectorize(enable)' when "
3269 "compiling with -Os/-Oz",
3270 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3271 return true;
3272 }
3273
3274 // FIXME: Avoid specializing for stride==1 instead of bailing out.
3275 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
3276 reportVectorizationFailure("Runtime stride check for small trip count",
3277 "runtime stride == 1 checks needed. Enable vectorization of "
3278 "this loop without such check by compiling with -Os/-Oz",
3279 "CantVersionLoopWithOptForSize", ORE, TheLoop);
3280 return true;
3281 }
3282
3283 return false;
3284}
3285
3286bool LoopVectorizationCostModel::isScalableVectorizationAllowed() {
3287 if (IsScalableVectorizationAllowed)
3288 return *IsScalableVectorizationAllowed;
3289
3290 IsScalableVectorizationAllowed = false;
3291 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors)
3292 return false;
3293
3294 if (Hints->isScalableVectorizationDisabled()) {
3295 reportVectorizationInfo("Scalable vectorization is explicitly disabled",
3296 "ScalableVectorizationDisabled", ORE, TheLoop);
3297 return false;
3298 }
3299
3300 LLVM_DEBUG(dbgs() << "LV: Scalable vectorization is available\n");
3301
3302 auto MaxScalableVF = ElementCount::getScalable(
3303 std::numeric_limits<ElementCount::ScalarTy>::max());
3304
3305 // Test that the loop-vectorizer can legalize all operations for this MaxVF.
3306 // FIXME: While for scalable vectors this is currently sufficient, this should
3307 // be replaced by a more detailed mechanism that filters out specific VFs,
3308 // instead of invalidating vectorization for a whole set of VFs based on the
3309 // MaxVF.
3310
3311 // Disable scalable vectorization if the loop contains unsupported reductions.
3312 if (!canVectorizeReductions(MaxScalableVF)) {
3314 "Scalable vectorization not supported for the reduction "
3315 "operations found in this loop.",
3316 "ScalableVFUnfeasible", ORE, TheLoop);
3317 return false;
3318 }
3319
3320 // Disable scalable vectorization if the loop contains any instructions
3321 // with element types not supported for scalable vectors.
3322 if (any_of(ElementTypesInLoop, [&](Type *Ty) {
3323 return !Ty->isVoidTy() &&
3325 })) {
3326 reportVectorizationInfo("Scalable vectorization is not supported "
3327 "for all element types found in this loop.",
3328 "ScalableVFUnfeasible", ORE, TheLoop);
3329 return false;
3330 }
3331
3332 if (!Legal->isSafeForAnyVectorWidth() && !getMaxVScale(*TheFunction, TTI)) {
3333 reportVectorizationInfo("The target does not provide maximum vscale value "
3334 "for safe distance analysis.",
3335 "ScalableVFUnfeasible", ORE, TheLoop);
3336 return false;
3337 }
3338
3339 IsScalableVectorizationAllowed = true;
3340 return true;
3341}
3342
3343ElementCount
3344LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
3345 if (!isScalableVectorizationAllowed())
3346 return ElementCount::getScalable(0);
3347
3348 auto MaxScalableVF = ElementCount::getScalable(
3349 std::numeric_limits<ElementCount::ScalarTy>::max());
3350 if (Legal->isSafeForAnyVectorWidth())
3351 return MaxScalableVF;
3352
3353 std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
3354 // Limit MaxScalableVF by the maximum safe dependence distance.
3355 MaxScalableVF = ElementCount::getScalable(MaxSafeElements / *MaxVScale);
3356
3357 if (!MaxScalableVF)
3359 "Max legal vector width too small, scalable vectorization "
3360 "unfeasible.",
3361 "ScalableVFUnfeasible", ORE, TheLoop);
3362
3363 return MaxScalableVF;
3364}
3365
3366FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
3367 unsigned MaxTripCount, ElementCount UserVF, bool FoldTailByMasking) {
3368 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
3369 unsigned SmallestType, WidestType;
3370 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
3371
3372 // Get the maximum safe dependence distance in bits computed by LAA.
3373 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
3374 // the memory accesses that is most restrictive (involved in the smallest
3375 // dependence distance).
3376 unsigned MaxSafeElementsPowerOf2 =
3377 bit_floor(Legal->getMaxSafeVectorWidthInBits() / WidestType);
3378 if (!Legal->isSafeForAnyStoreLoadForwardDistances()) {
3379 unsigned SLDist = Legal->getMaxStoreLoadForwardSafeDistanceInBits();
3380 MaxSafeElementsPowerOf2 =
3381 std::min(MaxSafeElementsPowerOf2, SLDist / WidestType);
3382 }
3383 auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElementsPowerOf2);
3384 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElementsPowerOf2);
3385
3386 if (!Legal->isSafeForAnyVectorWidth())
3387 this->MaxSafeElements = MaxSafeElementsPowerOf2;
3388
3389 LLVM_DEBUG(dbgs() << "LV: The max safe fixed VF is: " << MaxSafeFixedVF
3390 << ".\n");
3391 LLVM_DEBUG(dbgs() << "LV: The max safe scalable VF is: " << MaxSafeScalableVF
3392 << ".\n");
3393
3394 // First analyze the UserVF, fall back if the UserVF should be ignored.
3395 if (UserVF) {
3396 auto MaxSafeUserVF =
3397 UserVF.isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
3398
3399 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
3400 // If `VF=vscale x N` is safe, then so is `VF=N`
3401 if (UserVF.isScalable())
3402 return FixedScalableVFPair(
3403 ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
3404
3405 return UserVF;
3406 }
3407
3408 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF));
3409
3410 // Only clamp if the UserVF is not scalable. If the UserVF is scalable, it
3411 // is better to ignore the hint and let the compiler choose a suitable VF.
3412 if (!UserVF.isScalable()) {
3413 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3414 << " is unsafe, clamping to max safe VF="
3415 << MaxSafeFixedVF << ".\n");
3416 ORE->emit([&]() {
3417 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3418 TheLoop->getStartLoc(),
3419 TheLoop->getHeader())
3420 << "User-specified vectorization factor "
3421 << ore::NV("UserVectorizationFactor", UserVF)
3422 << " is unsafe, clamping to maximum safe vectorization factor "
3423 << ore::NV("VectorizationFactor", MaxSafeFixedVF);
3424 });
3425 return MaxSafeFixedVF;
3426 }
3427
3429 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3430 << " is ignored because scalable vectors are not "
3431 "available.\n");
3432 ORE->emit([&]() {
3433 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3434 TheLoop->getStartLoc(),
3435 TheLoop->getHeader())
3436 << "User-specified vectorization factor "
3437 << ore::NV("UserVectorizationFactor", UserVF)
3438 << " is ignored because the target does not support scalable "
3439 "vectors. The compiler will pick a more suitable value.";
3440 });
3441 } else {
3442 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
3443 << " is unsafe. Ignoring scalable UserVF.\n");
3444 ORE->emit([&]() {
3445 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
3446 TheLoop->getStartLoc(),
3447 TheLoop->getHeader())
3448 << "User-specified vectorization factor "
3449 << ore::NV("UserVectorizationFactor", UserVF)
3450 << " is unsafe. Ignoring the hint to let the compiler pick a "
3451 "more suitable value.";
3452 });
3453 }
3454 }
3455
3456 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestType
3457 << " / " << WidestType << " bits.\n");
3458
3459 FixedScalableVFPair Result(ElementCount::getFixed(1),
3461 if (auto MaxVF =
3462 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3463 MaxSafeFixedVF, FoldTailByMasking))
3464 Result.FixedVF = MaxVF;
3465
3466 if (auto MaxVF =
3467 getMaximizedVFForTarget(MaxTripCount, SmallestType, WidestType,
3468 MaxSafeScalableVF, FoldTailByMasking))
3469 if (MaxVF.isScalable()) {
3470 Result.ScalableVF = MaxVF;
3471 LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVF
3472 << "\n");
3473 }
3474
3475 return Result;
3476}
3477
3478FixedScalableVFPair
3480 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
3481 // TODO: It may be useful to do since it's still likely to be dynamically
3482 // uniform if the target can skip.
3484 "Not inserting runtime ptr check for divergent target",
3485 "runtime pointer checks needed. Not enabled for divergent target",
3486 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
3488 }
3489
3490 ScalarEvolution *SE = PSE.getSE();
3492 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
3493 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
3494 if (TC != ElementCount::getFixed(MaxTC))
3495 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
3496 if (TC.isScalar()) {
3497 reportVectorizationFailure("Single iteration (non) loop",
3498 "loop trip count is one, irrelevant for vectorization",
3499 "SingleIterationLoop", ORE, TheLoop);
3501 }
3502
3503 // If BTC matches the widest induction type and is -1 then the trip count
3504 // computation will wrap to 0 and the vector trip count will be 0. Do not try
3505 // to vectorize.
3506 const SCEV *BTC = SE->getBackedgeTakenCount(TheLoop);
3507 if (!isa<SCEVCouldNotCompute>(BTC) &&
3508 BTC->getType()->getScalarSizeInBits() >=
3509 Legal->getWidestInductionType()->getScalarSizeInBits() &&
3511 SE->getMinusOne(BTC->getType()))) {
3513 "Trip count computation wrapped",
3514 "backedge-taken count is -1, loop trip count wrapped to 0",
3515 "TripCountWrapped", ORE, TheLoop);
3517 }
3518
3519 switch (ScalarEpilogueStatus) {
3521 return computeFeasibleMaxVF(MaxTC, UserVF, false);
3523 [[fallthrough]];
3525 LLVM_DEBUG(
3526 dbgs() << "LV: vector predicate hint/switch found.\n"
3527 << "LV: Not allowing scalar epilogue, creating predicated "
3528 << "vector loop.\n");
3529 break;
3531 // fallthrough as a special case of OptForSize
3533 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
3534 LLVM_DEBUG(
3535 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
3536 else
3537 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "
3538 << "count.\n");
3539
3540 // Bail if runtime checks are required, which are not good when optimising
3541 // for size.
3544
3545 break;
3546 }
3547
3548 // Now try the tail folding
3549
3550 // Invalidate interleave groups that require an epilogue if we can't mask
3551 // the interleave-group.
3553 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
3554 "No decisions should have been taken at this point");
3555 // Note: There is no need to invalidate any cost modeling decisions here, as
3556 // none were taken so far.
3557 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
3558 }
3559
3560 FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(MaxTC, UserVF, true);
3561
3562 // Avoid tail folding if the trip count is known to be a multiple of any VF
3563 // we choose.
3564 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3565 MaxFactors.FixedVF.getFixedValue();
3566 if (MaxFactors.ScalableVF) {
3567 std::optional<unsigned> MaxVScale = getMaxVScale(*TheFunction, TTI);
3568 if (MaxVScale && TTI.isVScaleKnownToBeAPowerOfTwo()) {
3569 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3570 *MaxPowerOf2RuntimeVF,
3571 *MaxVScale * MaxFactors.ScalableVF.getKnownMinValue());
3572 } else
3573 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
3574 }
3575
3576 auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
3577 // Return false if the loop is neither a single-latch-exit loop nor an
3578 // early-exit loop as tail-folding is not supported in that case.
3579 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
3580 !Legal->hasUncountableEarlyExit())
3581 return false;
3582 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3583 ScalarEvolution *SE = PSE.getSE();
3584 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
3585 // with uncountable exits. For countable loops, the symbolic maximum must
3586 // remain identical to the known back-edge taken count.
3587 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
3588 assert((Legal->hasUncountableEarlyExit() ||
3589 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
3590 "Invalid loop count");
3591 const SCEV *ExitCount = SE->getAddExpr(
3592 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
3593 const SCEV *Rem = SE->getURemExpr(
3594 SE->applyLoopGuards(ExitCount, TheLoop),
3595 SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
3596 return Rem->isZero();
3597 };
3598
3599 if (MaxPowerOf2RuntimeVF > 0u) {
3600 assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
3601 "MaxFixedVF must be a power of 2");
3602 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3603 // Accept MaxFixedVF if we do not have a tail.
3604 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
3605 return MaxFactors;
3606 }
3607 }
3608
3609 auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
3610 if (ExpectedTC && ExpectedTC->isFixed() &&
3611 ExpectedTC->getFixedValue() <=
3612 TTI.getMinTripCountTailFoldingThreshold()) {
3613 if (MaxPowerOf2RuntimeVF > 0u) {
3614 // If we have a low-trip-count, and the fixed-width VF is known to divide
3615 // the trip count but the scalable factor does not, use the fixed-width
3616 // factor in preference to allow the generation of a non-predicated loop.
3617 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedLowTripLoop &&
3618 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
3619 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
3620 "remain for any chosen VF.\n");
3621 MaxFactors.ScalableVF = ElementCount::getScalable(0);
3622 return MaxFactors;
3623 }
3624 }
3625
3627 "The trip count is below the minial threshold value.",
3628 "loop trip count is too low, avoiding vectorization", "LowTripCount",
3629 ORE, TheLoop);
3631 }
3632
3633 // If we don't know the precise trip count, or if the trip count that we
3634 // found modulo the vectorization factor is not zero, try to fold the tail
3635 // by masking.
3636 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
3637 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
3638 setTailFoldingStyles(ContainsScalableVF, UserIC);
3639 if (foldTailByMasking()) {
3641 LLVM_DEBUG(
3642 dbgs()
3643 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3644 "try to generate VP Intrinsics with scalable vector "
3645 "factors only.\n");
3646 // Tail folded loop using VP intrinsics restricts the VF to be scalable
3647 // for now.
3648 // TODO: extend it for fixed vectors, if required.
3649 assert(ContainsScalableVF && "Expected scalable vector factor.");
3650
3651 MaxFactors.FixedVF = ElementCount::getFixed(1);
3652 }
3653 return MaxFactors;
3654 }
3655
3656 // If there was a tail-folding hint/switch, but we can't fold the tail by
3657 // masking, fallback to a vectorization with a scalar epilogue.
3658 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
3659 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
3660 "scalar epilogue instead.\n");
3661 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
3662 return MaxFactors;
3663 }
3664
3665 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
3666 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
3668 }
3669
3670 if (TC.isZero()) {
3672 "unable to calculate the loop count due to complex control flow",
3673 "UnknownLoopCountComplexCFG", ORE, TheLoop);
3675 }
3676
3678 "Cannot optimize for size and vectorize at the same time.",
3679 "cannot optimize for size and vectorize at the same time. "
3680 "Enable vectorization of this loop with '#pragma clang loop "
3681 "vectorize(enable)' when compiling with -Os/-Oz",
3682 "NoTailLoopWithOptForSize", ORE, TheLoop);
3684}
3685
3687 ElementCount VF) {
3688 if (ConsiderRegPressure.getNumOccurrences())
3689 return ConsiderRegPressure;
3690
3691 // TODO: We should eventually consider register pressure for all targets. The
3692 // TTI hook is temporary whilst target-specific issues are being fixed.
3693 if (TTI.shouldConsiderVectorizationRegPressure())
3694 return true;
3695
3696 if (!useMaxBandwidth(VF.isScalable()
3699 return false;
3700 // Only calculate register pressure for VFs enabled by MaxBandwidth.
3702 VF, VF.isScalable() ? MaxPermissibleVFWithoutMaxBW.ScalableVF
3704}
3705
3708 return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 &&
3709 (TTI.shouldMaximizeVectorBandwidth(RegKind) ||
3711 Legal->hasVectorCallVariants())));
3712}
3713
3714ElementCount LoopVectorizationCostModel::clampVFByMaxTripCount(
3715 ElementCount VF, unsigned MaxTripCount, bool FoldTailByMasking) const {
3716 unsigned EstimatedVF = VF.getKnownMinValue();
3717 if (VF.isScalable() && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
3718 auto Attr = TheFunction->getFnAttribute(Attribute::VScaleRange);
3719 auto Min = Attr.getVScaleRangeMin();
3720 EstimatedVF *= Min;
3721 }
3722
3723 // When a scalar epilogue is required, at least one iteration of the scalar
3724 // loop has to execute. Adjust MaxTripCount accordingly to avoid picking a
3725 // max VF that results in a dead vector loop.
3726 if (MaxTripCount > 0 && requiresScalarEpilogue(true))
3727 MaxTripCount -= 1;
3728
3729 if (MaxTripCount && MaxTripCount <= EstimatedVF &&
3730 (!FoldTailByMasking || isPowerOf2_32(MaxTripCount))) {
3731 // If upper bound loop trip count (TC) is known at compile time there is no
3732 // point in choosing VF greater than TC (as done in the loop below). Select
3733 // maximum power of two which doesn't exceed TC. If VF is
3734 // scalable, we only fall back on a fixed VF when the TC is less than or
3735 // equal to the known number of lanes.
3736 auto ClampedUpperTripCount = llvm::bit_floor(MaxTripCount);
3737 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
3738 "exceeding the constant trip count: "
3739 << ClampedUpperTripCount << "\n");
3740 return ElementCount::get(ClampedUpperTripCount,
3741 FoldTailByMasking ? VF.isScalable() : false);
3742 }
3743 return VF;
3744}
3745
3746ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
3747 unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType,
3748 ElementCount MaxSafeVF, bool FoldTailByMasking) {
3749 bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
3750 const TypeSize WidestRegister = TTI.getRegisterBitWidth(
3751 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
3753
3754 // Convenience function to return the minimum of two ElementCounts.
3755 auto MinVF = [](const ElementCount &LHS, const ElementCount &RHS) {
3756 assert((LHS.isScalable() == RHS.isScalable()) &&
3757 "Scalable flags must match");
3758 return ElementCount::isKnownLT(LHS, RHS) ? LHS : RHS;
3759 };
3760
3761 // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
3762 // Note that both WidestRegister and WidestType may not be a powers of 2.
3763 auto MaxVectorElementCount = ElementCount::get(
3764 llvm::bit_floor(WidestRegister.getKnownMinValue() / WidestType),
3765 ComputeScalableMaxVF);
3766 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
3767 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "
3768 << (MaxVectorElementCount * WidestType) << " bits.\n");
3769
3770 if (!MaxVectorElementCount) {
3771 LLVM_DEBUG(dbgs() << "LV: The target has no "
3772 << (ComputeScalableMaxVF ? "scalable" : "fixed")
3773 << " vector registers.\n");
3774 return ElementCount::getFixed(1);
3775 }
3776
3777 ElementCount MaxVF = clampVFByMaxTripCount(MaxVectorElementCount,
3778 MaxTripCount, FoldTailByMasking);
3779 // If the MaxVF was already clamped, there's no point in trying to pick a
3780 // larger one.
3781 if (MaxVF != MaxVectorElementCount)
3782 return MaxVF;
3783
3785 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
3787
3788 if (MaxVF.isScalable())
3789 MaxPermissibleVFWithoutMaxBW.ScalableVF = MaxVF;
3790 else
3791 MaxPermissibleVFWithoutMaxBW.FixedVF = MaxVF;
3792
3793 if (useMaxBandwidth(RegKind)) {
3794 auto MaxVectorElementCountMaxBW = ElementCount::get(
3795 llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType),
3796 ComputeScalableMaxVF);
3797 MaxVF = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
3798
3799 if (ElementCount MinVF =
3800 TTI.getMinimumVF(SmallestType, ComputeScalableMaxVF)) {
3801 if (ElementCount::isKnownLT(MaxVF, MinVF)) {
3802 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVF
3803 << ") with target's minimum: " << MinVF << '\n');
3804 MaxVF = MinVF;
3805 }
3806 }
3807
3808 MaxVF = clampVFByMaxTripCount(MaxVF, MaxTripCount, FoldTailByMasking);
3809
3810 if (MaxVectorElementCount != MaxVF) {
3811 // Invalidate any widening decisions we might have made, in case the loop
3812 // requires prediction (decided later), but we have already made some
3813 // load/store widening decisions.
3814 invalidateCostModelingDecisions();
3815 }
3816 }
3817 return MaxVF;
3818}
3819
3820bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
3821 const VectorizationFactor &B,
3822 const unsigned MaxTripCount,
3823 bool HasTail,
3824 bool IsEpilogue) const {
3825 InstructionCost CostA = A.Cost;
3826 InstructionCost CostB = B.Cost;
3827
3828 // Improve estimate for the vector width if it is scalable.
3829 unsigned EstimatedWidthA = A.Width.getKnownMinValue();
3830 unsigned EstimatedWidthB = B.Width.getKnownMinValue();
3831 if (std::optional<unsigned> VScale = CM.getVScaleForTuning()) {
3832 if (A.Width.isScalable())
3833 EstimatedWidthA *= *VScale;
3834 if (B.Width.isScalable())
3835 EstimatedWidthB *= *VScale;
3836 }
3837
3838 // When optimizing for size choose whichever is smallest, which will be the
3839 // one with the smallest cost for the whole loop. On a tie pick the larger
3840 // vector width, on the assumption that throughput will be greater.
3841 if (CM.CostKind == TTI::TCK_CodeSize)
3842 return CostA < CostB ||
3843 (CostA == CostB && EstimatedWidthA > EstimatedWidthB);
3844
3845 // Assume vscale may be larger than 1 (or the value being tuned for),
3846 // so that scalable vectorization is slightly favorable over fixed-width
3847 // vectorization.
3848 bool PreferScalable = !TTI.preferFixedOverScalableIfEqualCost(IsEpilogue) &&
3849 A.Width.isScalable() && !B.Width.isScalable();
3850
3851 auto CmpFn = [PreferScalable](const InstructionCost &LHS,
3852 const InstructionCost &RHS) {
3853 return PreferScalable ? LHS <= RHS : LHS < RHS;
3854 };
3855
3856 // To avoid the need for FP division:
3857 // (CostA / EstimatedWidthA) < (CostB / EstimatedWidthB)
3858 // <=> (CostA * EstimatedWidthB) < (CostB * EstimatedWidthA)
3859 if (!MaxTripCount)
3860 return CmpFn(CostA * EstimatedWidthB, CostB * EstimatedWidthA);
3861
3862 auto GetCostForTC = [MaxTripCount, HasTail](unsigned VF,
3863 InstructionCost VectorCost,
3864 InstructionCost ScalarCost) {
3865 // If the trip count is a known (possibly small) constant, the trip count
3866 // will be rounded up to an integer number of iterations under
3867 // FoldTailByMasking. The total cost in that case will be
3868 // VecCost*ceil(TripCount/VF). When not folding the tail, the total
3869 // cost will be VecCost*floor(TC/VF) + ScalarCost*(TC%VF). There will be
3870 // some extra overheads, but for the purpose of comparing the costs of
3871 // different VFs we can use this to compare the total loop-body cost
3872 // expected after vectorization.
3873 if (HasTail)
3874 return VectorCost * (MaxTripCount / VF) +
3875 ScalarCost * (MaxTripCount % VF);
3876 return VectorCost * divideCeil(MaxTripCount, VF);
3877 };
3878
3879 auto RTCostA = GetCostForTC(EstimatedWidthA, CostA, A.ScalarCost);
3880 auto RTCostB = GetCostForTC(EstimatedWidthB, CostB, B.ScalarCost);
3881 return CmpFn(RTCostA, RTCostB);
3882}
3883
3884bool LoopVectorizationPlanner::isMoreProfitable(const VectorizationFactor &A,
3885 const VectorizationFactor &B,
3886 bool HasTail,
3887 bool IsEpilogue) const {
3888 const unsigned MaxTripCount = PSE.getSmallConstantMaxTripCount();
3889 return LoopVectorizationPlanner::isMoreProfitable(A, B, MaxTripCount, HasTail,
3890 IsEpilogue);
3891}
3892
3895 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3896 SmallVector<RecipeVFPair> InvalidCosts;
3897 for (const auto &Plan : VPlans) {
3898 for (ElementCount VF : Plan->vectorFactors()) {
3899 // The VPlan-based cost model is designed for computing vector cost.
3900 // Querying VPlan-based cost model with a scarlar VF will cause some
3901 // errors because we expect the VF is vector for most of the widen
3902 // recipes.
3903 if (VF.isScalar())
3904 continue;
3905
3906 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind,
3907 *CM.PSE.getSE());
3908 precomputeCosts(*Plan, VF, CostCtx);
3909 auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
3911 for (auto &R : *VPBB) {
3912 if (!R.cost(VF, CostCtx).isValid())
3913 InvalidCosts.emplace_back(&R, VF);
3914 }
3915 }
3916 }
3917 }
3918 if (InvalidCosts.empty())
3919 return;
3920
3921 // Emit a report of VFs with invalid costs in the loop.
3922
3923 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
3925 unsigned I = 0;
3926 for (auto &Pair : InvalidCosts)
3927 if (Numbering.try_emplace(Pair.first, I).second)
3928 ++I;
3929
3930 // Sort the list, first on recipe(number) then on VF.
3931 sort(InvalidCosts, [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
3932 unsigned NA = Numbering[A.first];
3933 unsigned NB = Numbering[B.first];
3934 if (NA != NB)
3935 return NA < NB;
3936 return ElementCount::isKnownLT(A.second, B.second);
3937 });
3938
3939 // For a list of ordered recipe-VF pairs:
3940 // [(load, VF1), (load, VF2), (store, VF1)]
3941 // group the recipes together to emit separate remarks for:
3942 // load (VF1, VF2)
3943 // store (VF1)
3944 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
3945 auto Subset = ArrayRef<RecipeVFPair>();
3946 do {
3947 if (Subset.empty())
3948 Subset = Tail.take_front(1);
3949
3950 VPRecipeBase *R = Subset.front().first;
3951
3952 unsigned Opcode =
3955 [](const auto *R) { return Instruction::PHI; })
3956 .Case<VPWidenSelectRecipe>(
3957 [](const auto *R) { return Instruction::Select; })
3958 .Case<VPWidenStoreRecipe>(
3959 [](const auto *R) { return Instruction::Store; })
3960 .Case<VPWidenLoadRecipe>(
3961 [](const auto *R) { return Instruction::Load; })
3962 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3963 [](const auto *R) { return Instruction::Call; })
3966 [](const auto *R) { return R->getOpcode(); })
3967 .Case<VPInterleaveRecipe>([](const VPInterleaveRecipe *R) {
3968 return R->getStoredValues().empty() ? Instruction::Load
3969 : Instruction::Store;
3970 });
3971
3972 // If the next recipe is different, or if there are no other pairs,
3973 // emit a remark for the collated subset. e.g.
3974 // [(load, VF1), (load, VF2))]
3975 // to emit:
3976 // remark: invalid costs for 'load' at VF=(VF1, VF2)
3977 if (Subset == Tail || Tail[Subset.size()].first != R) {
3978 std::string OutString;
3979 raw_string_ostream OS(OutString);
3980 assert(!Subset.empty() && "Unexpected empty range");
3981 OS << "Recipe with invalid costs prevented vectorization at VF=(";
3982 for (const auto &Pair : Subset)
3983 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
3984 OS << "):";
3985 if (Opcode == Instruction::Call) {
3986 StringRef Name = "";
3987 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(R)) {
3988 Name = Int->getIntrinsicName();
3989 } else {
3990 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(R);
3991 Function *CalledFn =
3992 WidenCall ? WidenCall->getCalledScalarFunction()
3993 : cast<Function>(R->getOperand(R->getNumOperands() - 1)
3994 ->getLiveInIRValue());
3995 Name = CalledFn->getName();
3996 }
3997 OS << " call to " << Name;
3998 } else
3999 OS << " " << Instruction::getOpcodeName(Opcode);
4000 reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr,
4001 R->getDebugLoc());
4002 Tail = Tail.drop_front(Subset.size());
4003 Subset = {};
4004 } else
4005 // Grow the subset by one element
4006 Subset = Tail.take_front(Subset.size() + 1);
4007 } while (!Tail.empty());
4008}
4009
4010/// Check if any recipe of \p Plan will generate a vector value, which will be
4011/// assigned a vector register.
4013 const TargetTransformInfo &TTI) {
4014 assert(VF.isVector() && "Checking a scalar VF?");
4015 VPTypeAnalysis TypeInfo(Plan);
4016 DenseSet<VPRecipeBase *> EphemeralRecipes;
4017 collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
4018 // Set of already visited types.
4019 DenseSet<Type *> Visited;
4022 for (VPRecipeBase &R : *VPBB) {
4023 if (EphemeralRecipes.contains(&R))
4024 continue;
4025 // Continue early if the recipe is considered to not produce a vector
4026 // result. Note that this includes VPInstruction where some opcodes may
4027 // produce a vector, to preserve existing behavior as VPInstructions model
4028 // aspects not directly mapped to existing IR instructions.
4029 switch (R.getVPDefID()) {
4030 case VPDef::VPDerivedIVSC:
4031 case VPDef::VPScalarIVStepsSC:
4032 case VPDef::VPReplicateSC:
4033 case VPDef::VPInstructionSC:
4034 case VPDef::VPCanonicalIVPHISC:
4035 case VPDef::VPVectorPointerSC:
4036 case VPDef::VPVectorEndPointerSC:
4037 case VPDef::VPExpandSCEVSC:
4038 case VPDef::VPEVLBasedIVPHISC:
4039 case VPDef::VPPredInstPHISC:
4040 case VPDef::VPBranchOnMaskSC:
4041 continue;
4042 case VPDef::VPReductionSC:
4043 case VPDef::VPActiveLaneMaskPHISC:
4044 case VPDef::VPWidenCallSC:
4045 case VPDef::VPWidenCanonicalIVSC:
4046 case VPDef::VPWidenCastSC:
4047 case VPDef::VPWidenGEPSC:
4048 case VPDef::VPWidenIntrinsicSC:
4049 case VPDef::VPWidenSC:
4050 case VPDef::VPWidenSelectSC:
4051 case VPDef::VPBlendSC:
4052 case VPDef::VPFirstOrderRecurrencePHISC:
4053 case VPDef::VPHistogramSC:
4054 case VPDef::VPWidenPHISC:
4055 case VPDef::VPWidenIntOrFpInductionSC:
4056 case VPDef::VPWidenPointerInductionSC:
4057 case VPDef::VPReductionPHISC:
4058 case VPDef::VPInterleaveEVLSC:
4059 case VPDef::VPInterleaveSC:
4060 case VPDef::VPWidenLoadEVLSC:
4061 case VPDef::VPWidenLoadSC:
4062 case VPDef::VPWidenStoreEVLSC:
4063 case VPDef::VPWidenStoreSC:
4064 break;
4065 default:
4066 llvm_unreachable("unhandled recipe");
4067 }
4068
4069 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
4070 unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
4071 if (!NumLegalParts)
4072 return false;
4073 if (VF.isScalable()) {
4074 // <vscale x 1 x iN> is assumed to be profitable over iN because
4075 // scalable registers are a distinct register class from scalar
4076 // ones. If we ever find a target which wants to lower scalable
4077 // vectors back to scalars, we'll need to update this code to
4078 // explicitly ask TTI about the register class uses for each part.
4079 return NumLegalParts <= VF.getKnownMinValue();
4080 }
4081 // Two or more elements that share a register - are vectorized.
4082 return NumLegalParts < VF.getFixedValue();
4083 };
4084
4085 // If no def nor is a store, e.g., branches, continue - no value to check.
4086 if (R.getNumDefinedValues() == 0 &&
4088 continue;
4089 // For multi-def recipes, currently only interleaved loads, suffice to
4090 // check first def only.
4091 // For stores check their stored value; for interleaved stores suffice
4092 // the check first stored value only. In all cases this is the second
4093 // operand.
4094 VPValue *ToCheck =
4095 R.getNumDefinedValues() >= 1 ? R.getVPValue(0) : R.getOperand(1);
4096 Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
4097 if (!Visited.insert({ScalarTy}).second)
4098 continue;
4099 Type *WideTy = toVectorizedTy(ScalarTy, VF);
4100 if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
4101 return true;
4102 }
4103 }
4104
4105 return false;
4106}
4107
4108static bool hasReplicatorRegion(VPlan &Plan) {
4110 Plan.getVectorLoopRegion()->getEntry())),
4111 [](auto *VPRB) { return VPRB->isReplicator(); });
4112}
4113
4114#ifndef NDEBUG
4115VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
4116 InstructionCost ExpectedCost = CM.expectedCost(ElementCount::getFixed(1));
4117 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
4118 assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
4119 assert(
4120 any_of(VPlans,
4121 [](std::unique_ptr<VPlan> &P) { return P->hasScalarVFOnly(); }) &&
4122 "Expected Scalar VF to be a candidate");
4123
4124 const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost,
4125 ExpectedCost);
4126 VectorizationFactor ChosenFactor = ScalarCost;
4127
4128 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
4129 if (ForceVectorization &&
4130 (VPlans.size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
4131 // Ignore scalar width, because the user explicitly wants vectorization.
4132 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
4133 // evaluation.
4134 ChosenFactor.Cost = InstructionCost::getMax();
4135 }
4136
4137 for (auto &P : VPlans) {
4138 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
4139 P->vectorFactors().end());
4140
4142 if (any_of(VFs, [this](ElementCount VF) {
4143 return CM.shouldConsiderRegPressureForVF(VF);
4144 }))
4145 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
4146
4147 for (unsigned I = 0; I < VFs.size(); I++) {
4148 ElementCount VF = VFs[I];
4149 // The cost for scalar VF=1 is already calculated, so ignore it.
4150 if (VF.isScalar())
4151 continue;
4152
4153 /// If the register pressure needs to be considered for VF,
4154 /// don't consider the VF as valid if it exceeds the number
4155 /// of registers for the target.
4156 if (CM.shouldConsiderRegPressureForVF(VF) &&
4157 RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs))
4158 continue;
4159
4160 InstructionCost C = CM.expectedCost(VF);
4161
4162 // Add on other costs that are modelled in VPlan, but not in the legacy
4163 // cost model.
4164 VPCostContext CostCtx(CM.TTI, *CM.TLI, *P, CM, CM.CostKind,
4165 *CM.PSE.getSE());
4166 VPRegionBlock *VectorRegion = P->getVectorLoopRegion();
4167 assert(VectorRegion && "Expected to have a vector region!");
4168 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4169 vp_depth_first_shallow(VectorRegion->getEntry()))) {
4170 for (VPRecipeBase &R : *VPBB) {
4171 auto *VPI = dyn_cast<VPInstruction>(&R);
4172 if (!VPI)
4173 continue;
4174 switch (VPI->getOpcode()) {
4175 // Selects are only modelled in the legacy cost model for safe
4176 // divisors.
4177 case Instruction::Select: {
4178 VPValue *VPV = VPI->getVPSingleValue();
4179 if (VPV->getNumUsers() == 1) {
4180 if (auto *WR = dyn_cast<VPWidenRecipe>(*VPV->user_begin())) {
4181 switch (WR->getOpcode()) {
4182 case Instruction::UDiv:
4183 case Instruction::SDiv:
4184 case Instruction::URem:
4185 case Instruction::SRem:
4186 continue;
4187 default:
4188 break;
4189 }
4190 }
4191 }
4192 C += VPI->cost(VF, CostCtx);
4193 break;
4194 }
4196 unsigned Multiplier =
4197 cast<ConstantInt>(VPI->getOperand(2)->getLiveInIRValue())
4198 ->getZExtValue();
4199 C += VPI->cost(VF * Multiplier, CostCtx);
4200 break;
4201 }
4203 C += VPI->cost(VF, CostCtx);
4204 break;
4205 default:
4206 break;
4207 }
4208 }
4209 }
4210
4211 VectorizationFactor Candidate(VF, C, ScalarCost.ScalarCost);
4212 unsigned Width =
4213 estimateElementCount(Candidate.Width, CM.getVScaleForTuning());
4214 LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << VF
4215 << " costs: " << (Candidate.Cost / Width));
4216 if (VF.isScalable())
4217 LLVM_DEBUG(dbgs() << " (assuming a minimum vscale of "
4218 << CM.getVScaleForTuning().value_or(1) << ")");
4219 LLVM_DEBUG(dbgs() << ".\n");
4220
4221 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
4222 LLVM_DEBUG(
4223 dbgs()
4224 << "LV: Not considering vector loop of width " << VF
4225 << " because it will not generate any vector instructions.\n");
4226 continue;
4227 }
4228
4229 if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
4230 LLVM_DEBUG(
4231 dbgs()
4232 << "LV: Not considering vector loop of width " << VF
4233 << " because it would cause replicated blocks to be generated,"
4234 << " which isn't allowed when optimizing for size.\n");
4235 continue;
4236 }
4237
4238 if (isMoreProfitable(Candidate, ChosenFactor, P->hasScalarTail()))
4239 ChosenFactor = Candidate;
4240 }
4241 }
4242
4243 if (!EnableCondStoresVectorization && CM.hasPredStores()) {
4245 "There are conditional stores.",
4246 "store that is conditionally executed prevents vectorization",
4247 "ConditionalStore", ORE, OrigLoop);
4248 ChosenFactor = ScalarCost;
4249 }
4250
4251 LLVM_DEBUG(if (ForceVectorization && !ChosenFactor.Width.isScalar() &&
4252 !isMoreProfitable(ChosenFactor, ScalarCost,
4253 !CM.foldTailByMasking())) dbgs()
4254 << "LV: Vectorization seems to be not beneficial, "
4255 << "but was forced by a user.\n");
4256 return ChosenFactor;
4257}
4258#endif
4259
4260bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
4261 ElementCount VF) const {
4262 // Cross iteration phis such as fixed-order recurrences and FMaxNum/FMinNum
4263 // reductions need special handling and are currently unsupported.
4264 if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
4265 if (!Legal->isReductionVariable(&Phi))
4266 return Legal->isFixedOrderRecurrence(&Phi);
4267 RecurKind RK = Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
4268 return RK == RecurKind::FMinNum || RK == RecurKind::FMaxNum;
4269 }))
4270 return false;
4271
4272 // Phis with uses outside of the loop require special handling and are
4273 // currently unsupported.
4274 for (const auto &Entry : Legal->getInductionVars()) {
4275 // Look for uses of the value of the induction at the last iteration.
4276 Value *PostInc =
4277 Entry.first->getIncomingValueForBlock(OrigLoop->getLoopLatch());
4278 for (User *U : PostInc->users())
4279 if (!OrigLoop->contains(cast<Instruction>(U)))
4280 return false;
4281 // Look for uses of penultimate value of the induction.
4282 for (User *U : Entry.first->users())
4283 if (!OrigLoop->contains(cast<Instruction>(U)))
4284 return false;
4285 }
4286
4287 // Epilogue vectorization code has not been auditted to ensure it handles
4288 // non-latch exits properly. It may be fine, but it needs auditted and
4289 // tested.
4290 // TODO: Add support for loops with an early exit.
4291 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
4292 return false;
4293
4294 return true;
4295}
4296
4298 const ElementCount VF, const unsigned IC) const {
4299 // FIXME: We need a much better cost-model to take different parameters such
4300 // as register pressure, code size increase and cost of extra branches into
4301 // account. For now we apply a very crude heuristic and only consider loops
4302 // with vectorization factors larger than a certain value.
4303
4304 // Allow the target to opt out entirely.
4305 if (!TTI.preferEpilogueVectorization())
4306 return false;
4307
4308 // We also consider epilogue vectorization unprofitable for targets that don't
4309 // consider interleaving beneficial (eg. MVE).
4310 if (TTI.getMaxInterleaveFactor(VF) <= 1)
4311 return false;
4312
4313 unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
4315 : TTI.getEpilogueVectorizationMinVF();
4316 return estimateElementCount(VF * IC, VScaleForTuning) >= MinVFThreshold;
4317}
4318
4320 const ElementCount MainLoopVF, unsigned IC) {
4323 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
4324 return Result;
4325 }
4326
4327 if (!CM.isScalarEpilogueAllowed()) {
4328 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
4329 "epilogue is allowed.\n");
4330 return Result;
4331 }
4332
4333 // Not really a cost consideration, but check for unsupported cases here to
4334 // simplify the logic.
4335 if (!isCandidateForEpilogueVectorization(MainLoopVF)) {
4336 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
4337 "is not a supported candidate.\n");
4338 return Result;
4339 }
4340
4342 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
4344 if (hasPlanWithVF(ForcedEC))
4345 return {ForcedEC, 0, 0};
4346
4347 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
4348 "viable.\n");
4349 return Result;
4350 }
4351
4352 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
4353 LLVM_DEBUG(
4354 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
4355 return Result;
4356 }
4357
4358 if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
4359 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
4360 "this loop\n");
4361 return Result;
4362 }
4363
4364 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
4365 // the main loop handles 8 lanes per iteration. We could still benefit from
4366 // vectorizing the epilogue loop with VF=4.
4367 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
4368 estimateElementCount(MainLoopVF, CM.getVScaleForTuning()));
4369
4370 ScalarEvolution &SE = *PSE.getSE();
4371 Type *TCType = Legal->getWidestInductionType();
4372 const SCEV *RemainingIterations = nullptr;
4373 unsigned MaxTripCount = 0;
4374 const SCEV *TC =
4375 vputils::getSCEVExprForVPValue(getPlanFor(MainLoopVF).getTripCount(), SE);
4376 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
4377 RemainingIterations =
4378 SE.getURemExpr(TC, SE.getElementCount(TCType, MainLoopVF * IC));
4379
4380 // No iterations left to process in the epilogue.
4381 if (RemainingIterations->isZero())
4382 return Result;
4383
4384 if (MainLoopVF.isFixed()) {
4385 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
4386 if (SE.isKnownPredicate(CmpInst::ICMP_ULT, RemainingIterations,
4387 SE.getConstant(TCType, MaxTripCount))) {
4388 MaxTripCount = SE.getUnsignedRangeMax(RemainingIterations).getZExtValue();
4389 }
4390 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
4391 << MaxTripCount << "\n");
4392 }
4393
4394 for (auto &NextVF : ProfitableVFs) {
4395 // Skip candidate VFs without a corresponding VPlan.
4396 if (!hasPlanWithVF(NextVF.Width))
4397 continue;
4398
4399 // Skip candidate VFs with widths >= the (estimated) runtime VF (scalable
4400 // vectors) or > the VF of the main loop (fixed vectors).
4401 if ((!NextVF.Width.isScalable() && MainLoopVF.isScalable() &&
4402 ElementCount::isKnownGE(NextVF.Width, EstimatedRuntimeVF)) ||
4403 (NextVF.Width.isScalable() &&
4404 ElementCount::isKnownGE(NextVF.Width, MainLoopVF)) ||
4405 (!NextVF.Width.isScalable() && !MainLoopVF.isScalable() &&
4406 ElementCount::isKnownGT(NextVF.Width, MainLoopVF)))
4407 continue;
4408
4409 // If NextVF is greater than the number of remaining iterations, the
4410 // epilogue loop would be dead. Skip such factors.
4411 if (RemainingIterations && !NextVF.Width.isScalable()) {
4412 if (SE.isKnownPredicate(
4414 SE.getConstant(TCType, NextVF.Width.getFixedValue()),
4415 RemainingIterations))
4416 continue;
4417 }
4418
4419 if (Result.Width.isScalar() ||
4420 isMoreProfitable(NextVF, Result, MaxTripCount, !CM.foldTailByMasking(),
4421 /*IsEpilogue*/ true))
4422 Result = NextVF;
4423 }
4424
4425 if (Result != VectorizationFactor::Disabled())
4426 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
4427 << Result.Width << "\n");
4428 return Result;
4429}
4430
4431std::pair<unsigned, unsigned>
4433 unsigned MinWidth = -1U;
4434 unsigned MaxWidth = 8;
4435 const DataLayout &DL = TheFunction->getDataLayout();
4436 // For in-loop reductions, no element types are added to ElementTypesInLoop
4437 // if there are no loads/stores in the loop. In this case, check through the
4438 // reduction variables to determine the maximum width.
4439 if (ElementTypesInLoop.empty() && !Legal->getReductionVars().empty()) {
4440 for (const auto &PhiDescriptorPair : Legal->getReductionVars()) {
4441 const RecurrenceDescriptor &RdxDesc = PhiDescriptorPair.second;
4442 // When finding the min width used by the recurrence we need to account
4443 // for casts on the input operands of the recurrence.
4444 MinWidth = std::min(
4445 MinWidth,
4446 std::min(RdxDesc.getMinWidthCastToRecurrenceTypeInBits(),
4448 MaxWidth = std::max(MaxWidth,
4450 }
4451 } else {
4452 for (Type *T : ElementTypesInLoop) {
4453 MinWidth = std::min<unsigned>(
4454 MinWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
4455 MaxWidth = std::max<unsigned>(
4456 MaxWidth, DL.getTypeSizeInBits(T->getScalarType()).getFixedValue());
4457 }
4458 }
4459 return {MinWidth, MaxWidth};
4460}
4461
4463 ElementTypesInLoop.clear();
4464 // For each block.
4465 for (BasicBlock *BB : TheLoop->blocks()) {
4466 // For each instruction in the loop.
4467 for (Instruction &I : BB->instructionsWithoutDebug()) {
4468 Type *T = I.getType();
4469
4470 // Skip ignored values.
4471 if (ValuesToIgnore.count(&I))
4472 continue;
4473
4474 // Only examine Loads, Stores and PHINodes.
4475 if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))
4476 continue;
4477
4478 // Examine PHI nodes that are reduction variables. Update the type to
4479 // account for the recurrence type.
4480 if (auto *PN = dyn_cast<PHINode>(&I)) {
4481 if (!Legal->isReductionVariable(PN))
4482 continue;
4483 const RecurrenceDescriptor &RdxDesc =
4484 Legal->getRecurrenceDescriptor(PN);
4486 TTI.preferInLoopReduction(RdxDesc.getRecurrenceKind(),
4487 RdxDesc.getRecurrenceType()))
4488 continue;
4489 T = RdxDesc.getRecurrenceType();
4490 }
4491
4492 // Examine the stored values.
4493 if (auto *ST = dyn_cast<StoreInst>(&I))
4494 T = ST->getValueOperand()->getType();
4495
4496 assert(T->isSized() &&
4497 "Expected the load/store/recurrence type to be sized");
4498
4499 ElementTypesInLoop.insert(T);
4500 }
4501 }
4502}
4503
4504unsigned
4506 InstructionCost LoopCost) {
4507 // -- The interleave heuristics --
4508 // We interleave the loop in order to expose ILP and reduce the loop overhead.
4509 // There are many micro-architectural considerations that we can't predict
4510 // at this level. For example, frontend pressure (on decode or fetch) due to
4511 // code size, or the number and capabilities of the execution ports.
4512 //
4513 // We use the following heuristics to select the interleave count:
4514 // 1. If the code has reductions, then we interleave to break the cross
4515 // iteration dependency.
4516 // 2. If the loop is really small, then we interleave to reduce the loop
4517 // overhead.
4518 // 3. We don't interleave if we think that we will spill registers to memory
4519 // due to the increased register pressure.
4520
4521 if (!CM.isScalarEpilogueAllowed())
4522 return 1;
4523
4526 LLVM_DEBUG(dbgs() << "LV: Preference for VP intrinsics indicated. "
4527 "Unroll factor forced to be 1.\n");
4528 return 1;
4529 }
4530
4531 // We used the distance for the interleave count.
4532 if (!Legal->isSafeForAnyVectorWidth())
4533 return 1;
4534
4535 // We don't attempt to perform interleaving for loops with uncountable early
4536 // exits because the VPInstruction::AnyOf code cannot currently handle
4537 // multiple parts.
4538 if (Plan.hasEarlyExit())
4539 return 1;
4540
4541 const bool HasReductions =
4544
4545 // If we did not calculate the cost for VF (because the user selected the VF)
4546 // then we calculate the cost of VF here.
4547 if (LoopCost == 0) {
4548 if (VF.isScalar())
4549 LoopCost = CM.expectedCost(VF);
4550 else
4551 LoopCost = cost(Plan, VF);
4552 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
4553
4554 // Loop body is free and there is no need for interleaving.
4555 if (LoopCost == 0)
4556 return 1;
4557 }
4558
4559 VPRegisterUsage R =
4560 calculateRegisterUsageForPlan(Plan, {VF}, TTI, CM.ValuesToIgnore)[0];
4561 // We divide by these constants so assume that we have at least one
4562 // instruction that uses at least one register.
4563 for (auto &Pair : R.MaxLocalUsers) {
4564 Pair.second = std::max(Pair.second, 1U);
4565 }
4566
4567 // We calculate the interleave count using the following formula.
4568 // Subtract the number of loop invariants from the number of available
4569 // registers. These registers are used by all of the interleaved instances.
4570 // Next, divide the remaining registers by the number of registers that is
4571 // required by the loop, in order to estimate how many parallel instances
4572 // fit without causing spills. All of this is rounded down if necessary to be
4573 // a power of two. We want power of two interleave count to simplify any
4574 // addressing operations or alignment considerations.
4575 // We also want power of two interleave counts to ensure that the induction
4576 // variable of the vector loop wraps to zero, when tail is folded by masking;
4577 // this currently happens when OptForSize, in which case IC is set to 1 above.
4578 unsigned IC = UINT_MAX;
4579
4580 for (const auto &Pair : R.MaxLocalUsers) {
4581 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(Pair.first);
4582 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
4583 << " registers of "
4584 << TTI.getRegisterClassName(Pair.first)
4585 << " register class\n");
4586 if (VF.isScalar()) {
4587 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
4588 TargetNumRegisters = ForceTargetNumScalarRegs;
4589 } else {
4590 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
4591 TargetNumRegisters = ForceTargetNumVectorRegs;
4592 }
4593 unsigned MaxLocalUsers = Pair.second;
4594 unsigned LoopInvariantRegs = 0;
4595 if (R.LoopInvariantRegs.contains(Pair.first))
4596 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
4597
4598 unsigned TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs) /
4599 MaxLocalUsers);
4600 // Don't count the induction variable as interleaved.
4602 TmpIC = llvm::bit_floor((TargetNumRegisters - LoopInvariantRegs - 1) /
4603 std::max(1U, (MaxLocalUsers - 1)));
4604 }
4605
4606 IC = std::min(IC, TmpIC);
4607 }
4608
4609 // Clamp the interleave ranges to reasonable counts.
4610 unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
4611
4612 // Check if the user has overridden the max.
4613 if (VF.isScalar()) {
4614 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
4615 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
4616 } else {
4617 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
4618 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
4619 }
4620
4621 // Try to get the exact trip count, or an estimate based on profiling data or
4622 // ConstantMax from PSE, failing that.
4623 auto BestKnownTC = getSmallBestKnownTC(PSE, OrigLoop);
4624
4625 // For fixed length VFs treat a scalable trip count as unknown.
4626 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
4627 // Re-evaluate trip counts and VFs to be in the same numerical space.
4628 unsigned AvailableTC =
4629 estimateElementCount(*BestKnownTC, CM.getVScaleForTuning());
4630 unsigned EstimatedVF = estimateElementCount(VF, CM.getVScaleForTuning());
4631
4632 // At least one iteration must be scalar when this constraint holds. So the
4633 // maximum available iterations for interleaving is one less.
4634 if (CM.requiresScalarEpilogue(VF.isVector()))
4635 --AvailableTC;
4636
4637 unsigned InterleaveCountLB = bit_floor(std::max(
4638 1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
4639
4640 if (getSmallConstantTripCount(PSE.getSE(), OrigLoop).isNonZero()) {
4641 // If the best known trip count is exact, we select between two
4642 // prospective ICs, where
4643 //
4644 // 1) the aggressive IC is capped by the trip count divided by VF
4645 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
4646 //
4647 // The final IC is selected in a way that the epilogue loop trip count is
4648 // minimized while maximizing the IC itself, so that we either run the
4649 // vector loop at least once if it generates a small epilogue loop, or
4650 // else we run the vector loop at least twice.
4651
4652 unsigned InterleaveCountUB = bit_floor(std::max(
4653 1u, std::min(AvailableTC / EstimatedVF, MaxInterleaveCount)));
4654 MaxInterleaveCount = InterleaveCountLB;
4655
4656 if (InterleaveCountUB != InterleaveCountLB) {
4657 unsigned TailTripCountUB =
4658 (AvailableTC % (EstimatedVF * InterleaveCountUB));
4659 unsigned TailTripCountLB =
4660 (AvailableTC % (EstimatedVF * InterleaveCountLB));
4661 // If both produce same scalar tail, maximize the IC to do the same work
4662 // in fewer vector loop iterations
4663 if (TailTripCountUB == TailTripCountLB)
4664 MaxInterleaveCount = InterleaveCountUB;
4665 }
4666 } else {
4667 // If trip count is an estimated compile time constant, limit the
4668 // IC to be capped by the trip count divided by VF * 2, such that the
4669 // vector loop runs at least twice to make interleaving seem profitable
4670 // when there is an epilogue loop present. Since exact Trip count is not
4671 // known we choose to be conservative in our IC estimate.
4672 MaxInterleaveCount = InterleaveCountLB;
4673 }
4674 }
4675
4676 assert(MaxInterleaveCount > 0 &&
4677 "Maximum interleave count must be greater than 0");
4678
4679 // Clamp the calculated IC to be between the 1 and the max interleave count
4680 // that the target and trip count allows.
4681 if (IC > MaxInterleaveCount)
4682 IC = MaxInterleaveCount;
4683 else
4684 // Make sure IC is greater than 0.
4685 IC = std::max(1u, IC);
4686
4687 assert(IC > 0 && "Interleave count must be greater than 0.");
4688
4689 // Interleave if we vectorized this loop and there is a reduction that could
4690 // benefit from interleaving.
4691 if (VF.isVector() && HasReductions) {
4692 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
4693 return IC;
4694 }
4695
4696 // For any scalar loop that either requires runtime checks or predication we
4697 // are better off leaving this to the unroller. Note that if we've already
4698 // vectorized the loop we will have done the runtime check and so interleaving
4699 // won't require further checks.
4700 bool ScalarInterleavingRequiresPredication =
4701 (VF.isScalar() && any_of(OrigLoop->blocks(), [this](BasicBlock *BB) {
4702 return Legal->blockNeedsPredication(BB);
4703 }));
4704 bool ScalarInterleavingRequiresRuntimePointerCheck =
4705 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
4706
4707 // We want to interleave small loops in order to reduce the loop overhead and
4708 // potentially expose ILP opportunities.
4709 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
4710 << "LV: IC is " << IC << '\n'
4711 << "LV: VF is " << VF << '\n');
4712 const bool AggressivelyInterleaveReductions =
4713 TTI.enableAggressiveInterleaving(HasReductions);
4714 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
4715 !ScalarInterleavingRequiresPredication && LoopCost < SmallLoopCost) {
4716 // We assume that the cost overhead is 1 and we use the cost model
4717 // to estimate the cost of the loop and interleave until the cost of the
4718 // loop overhead is about 5% of the cost of the loop.
4719 unsigned SmallIC = std::min(IC, (unsigned)llvm::bit_floor<uint64_t>(
4720 SmallLoopCost / LoopCost.getValue()));
4721
4722 // Interleave until store/load ports (estimated by max interleave count) are
4723 // saturated.
4724 unsigned NumStores = 0;
4725 unsigned NumLoads = 0;
4728 for (VPRecipeBase &R : *VPBB) {
4730 NumLoads++;
4731 continue;
4732 }
4734 NumStores++;
4735 continue;
4736 }
4737
4738 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(&R)) {
4739 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
4740 NumStores += StoreOps;
4741 else
4742 NumLoads += InterleaveR->getNumDefinedValues();
4743 continue;
4744 }
4745 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
4746 NumLoads += isa<LoadInst>(RepR->getUnderlyingInstr());
4747 NumStores += isa<StoreInst>(RepR->getUnderlyingInstr());
4748 continue;
4749 }
4750 if (isa<VPHistogramRecipe>(&R)) {
4751 NumLoads++;
4752 NumStores++;
4753 continue;
4754 }
4755 }
4756 }
4757 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
4758 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
4759
4760 // There is little point in interleaving for reductions containing selects
4761 // and compares when VF=1 since it may just create more overhead than it's
4762 // worth for loops with small trip counts. This is because we still have to
4763 // do the final reduction after the loop.
4764 bool HasSelectCmpReductions =
4765 HasReductions &&
4767 [](VPRecipeBase &R) {
4768 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4769 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
4770 RedR->getRecurrenceKind()) ||
4771 RecurrenceDescriptor::isFindIVRecurrenceKind(
4772 RedR->getRecurrenceKind()));
4773 });
4774 if (HasSelectCmpReductions) {
4775 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
4776 return 1;
4777 }
4778
4779 // If we have a scalar reduction (vector reductions are already dealt with
4780 // by this point), we can increase the critical path length if the loop
4781 // we're interleaving is inside another loop. For tree-wise reductions
4782 // set the limit to 2, and for ordered reductions it's best to disable
4783 // interleaving entirely.
4784 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
4785 bool HasOrderedReductions =
4787 [](VPRecipeBase &R) {
4788 auto *RedR = dyn_cast<VPReductionPHIRecipe>(&R);
4789
4790 return RedR && RedR->isOrdered();
4791 });
4792 if (HasOrderedReductions) {
4793 LLVM_DEBUG(
4794 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
4795 return 1;
4796 }
4797
4798 unsigned F = MaxNestedScalarReductionIC;
4799 SmallIC = std::min(SmallIC, F);
4800 StoresIC = std::min(StoresIC, F);
4801 LoadsIC = std::min(LoadsIC, F);
4802 }
4803
4805 std::max(StoresIC, LoadsIC) > SmallIC) {
4806 LLVM_DEBUG(
4807 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
4808 return std::max(StoresIC, LoadsIC);
4809 }
4810
4811 // If there are scalar reductions and TTI has enabled aggressive
4812 // interleaving for reductions, we will interleave to expose ILP.
4813 if (VF.isScalar() && AggressivelyInterleaveReductions) {
4814 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
4815 // Interleave no less than SmallIC but not as aggressive as the normal IC
4816 // to satisfy the rare situation when resources are too limited.
4817 return std::max(IC / 2, SmallIC);
4818 }
4819
4820 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
4821 return SmallIC;
4822 }
4823
4824 // Interleave if this is a large loop (small loops are already dealt with by
4825 // this point) that could benefit from interleaving.
4826 if (AggressivelyInterleaveReductions) {
4827 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
4828 return IC;
4829 }
4830
4831 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
4832 return 1;
4833}
4834
4835bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
4836 ElementCount VF) {
4837 // TODO: Cost model for emulated masked load/store is completely
4838 // broken. This hack guides the cost model to use an artificially
4839 // high enough value to practically disable vectorization with such
4840 // operations, except where previously deployed legality hack allowed
4841 // using very low cost values. This is to avoid regressions coming simply
4842 // from moving "masked load/store" check from legality to cost model.
4843 // Masked Load/Gather emulation was previously never allowed.
4844 // Limited number of Masked Store/Scatter emulation was allowed.
4845 assert((isPredicatedInst(I)) &&
4846 "Expecting a scalar emulated instruction");
4847 return isa<LoadInst>(I) ||
4848 (isa<StoreInst>(I) &&
4849 NumPredStores > NumberOfStoresToPredicate);
4850}
4851
4853 assert(VF.isVector() && "Expected VF >= 2");
4854
4855 // If we've already collected the instructions to scalarize or the predicated
4856 // BBs after vectorization, there's nothing to do. Collection may already have
4857 // occurred if we have a user-selected VF and are now computing the expected
4858 // cost for interleaving.
4859 if (InstsToScalarize.contains(VF) ||
4860 PredicatedBBsAfterVectorization.contains(VF))
4861 return;
4862
4863 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
4864 // not profitable to scalarize any instructions, the presence of VF in the
4865 // map will indicate that we've analyzed it already.
4866 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4867
4868 // Find all the instructions that are scalar with predication in the loop and
4869 // determine if it would be better to not if-convert the blocks they are in.
4870 // If so, we also record the instructions to scalarize.
4871 for (BasicBlock *BB : TheLoop->blocks()) {
4873 continue;
4874 for (Instruction &I : *BB)
4875 if (isScalarWithPredication(&I, VF)) {
4876 ScalarCostsTy ScalarCosts;
4877 // Do not apply discount logic for:
4878 // 1. Scalars after vectorization, as there will only be a single copy
4879 // of the instruction.
4880 // 2. Scalable VF, as that would lead to invalid scalarization costs.
4881 // 3. Emulated masked memrefs, if a hacked cost is needed.
4882 if (!isScalarAfterVectorization(&I, VF) && !VF.isScalable() &&
4883 !useEmulatedMaskMemRefHack(&I, VF) &&
4884 computePredInstDiscount(&I, ScalarCosts, VF) >= 0) {
4885 for (const auto &[I, IC] : ScalarCosts)
4886 ScalarCostsVF.insert({I, IC});
4887 // Check if we decided to scalarize a call. If so, update the widening
4888 // decision of the call to CM_Scalarize with the computed scalar cost.
4889 for (const auto &[I, Cost] : ScalarCosts) {
4890 auto *CI = dyn_cast<CallInst>(I);
4891 if (!CI || !CallWideningDecisions.contains({CI, VF}))
4892 continue;
4893 CallWideningDecisions[{CI, VF}].Kind = CM_Scalarize;
4894 CallWideningDecisions[{CI, VF}].Cost = Cost;
4895 }
4896 }
4897 // Remember that BB will remain after vectorization.
4898 PredicatedBBsAfterVectorization[VF].insert(BB);
4899 for (auto *Pred : predecessors(BB)) {
4900 if (Pred->getSingleSuccessor() == BB)
4901 PredicatedBBsAfterVectorization[VF].insert(Pred);
4902 }
4903 }
4904 }
4905}
4906
4907InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
4908 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
4909 assert(!isUniformAfterVectorization(PredInst, VF) &&
4910 "Instruction marked uniform-after-vectorization will be predicated");
4911
4912 // Initialize the discount to zero, meaning that the scalar version and the
4913 // vector version cost the same.
4914 InstructionCost Discount = 0;
4915
4916 // Holds instructions to analyze. The instructions we visit are mapped in
4917 // ScalarCosts. Those instructions are the ones that would be scalarized if
4918 // we find that the scalar version costs less.
4920
4921 // Returns true if the given instruction can be scalarized.
4922 auto CanBeScalarized = [&](Instruction *I) -> bool {
4923 // We only attempt to scalarize instructions forming a single-use chain
4924 // from the original predicated block that would otherwise be vectorized.
4925 // Although not strictly necessary, we give up on instructions we know will
4926 // already be scalar to avoid traversing chains that are unlikely to be
4927 // beneficial.
4928 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
4929 isScalarAfterVectorization(I, VF))
4930 return false;
4931
4932 // If the instruction is scalar with predication, it will be analyzed
4933 // separately. We ignore it within the context of PredInst.
4934 if (isScalarWithPredication(I, VF))
4935 return false;
4936
4937 // If any of the instruction's operands are uniform after vectorization,
4938 // the instruction cannot be scalarized. This prevents, for example, a
4939 // masked load from being scalarized.
4940 //
4941 // We assume we will only emit a value for lane zero of an instruction
4942 // marked uniform after vectorization, rather than VF identical values.
4943 // Thus, if we scalarize an instruction that uses a uniform, we would
4944 // create uses of values corresponding to the lanes we aren't emitting code
4945 // for. This behavior can be changed by allowing getScalarValue to clone
4946 // the lane zero values for uniforms rather than asserting.
4947 for (Use &U : I->operands())
4948 if (auto *J = dyn_cast<Instruction>(U.get()))
4949 if (isUniformAfterVectorization(J, VF))
4950 return false;
4951
4952 // Otherwise, we can scalarize the instruction.
4953 return true;
4954 };
4955
4956 // Compute the expected cost discount from scalarizing the entire expression
4957 // feeding the predicated instruction. We currently only consider expressions
4958 // that are single-use instruction chains.
4959 Worklist.push_back(PredInst);
4960 while (!Worklist.empty()) {
4961 Instruction *I = Worklist.pop_back_val();
4962
4963 // If we've already analyzed the instruction, there's nothing to do.
4964 if (ScalarCosts.contains(I))
4965 continue;
4966
4967 // Cannot scalarize fixed-order recurrence phis at the moment.
4968 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
4969 continue;
4970
4971 // Compute the cost of the vector instruction. Note that this cost already
4972 // includes the scalarization overhead of the predicated instruction.
4973 InstructionCost VectorCost = getInstructionCost(I, VF);
4974
4975 // Compute the cost of the scalarized instruction. This cost is the cost of
4976 // the instruction as if it wasn't if-converted and instead remained in the
4977 // predicated block. We will scale this cost by block probability after
4978 // computing the scalarization overhead.
4979 InstructionCost ScalarCost =
4980 VF.getFixedValue() * getInstructionCost(I, ElementCount::getFixed(1));
4981
4982 // Compute the scalarization overhead of needed insertelement instructions
4983 // and phi nodes.
4984 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
4985 Type *WideTy = toVectorizedTy(I->getType(), VF);
4986 for (Type *VectorTy : getContainedTypes(WideTy)) {
4987 ScalarCost += TTI.getScalarizationOverhead(
4989 /*Insert=*/true,
4990 /*Extract=*/false, CostKind);
4991 }
4992 ScalarCost +=
4993 VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
4994 }
4995
4996 // Compute the scalarization overhead of needed extractelement
4997 // instructions. For each of the instruction's operands, if the operand can
4998 // be scalarized, add it to the worklist; otherwise, account for the
4999 // overhead.
5000 for (Use &U : I->operands())
5001 if (auto *J = dyn_cast<Instruction>(U.get())) {
5002 assert(canVectorizeTy(J->getType()) &&
5003 "Instruction has non-scalar type");
5004 if (CanBeScalarized(J))
5005 Worklist.push_back(J);
5006 else if (needsExtract(J, VF)) {
5007 Type *WideTy = toVectorizedTy(J->getType(), VF);
5008 for (Type *VectorTy : getContainedTypes(WideTy)) {
5009 ScalarCost += TTI.getScalarizationOverhead(
5010 cast<VectorType>(VectorTy),
5011 APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
5012 /*Extract*/ true, CostKind);
5013 }
5014 }
5015 }
5016
5017 // Scale the total scalar cost by block probability.
5018 ScalarCost /= getPredBlockCostDivisor(CostKind);
5019
5020 // Compute the discount. A non-negative discount means the vector version
5021 // of the instruction costs more, and scalarizing would be beneficial.
5022 Discount += VectorCost - ScalarCost;
5023 ScalarCosts[I] = ScalarCost;
5024 }
5025
5026 return Discount;
5027}
5028
5031
5032 // If the vector loop gets executed exactly once with the given VF, ignore the
5033 // costs of comparison and induction instructions, as they'll get simplified
5034 // away.
5035 SmallPtrSet<Instruction *, 2> ValuesToIgnoreForVF;
5036 auto TC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
5037 if (TC == VF && !foldTailByMasking())
5039 ValuesToIgnoreForVF);
5040
5041 // For each block.
5042 for (BasicBlock *BB : TheLoop->blocks()) {
5043 InstructionCost BlockCost;
5044
5045 // For each instruction in the old loop.
5046 for (Instruction &I : BB->instructionsWithoutDebug()) {
5047 // Skip ignored values.
5048 if (ValuesToIgnore.count(&I) || ValuesToIgnoreForVF.count(&I) ||
5049 (VF.isVector() && VecValuesToIgnore.count(&I)))
5050 continue;
5051
5053
5054 // Check if we should override the cost.
5055 if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
5057
5058 BlockCost += C;
5059 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
5060 << VF << " For instruction: " << I << '\n');
5061 }
5062
5063 // If we are vectorizing a predicated block, it will have been
5064 // if-converted. This means that the block's instructions (aside from
5065 // stores and instructions that may divide by zero) will now be
5066 // unconditionally executed. For the scalar case, we may not always execute
5067 // the predicated block, if it is an if-else block. Thus, scale the block's
5068 // cost by the probability of executing it. blockNeedsPredication from
5069 // Legal is used so as to not include all blocks in tail folded loops.
5070 if (VF.isScalar() && Legal->blockNeedsPredication(BB))
5071 BlockCost /= getPredBlockCostDivisor(CostKind);
5072
5073 Cost += BlockCost;
5074 }
5075
5076 return Cost;
5077}
5078
5079/// Gets Address Access SCEV after verifying that the access pattern
5080/// is loop invariant except the induction variable dependence.
5081///
5082/// This SCEV can be sent to the Target in order to estimate the address
5083/// calculation cost.
5085 Value *Ptr,
5088 const Loop *TheLoop) {
5089
5090 auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);
5091 if (!Gep)
5092 return nullptr;
5093
5094 // We are looking for a gep with all loop invariant indices except for one
5095 // which should be an induction variable.
5096 auto *SE = PSE.getSE();
5097 unsigned NumOperands = Gep->getNumOperands();
5098 for (unsigned Idx = 1; Idx < NumOperands; ++Idx) {
5099 Value *Opd = Gep->getOperand(Idx);
5100 if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
5101 !Legal->isInductionVariable(Opd))
5102 return nullptr;
5103 }
5104
5105 // Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.
5106 return PSE.getSCEV(Ptr);
5107}
5108
5110LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
5111 ElementCount VF) {
5112 assert(VF.isVector() &&
5113 "Scalarization cost of instruction implies vectorization.");
5114 if (VF.isScalable())
5115 return InstructionCost::getInvalid();
5116
5117 Type *ValTy = getLoadStoreType(I);
5118 auto *SE = PSE.getSE();
5119
5120 unsigned AS = getLoadStoreAddressSpace(I);
5122 Type *PtrTy = toVectorTy(Ptr->getType(), VF);
5123 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
5124 // that it is being called from this specific place.
5125
5126 // Figure out whether the access is strided and get the stride value
5127 // if it's known in compile time
5128 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);
5129
5130 // Get the cost of the scalar memory instruction and address computation.
5132 PtrTy, SE, PtrSCEV, CostKind);
5133
5134 // Don't pass *I here, since it is scalar but will actually be part of a
5135 // vectorized loop where the user of it is a vectorized instruction.
5136 const Align Alignment = getLoadStoreAlignment(I);
5137 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5138 Cost += VF.getFixedValue() *
5139 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
5140 AS, CostKind, OpInfo);
5141
5142 // Get the overhead of the extractelement and insertelement instructions
5143 // we might create due to scalarization.
5145
5146 // If we have a predicated load/store, it will need extra i1 extracts and
5147 // conditional branches, but may not be executed for each vector lane. Scale
5148 // the cost by the probability of executing the predicated block.
5149 if (isPredicatedInst(I)) {
5151
5152 // Add the cost of an i1 extract and a branch
5153 auto *VecI1Ty =
5154 VectorType::get(IntegerType::getInt1Ty(ValTy->getContext()), VF);
5156 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
5157 /*Insert=*/false, /*Extract=*/true, CostKind);
5158 Cost += TTI.getCFInstrCost(Instruction::Br, CostKind);
5159
5160 if (useEmulatedMaskMemRefHack(I, VF))
5161 // Artificially setting to a high enough value to practically disable
5162 // vectorization with such operations.
5163 Cost = 3000000;
5164 }
5165
5166 return Cost;
5167}
5168
5170LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I,
5171 ElementCount VF) {
5172 Type *ValTy = getLoadStoreType(I);
5173 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5175 unsigned AS = getLoadStoreAddressSpace(I);
5176 int ConsecutiveStride = Legal->isConsecutivePtr(ValTy, Ptr);
5177
5178 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5179 "Stride should be 1 or -1 for consecutive memory access");
5180 const Align Alignment = getLoadStoreAlignment(I);
5182 if (Legal->isMaskRequired(I)) {
5183 Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
5184 CostKind);
5185 } else {
5186 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5187 Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
5188 CostKind, OpInfo, I);
5189 }
5190
5191 bool Reverse = ConsecutiveStride < 0;
5192 if (Reverse)
5194 VectorTy, {}, CostKind, 0);
5195 return Cost;
5196}
5197
5199LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
5200 ElementCount VF) {
5201 assert(Legal->isUniformMemOp(*I, VF));
5202
5203 Type *ValTy = getLoadStoreType(I);
5205 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5206 const Align Alignment = getLoadStoreAlignment(I);
5207 unsigned AS = getLoadStoreAddressSpace(I);
5208 if (isa<LoadInst>(I)) {
5209 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5210 TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
5211 CostKind) +
5213 VectorTy, {}, CostKind);
5214 }
5215 StoreInst *SI = cast<StoreInst>(I);
5216
5217 bool IsLoopInvariantStoreValue = Legal->isInvariant(SI->getValueOperand());
5218 // TODO: We have existing tests that request the cost of extracting element
5219 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
5220 // the actual generated code, which involves extracting the last element of
5221 // a scalable vector where the lane to extract is unknown at compile time.
5223 TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5224 TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS, CostKind);
5225 if (!IsLoopInvariantStoreValue)
5226 Cost += TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
5227 VectorTy, CostKind, 0);
5228 return Cost;
5229}
5230
5232LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
5233 ElementCount VF) {
5234 Type *ValTy = getLoadStoreType(I);
5235 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5236 const Align Alignment = getLoadStoreAlignment(I);
5238 Type *PtrTy = Ptr->getType();
5239
5240 if (!Legal->isUniform(Ptr, VF))
5241 PtrTy = toVectorTy(PtrTy, VF);
5242
5243 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5244 TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
5245 Legal->isMaskRequired(I), Alignment,
5246 CostKind, I);
5247}
5248
5250LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
5251 ElementCount VF) {
5252 const auto *Group = getInterleavedAccessGroup(I);
5253 assert(Group && "Fail to get an interleaved access group.");
5254
5255 Instruction *InsertPos = Group->getInsertPos();
5256 Type *ValTy = getLoadStoreType(InsertPos);
5257 auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
5258 unsigned AS = getLoadStoreAddressSpace(InsertPos);
5259
5260 unsigned InterleaveFactor = Group->getFactor();
5261 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);
5262
5263 // Holds the indices of existing members in the interleaved group.
5264 SmallVector<unsigned, 4> Indices;
5265 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
5266 if (Group->getMember(IF))
5267 Indices.push_back(IF);
5268
5269 // Calculate the cost of the whole interleaved group.
5270 bool UseMaskForGaps =
5271 (Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed()) ||
5272 (isa<StoreInst>(I) && !Group->isFull());
5274 InsertPos->getOpcode(), WideVecTy, Group->getFactor(), Indices,
5275 Group->getAlign(), AS, CostKind, Legal->isMaskRequired(I),
5276 UseMaskForGaps);
5277
5278 if (Group->isReverse()) {
5279 // TODO: Add support for reversed masked interleaved access.
5280 assert(!Legal->isMaskRequired(I) &&
5281 "Reverse masked interleaved access not supported.");
5282 Cost += Group->getNumMembers() *
5284 VectorTy, {}, CostKind, 0);
5285 }
5286 return Cost;
5287}
5288
5289std::optional<InstructionCost>
5291 ElementCount VF,
5292 Type *Ty) const {
5293 using namespace llvm::PatternMatch;
5294 // Early exit for no inloop reductions
5295 if (InLoopReductions.empty() || VF.isScalar() || !isa<VectorType>(Ty))
5296 return std::nullopt;
5297 auto *VectorTy = cast<VectorType>(Ty);
5298
5299 // We are looking for a pattern of, and finding the minimal acceptable cost:
5300 // reduce(mul(ext(A), ext(B))) or
5301 // reduce(mul(A, B)) or
5302 // reduce(ext(A)) or
5303 // reduce(A).
5304 // The basic idea is that we walk down the tree to do that, finding the root
5305 // reduction instruction in InLoopReductionImmediateChains. From there we find
5306 // the pattern of mul/ext and test the cost of the entire pattern vs the cost
5307 // of the components. If the reduction cost is lower then we return it for the
5308 // reduction instruction and 0 for the other instructions in the pattern. If
5309 // it is not we return an invalid cost specifying the orignal cost method
5310 // should be used.
5311 Instruction *RetI = I;
5312 if (match(RetI, m_ZExtOrSExt(m_Value()))) {
5313 if (!RetI->hasOneUser())
5314 return std::nullopt;
5315 RetI = RetI->user_back();
5316 }
5317
5318 if (match(RetI, m_OneUse(m_Mul(m_Value(), m_Value()))) &&
5319 RetI->user_back()->getOpcode() == Instruction::Add) {
5320 RetI = RetI->user_back();
5321 }
5322
5323 // Test if the found instruction is a reduction, and if not return an invalid
5324 // cost specifying the parent to use the original cost modelling.
5325 Instruction *LastChain = InLoopReductionImmediateChains.lookup(RetI);
5326 if (!LastChain)
5327 return std::nullopt;
5328
5329 // Find the reduction this chain is a part of and calculate the basic cost of
5330 // the reduction on its own.
5331 Instruction *ReductionPhi = LastChain;
5332 while (!isa<PHINode>(ReductionPhi))
5333 ReductionPhi = InLoopReductionImmediateChains.at(ReductionPhi);
5334
5335 const RecurrenceDescriptor &RdxDesc =
5336 Legal->getRecurrenceDescriptor(cast<PHINode>(ReductionPhi));
5337
5338 InstructionCost BaseCost;
5339 RecurKind RK = RdxDesc.getRecurrenceKind();
5342 BaseCost = TTI.getMinMaxReductionCost(MinMaxID, VectorTy,
5343 RdxDesc.getFastMathFlags(), CostKind);
5344 } else {
5345 BaseCost = TTI.getArithmeticReductionCost(
5346 RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind);
5347 }
5348
5349 // For a call to the llvm.fmuladd intrinsic we need to add the cost of a
5350 // normal fmul instruction to the cost of the fadd reduction.
5351 if (RK == RecurKind::FMulAdd)
5352 BaseCost +=
5353 TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy, CostKind);
5354
5355 // If we're using ordered reductions then we can just return the base cost
5356 // here, since getArithmeticReductionCost calculates the full ordered
5357 // reduction cost when FP reassociation is not allowed.
5358 if (useOrderedReductions(RdxDesc))
5359 return BaseCost;
5360
5361 // Get the operand that was not the reduction chain and match it to one of the
5362 // patterns, returning the better cost if it is found.
5363 Instruction *RedOp = RetI->getOperand(1) == LastChain
5366
5367 VectorTy = VectorType::get(I->getOperand(0)->getType(), VectorTy);
5368
5369 Instruction *Op0, *Op1;
5370 if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
5371 match(RedOp,
5373 match(Op0, m_ZExtOrSExt(m_Value())) &&
5374 Op0->getOpcode() == Op1->getOpcode() &&
5375 Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
5376 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1) &&
5377 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
5378
5379 // Matched reduce.add(ext(mul(ext(A), ext(B)))
5380 // Note that the extend opcodes need to all match, or if A==B they will have
5381 // been converted to zext(mul(sext(A), sext(A))) as it is known positive,
5382 // which is equally fine.
5383 bool IsUnsigned = isa<ZExtInst>(Op0);
5384 auto *ExtType = VectorType::get(Op0->getOperand(0)->getType(), VectorTy);
5385 auto *MulType = VectorType::get(Op0->getType(), VectorTy);
5386
5387 InstructionCost ExtCost =
5388 TTI.getCastInstrCost(Op0->getOpcode(), MulType, ExtType,
5390 InstructionCost MulCost =
5391 TTI.getArithmeticInstrCost(Instruction::Mul, MulType, CostKind);
5392 InstructionCost Ext2Cost =
5393 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, MulType,
5395
5396 InstructionCost RedCost = TTI.getMulAccReductionCost(
5397 IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
5398 CostKind);
5399
5400 if (RedCost.isValid() &&
5401 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
5402 return I == RetI ? RedCost : 0;
5403 } else if (RedOp && match(RedOp, m_ZExtOrSExt(m_Value())) &&
5404 !TheLoop->isLoopInvariant(RedOp)) {
5405 // Matched reduce(ext(A))
5406 bool IsUnsigned = isa<ZExtInst>(RedOp);
5407 auto *ExtType = VectorType::get(RedOp->getOperand(0)->getType(), VectorTy);
5408 InstructionCost RedCost = TTI.getExtendedReductionCost(
5409 RdxDesc.getOpcode(), IsUnsigned, RdxDesc.getRecurrenceType(), ExtType,
5410 RdxDesc.getFastMathFlags(), CostKind);
5411
5412 InstructionCost ExtCost =
5413 TTI.getCastInstrCost(RedOp->getOpcode(), VectorTy, ExtType,
5415 if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
5416 return I == RetI ? RedCost : 0;
5417 } else if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
5418 match(RedOp, m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) {
5419 if (match(Op0, m_ZExtOrSExt(m_Value())) &&
5420 Op0->getOpcode() == Op1->getOpcode() &&
5421 !TheLoop->isLoopInvariant(Op0) && !TheLoop->isLoopInvariant(Op1)) {
5422 bool IsUnsigned = isa<ZExtInst>(Op0);
5423 Type *Op0Ty = Op0->getOperand(0)->getType();
5424 Type *Op1Ty = Op1->getOperand(0)->getType();
5425 Type *LargestOpTy =
5426 Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
5427 : Op0Ty;
5428 auto *ExtType = VectorType::get(LargestOpTy, VectorTy);
5429
5430 // Matched reduce.add(mul(ext(A), ext(B))), where the two ext may be of
5431 // different sizes. We take the largest type as the ext to reduce, and add
5432 // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
5433 InstructionCost ExtCost0 = TTI.getCastInstrCost(
5434 Op0->getOpcode(), VectorTy, VectorType::get(Op0Ty, VectorTy),
5436 InstructionCost ExtCost1 = TTI.getCastInstrCost(
5437 Op1->getOpcode(), VectorTy, VectorType::get(Op1Ty, VectorTy),
5439 InstructionCost MulCost =
5440 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
5441
5442 InstructionCost RedCost = TTI.getMulAccReductionCost(
5443 IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
5444 CostKind);
5445 InstructionCost ExtraExtCost = 0;
5446 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
5447 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
5448 ExtraExtCost = TTI.getCastInstrCost(
5449 ExtraExtOp->getOpcode(), ExtType,
5450 VectorType::get(ExtraExtOp->getOperand(0)->getType(), VectorTy),
5452 }
5453
5454 if (RedCost.isValid() &&
5455 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
5456 return I == RetI ? RedCost : 0;
5457 } else if (!match(I, m_ZExtOrSExt(m_Value()))) {
5458 // Matched reduce.add(mul())
5459 InstructionCost MulCost =
5460 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
5461
5462 InstructionCost RedCost = TTI.getMulAccReductionCost(
5463 true, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), VectorTy,
5464 CostKind);
5465
5466 if (RedCost.isValid() && RedCost < MulCost + BaseCost)
5467 return I == RetI ? RedCost : 0;
5468 }
5469 }
5470
5471 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
5472}
5473
5475LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
5476 ElementCount VF) {
5477 // Calculate scalar cost only. Vectorization cost should be ready at this
5478 // moment.
5479 if (VF.isScalar()) {
5480 Type *ValTy = getLoadStoreType(I);
5482 const Align Alignment = getLoadStoreAlignment(I);
5483 unsigned AS = getLoadStoreAddressSpace(I);
5484
5485 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
5486 return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
5487 TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, CostKind,
5488 OpInfo, I);
5489 }
5490 return getWideningCost(I, VF);
5491}
5492
5494LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
5495 ElementCount VF) const {
5496
5497 // There is no mechanism yet to create a scalable scalarization loop,
5498 // so this is currently Invalid.
5499 if (VF.isScalable())
5500 return InstructionCost::getInvalid();
5501
5502 if (VF.isScalar())
5503 return 0;
5504
5506 Type *RetTy = toVectorizedTy(I->getType(), VF);
5507 if (!RetTy->isVoidTy() &&
5509
5510 for (Type *VectorTy : getContainedTypes(RetTy)) {
5513 /*Insert=*/true,
5514 /*Extract=*/false, CostKind);
5515 }
5516 }
5517
5518 // Some targets keep addresses scalar.
5520 return Cost;
5521
5522 // Some targets support efficient element stores.
5524 return Cost;
5525
5526 // Collect operands to consider.
5527 CallInst *CI = dyn_cast<CallInst>(I);
5528 Instruction::op_range Ops = CI ? CI->args() : I->operands();
5529
5530 // Skip operands that do not require extraction/scalarization and do not incur
5531 // any overhead.
5533 for (auto *V : filterExtractingOperands(Ops, VF))
5534 Tys.push_back(maybeVectorizeType(V->getType(), VF));
5536}
5537
5539 if (VF.isScalar())
5540 return;
5541 NumPredStores = 0;
5542 for (BasicBlock *BB : TheLoop->blocks()) {
5543 // For each instruction in the old loop.
5544 for (Instruction &I : *BB) {
5546 if (!Ptr)
5547 continue;
5548
5549 // TODO: We should generate better code and update the cost model for
5550 // predicated uniform stores. Today they are treated as any other
5551 // predicated store (see added test cases in
5552 // invariant-store-vectorization.ll).
5554 NumPredStores++;
5555
5556 if (Legal->isUniformMemOp(I, VF)) {
5557 auto IsLegalToScalarize = [&]() {
5558 if (!VF.isScalable())
5559 // Scalarization of fixed length vectors "just works".
5560 return true;
5561
5562 // We have dedicated lowering for unpredicated uniform loads and
5563 // stores. Note that even with tail folding we know that at least
5564 // one lane is active (i.e. generalized predication is not possible
5565 // here), and the logic below depends on this fact.
5566 if (!foldTailByMasking())
5567 return true;
5568
5569 // For scalable vectors, a uniform memop load is always
5570 // uniform-by-parts and we know how to scalarize that.
5571 if (isa<LoadInst>(I))
5572 return true;
5573
5574 // A uniform store isn't neccessarily uniform-by-part
5575 // and we can't assume scalarization.
5576 auto &SI = cast<StoreInst>(I);
5577 return TheLoop->isLoopInvariant(SI.getValueOperand());
5578 };
5579
5580 const InstructionCost GatherScatterCost =
5582 getGatherScatterCost(&I, VF) : InstructionCost::getInvalid();
5583
5584 // Load: Scalar load + broadcast
5585 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
5586 // FIXME: This cost is a significant under-estimate for tail folded
5587 // memory ops.
5588 const InstructionCost ScalarizationCost =
5589 IsLegalToScalarize() ? getUniformMemOpCost(&I, VF)
5591
5592 // Choose better solution for the current VF, Note that Invalid
5593 // costs compare as maximumal large. If both are invalid, we get
5594 // scalable invalid which signals a failure and a vectorization abort.
5595 if (GatherScatterCost < ScalarizationCost)
5596 setWideningDecision(&I, VF, CM_GatherScatter, GatherScatterCost);
5597 else
5598 setWideningDecision(&I, VF, CM_Scalarize, ScalarizationCost);
5599 continue;
5600 }
5601
5602 // We assume that widening is the best solution when possible.
5603 if (memoryInstructionCanBeWidened(&I, VF)) {
5604 InstructionCost Cost = getConsecutiveMemOpCost(&I, VF);
5605 int ConsecutiveStride = Legal->isConsecutivePtr(
5607 assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
5608 "Expected consecutive stride.");
5609 InstWidening Decision =
5610 ConsecutiveStride == 1 ? CM_Widen : CM_Widen_Reverse;
5611 setWideningDecision(&I, VF, Decision, Cost);
5612 continue;
5613 }
5614
5615 // Choose between Interleaving, Gather/Scatter or Scalarization.
5617 unsigned NumAccesses = 1;
5618 if (isAccessInterleaved(&I)) {
5619 const auto *Group = getInterleavedAccessGroup(&I);
5620 assert(Group && "Fail to get an interleaved access group.");
5621
5622 // Make one decision for the whole group.
5623 if (getWideningDecision(&I, VF) != CM_Unknown)
5624 continue;
5625
5626 NumAccesses = Group->getNumMembers();
5628 InterleaveCost = getInterleaveGroupCost(&I, VF);
5629 }
5630
5631 InstructionCost GatherScatterCost =
5633 ? getGatherScatterCost(&I, VF) * NumAccesses
5635
5636 InstructionCost ScalarizationCost =
5637 getMemInstScalarizationCost(&I, VF) * NumAccesses;
5638
5639 // Choose better solution for the current VF,
5640 // write down this decision and use it during vectorization.
5642 InstWidening Decision;
5643 if (InterleaveCost <= GatherScatterCost &&
5644 InterleaveCost < ScalarizationCost) {
5645 Decision = CM_Interleave;
5646 Cost = InterleaveCost;
5647 } else if (GatherScatterCost < ScalarizationCost) {
5648 Decision = CM_GatherScatter;
5649 Cost = GatherScatterCost;
5650 } else {
5651 Decision = CM_Scalarize;
5652 Cost = ScalarizationCost;
5653 }
5654 // If the instructions belongs to an interleave group, the whole group
5655 // receives the same decision. The whole group receives the cost, but
5656 // the cost will actually be assigned to one instruction.
5657 if (const auto *Group = getInterleavedAccessGroup(&I)) {
5658 if (Decision == CM_Scalarize) {
5659 for (unsigned Idx = 0; Idx < Group->getFactor(); ++Idx) {
5660 if (auto *I = Group->getMember(Idx)) {
5661 setWideningDecision(I, VF, Decision,
5662 getMemInstScalarizationCost(I, VF));
5663 }
5664 }
5665 } else {
5666 setWideningDecision(Group, VF, Decision, Cost);
5667 }
5668 } else
5669 setWideningDecision(&I, VF, Decision, Cost);
5670 }
5671 }
5672
5673 // Make sure that any load of address and any other address computation
5674 // remains scalar unless there is gather/scatter support. This avoids
5675 // inevitable extracts into address registers, and also has the benefit of
5676 // activating LSR more, since that pass can't optimize vectorized
5677 // addresses.
5678 if (TTI.prefersVectorizedAddressing())
5679 return;
5680
5681 // Start with all scalar pointer uses.
5683 for (BasicBlock *BB : TheLoop->blocks())
5684 for (Instruction &I : *BB) {
5685 Instruction *PtrDef =
5687 if (PtrDef && TheLoop->contains(PtrDef) &&
5689 AddrDefs.insert(PtrDef);
5690 }
5691
5692 // Add all instructions used to generate the addresses.
5694 append_range(Worklist, AddrDefs);
5695 while (!Worklist.empty()) {
5696 Instruction *I = Worklist.pop_back_val();
5697 for (auto &Op : I->operands())
5698 if (auto *InstOp = dyn_cast<Instruction>(Op))
5699 if ((InstOp->getParent() == I->getParent()) && !isa<PHINode>(InstOp) &&
5700 AddrDefs.insert(InstOp).second)
5701 Worklist.push_back(InstOp);
5702 }
5703
5704 for (auto *I : AddrDefs) {
5705 if (isa<LoadInst>(I)) {
5706 // Setting the desired widening decision should ideally be handled in
5707 // by cost functions, but since this involves the task of finding out
5708 // if the loaded register is involved in an address computation, it is
5709 // instead changed here when we know this is the case.
5710 InstWidening Decision = getWideningDecision(I, VF);
5711 if (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
5712 (!isPredicatedInst(I) && !Legal->isUniformMemOp(*I, VF) &&
5713 Decision == CM_Scalarize))
5714 // Scalarize a widened load of address or update the cost of a scalar
5715 // load of an address.
5717 I, VF, CM_Scalarize,
5718 (VF.getKnownMinValue() *
5719 getMemoryInstructionCost(I, ElementCount::getFixed(1))));
5720 else if (const auto *Group = getInterleavedAccessGroup(I)) {
5721 // Scalarize an interleave group of address loads.
5722 for (unsigned I = 0; I < Group->getFactor(); ++I) {
5723 if (Instruction *Member = Group->getMember(I))
5725 Member, VF, CM_Scalarize,
5726 (VF.getKnownMinValue() *
5727 getMemoryInstructionCost(Member, ElementCount::getFixed(1))));
5728 }
5729 }
5730 } else {
5731 // Cannot scalarize fixed-order recurrence phis at the moment.
5732 if (isa<PHINode>(I) && Legal->isFixedOrderRecurrence(cast<PHINode>(I)))
5733 continue;
5734
5735 // Make sure I gets scalarized and a cost estimate without
5736 // scalarization overhead.
5737 ForcedScalars[VF].insert(I);
5738 }
5739 }
5740}
5741
5743 assert(!VF.isScalar() &&
5744 "Trying to set a vectorization decision for a scalar VF");
5745
5746 auto ForcedScalar = ForcedScalars.find(VF);
5747 for (BasicBlock *BB : TheLoop->blocks()) {
5748 // For each instruction in the old loop.
5749 for (Instruction &I : *BB) {
5751
5752 if (!CI)
5753 continue;
5754
5758 Function *ScalarFunc = CI->getCalledFunction();
5759 Type *ScalarRetTy = CI->getType();
5760 SmallVector<Type *, 4> Tys, ScalarTys;
5761 for (auto &ArgOp : CI->args())
5762 ScalarTys.push_back(ArgOp->getType());
5763
5764 // Estimate cost of scalarized vector call. The source operands are
5765 // assumed to be vectors, so we need to extract individual elements from
5766 // there, execute VF scalar calls, and then gather the result into the
5767 // vector return value.
5768 if (VF.isFixed()) {
5769 InstructionCost ScalarCallCost =
5770 TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys, CostKind);
5771
5772 // Compute costs of unpacking argument values for the scalar calls and
5773 // packing the return values to a vector.
5774 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
5775 ScalarCost = ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
5776 } else {
5777 // There is no point attempting to calculate the scalar cost for a
5778 // scalable VF as we know it will be Invalid.
5780 "Unexpected valid cost for scalarizing scalable vectors");
5781 ScalarCost = InstructionCost::getInvalid();
5782 }
5783
5784 // Honor ForcedScalars and UniformAfterVectorization decisions.
5785 // TODO: For calls, it might still be more profitable to widen. Use
5786 // VPlan-based cost model to compare different options.
5787 if (VF.isVector() && ((ForcedScalar != ForcedScalars.end() &&
5788 ForcedScalar->second.contains(CI)) ||
5789 isUniformAfterVectorization(CI, VF))) {
5790 setCallWideningDecision(CI, VF, CM_Scalarize, nullptr,
5791 Intrinsic::not_intrinsic, std::nullopt,
5792 ScalarCost);
5793 continue;
5794 }
5795
5796 bool MaskRequired = Legal->isMaskRequired(CI);
5797 // Compute corresponding vector type for return value and arguments.
5798 Type *RetTy = toVectorizedTy(ScalarRetTy, VF);
5799 for (Type *ScalarTy : ScalarTys)
5800 Tys.push_back(toVectorizedTy(ScalarTy, VF));
5801
5802 // An in-loop reduction using an fmuladd intrinsic is a special case;
5803 // we don't want the normal cost for that intrinsic.
5805 if (auto RedCost = getReductionPatternCost(CI, VF, RetTy)) {
5808 std::nullopt, *RedCost);
5809 continue;
5810 }
5811
5812 // Find the cost of vectorizing the call, if we can find a suitable
5813 // vector variant of the function.
5814 VFInfo FuncInfo;
5815 Function *VecFunc = nullptr;
5816 // Search through any available variants for one we can use at this VF.
5817 for (VFInfo &Info : VFDatabase::getMappings(*CI)) {
5818 // Must match requested VF.
5819 if (Info.Shape.VF != VF)
5820 continue;
5821
5822 // Must take a mask argument if one is required
5823 if (MaskRequired && !Info.isMasked())
5824 continue;
5825
5826 // Check that all parameter kinds are supported
5827 bool ParamsOk = true;
5828 for (VFParameter Param : Info.Shape.Parameters) {
5829 switch (Param.ParamKind) {
5831 break;
5833 Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
5834 // Make sure the scalar parameter in the loop is invariant.
5835 if (!PSE.getSE()->isLoopInvariant(PSE.getSCEV(ScalarParam),
5836 TheLoop))
5837 ParamsOk = false;
5838 break;
5839 }
5841 Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
5842 // Find the stride for the scalar parameter in this loop and see if
5843 // it matches the stride for the variant.
5844 // TODO: do we need to figure out the cost of an extract to get the
5845 // first lane? Or do we hope that it will be folded away?
5846 ScalarEvolution *SE = PSE.getSE();
5847 if (!match(SE->getSCEV(ScalarParam),
5849 m_SCEV(), m_scev_SpecificSInt(Param.LinearStepOrPos),
5851 ParamsOk = false;
5852 break;
5853 }
5855 break;
5856 default:
5857 ParamsOk = false;
5858 break;
5859 }
5860 }
5861
5862 if (!ParamsOk)
5863 continue;
5864
5865 // Found a suitable candidate, stop here.
5866 VecFunc = CI->getModule()->getFunction(Info.VectorName);
5867 FuncInfo = Info;
5868 break;
5869 }
5870
5871 if (TLI && VecFunc && !CI->isNoBuiltin())
5872 VectorCost = TTI.getCallInstrCost(nullptr, RetTy, Tys, CostKind);
5873
5874 // Find the cost of an intrinsic; some targets may have instructions that
5875 // perform the operation without needing an actual call.
5877 if (IID != Intrinsic::not_intrinsic)
5879
5880 InstructionCost Cost = ScalarCost;
5881 InstWidening Decision = CM_Scalarize;
5882
5883 if (VectorCost <= Cost) {
5884 Cost = VectorCost;
5885 Decision = CM_VectorCall;
5886 }
5887
5888 if (IntrinsicCost <= Cost) {
5890 Decision = CM_IntrinsicCall;
5891 }
5892
5893 setCallWideningDecision(CI, VF, Decision, VecFunc, IID,
5895 }
5896 }
5897}
5898
5900 if (!Legal->isInvariant(Op))
5901 return false;
5902 // Consider Op invariant, if it or its operands aren't predicated
5903 // instruction in the loop. In that case, it is not trivially hoistable.
5904 auto *OpI = dyn_cast<Instruction>(Op);
5905 return !OpI || !TheLoop->contains(OpI) ||
5906 (!isPredicatedInst(OpI) &&
5907 (!isa<PHINode>(OpI) || OpI->getParent() != TheLoop->getHeader()) &&
5908 all_of(OpI->operands(),
5909 [this](Value *Op) { return shouldConsiderInvariant(Op); }));
5910}
5911
5914 ElementCount VF) {
5915 // If we know that this instruction will remain uniform, check the cost of
5916 // the scalar version.
5918 VF = ElementCount::getFixed(1);
5919
5920 if (VF.isVector() && isProfitableToScalarize(I, VF))
5921 return InstsToScalarize[VF][I];
5922
5923 // Forced scalars do not have any scalarization overhead.
5924 auto ForcedScalar = ForcedScalars.find(VF);
5925 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
5926 auto InstSet = ForcedScalar->second;
5927 if (InstSet.count(I))
5929 VF.getKnownMinValue();
5930 }
5931
5932 Type *RetTy = I->getType();
5934 RetTy = IntegerType::get(RetTy->getContext(), MinBWs[I]);
5935 auto *SE = PSE.getSE();
5936
5937 Type *VectorTy;
5938 if (isScalarAfterVectorization(I, VF)) {
5939 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
5940 [this](Instruction *I, ElementCount VF) -> bool {
5941 if (VF.isScalar())
5942 return true;
5943
5944 auto Scalarized = InstsToScalarize.find(VF);
5945 assert(Scalarized != InstsToScalarize.end() &&
5946 "VF not yet analyzed for scalarization profitability");
5947 return !Scalarized->second.count(I) &&
5948 llvm::all_of(I->users(), [&](User *U) {
5949 auto *UI = cast<Instruction>(U);
5950 return !Scalarized->second.count(UI);
5951 });
5952 };
5953
5954 // With the exception of GEPs and PHIs, after scalarization there should
5955 // only be one copy of the instruction generated in the loop. This is
5956 // because the VF is either 1, or any instructions that need scalarizing
5957 // have already been dealt with by the time we get here. As a result,
5958 // it means we don't have to multiply the instruction cost by VF.
5959 assert(I->getOpcode() == Instruction::GetElementPtr ||
5960 I->getOpcode() == Instruction::PHI ||
5961 (I->getOpcode() == Instruction::BitCast &&
5962 I->getType()->isPointerTy()) ||
5963 HasSingleCopyAfterVectorization(I, VF));
5964 VectorTy = RetTy;
5965 } else
5966 VectorTy = toVectorizedTy(RetTy, VF);
5967
5968 if (VF.isVector() && VectorTy->isVectorTy() &&
5969 !TTI.getNumberOfParts(VectorTy))
5971
5972 // TODO: We need to estimate the cost of intrinsic calls.
5973 switch (I->getOpcode()) {
5974 case Instruction::GetElementPtr:
5975 // We mark this instruction as zero-cost because the cost of GEPs in
5976 // vectorized code depends on whether the corresponding memory instruction
5977 // is scalarized or not. Therefore, we handle GEPs with the memory
5978 // instruction cost.
5979 return 0;
5980 case Instruction::Br: {
5981 // In cases of scalarized and predicated instructions, there will be VF
5982 // predicated blocks in the vectorized loop. Each branch around these
5983 // blocks requires also an extract of its vector compare i1 element.
5984 // Note that the conditional branch from the loop latch will be replaced by
5985 // a single branch controlling the loop, so there is no extra overhead from
5986 // scalarization.
5987 bool ScalarPredicatedBB = false;
5989 if (VF.isVector() && BI->isConditional() &&
5990 (PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(0)) ||
5991 PredicatedBBsAfterVectorization[VF].count(BI->getSuccessor(1))) &&
5992 BI->getParent() != TheLoop->getLoopLatch())
5993 ScalarPredicatedBB = true;
5994
5995 if (ScalarPredicatedBB) {
5996 // Not possible to scalarize scalable vector with predicated instructions.
5997 if (VF.isScalable())
5999 // Return cost for branches around scalarized and predicated blocks.
6000 auto *VecI1Ty =
6002 return (
6003 TTI.getScalarizationOverhead(
6004 VecI1Ty, APInt::getAllOnes(VF.getFixedValue()),
6005 /*Insert*/ false, /*Extract*/ true, CostKind) +
6006 (TTI.getCFInstrCost(Instruction::Br, CostKind) * VF.getFixedValue()));
6007 }
6008
6009 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
6010 // The back-edge branch will remain, as will all scalar branches.
6011 return TTI.getCFInstrCost(Instruction::Br, CostKind);
6012
6013 // This branch will be eliminated by if-conversion.
6014 return 0;
6015 // Note: We currently assume zero cost for an unconditional branch inside
6016 // a predicated block since it will become a fall-through, although we
6017 // may decide in the future to call TTI for all branches.
6018 }
6019 case Instruction::Switch: {
6020 if (VF.isScalar())
6021 return TTI.getCFInstrCost(Instruction::Switch, CostKind);
6022 auto *Switch = cast<SwitchInst>(I);
6023 return Switch->getNumCases() *
6024 TTI.getCmpSelInstrCost(
6025 Instruction::ICmp,
6026 toVectorTy(Switch->getCondition()->getType(), VF),
6027 toVectorTy(Type::getInt1Ty(I->getContext()), VF),
6029 }
6030 case Instruction::PHI: {
6031 auto *Phi = cast<PHINode>(I);
6032
6033 // First-order recurrences are replaced by vector shuffles inside the loop.
6034 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
6036 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
6037 return TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
6038 cast<VectorType>(VectorTy),
6039 cast<VectorType>(VectorTy), Mask, CostKind,
6040 VF.getKnownMinValue() - 1);
6041 }
6042
6043 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
6044 // converted into select instructions. We require N - 1 selects per phi
6045 // node, where N is the number of incoming values.
6046 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
6047 Type *ResultTy = Phi->getType();
6048
6049 // All instructions in an Any-of reduction chain are narrowed to bool.
6050 // Check if that is the case for this phi node.
6051 auto *HeaderUser = cast_if_present<PHINode>(
6052 find_singleton<User>(Phi->users(), [this](User *U, bool) -> User * {
6053 auto *Phi = dyn_cast<PHINode>(U);
6054 if (Phi && Phi->getParent() == TheLoop->getHeader())
6055 return Phi;
6056 return nullptr;
6057 }));
6058 if (HeaderUser) {
6059 auto &ReductionVars = Legal->getReductionVars();
6060 auto Iter = ReductionVars.find(HeaderUser);
6061 if (Iter != ReductionVars.end() &&
6063 Iter->second.getRecurrenceKind()))
6064 ResultTy = Type::getInt1Ty(Phi->getContext());
6065 }
6066 return (Phi->getNumIncomingValues() - 1) *
6067 TTI.getCmpSelInstrCost(
6068 Instruction::Select, toVectorTy(ResultTy, VF),
6069 toVectorTy(Type::getInt1Ty(Phi->getContext()), VF),
6071 }
6072
6073 // When tail folding with EVL, if the phi is part of an out of loop
6074 // reduction then it will be transformed into a wide vp_merge.
6075 if (VF.isVector() && foldTailWithEVL() &&
6076 Legal->getReductionVars().contains(Phi) && !isInLoopReduction(Phi)) {
6078 Intrinsic::vp_merge, toVectorTy(Phi->getType(), VF),
6079 {toVectorTy(Type::getInt1Ty(Phi->getContext()), VF)});
6080 return TTI.getIntrinsicInstrCost(ICA, CostKind);
6081 }
6082
6083 return TTI.getCFInstrCost(Instruction::PHI, CostKind);
6084 }
6085 case Instruction::UDiv:
6086 case Instruction::SDiv:
6087 case Instruction::URem:
6088 case Instruction::SRem:
6089 if (VF.isVector() && isPredicatedInst(I)) {
6090 const auto [ScalarCost, SafeDivisorCost] = getDivRemSpeculationCost(I, VF);
6091 return isDivRemScalarWithPredication(ScalarCost, SafeDivisorCost) ?
6092 ScalarCost : SafeDivisorCost;
6093 }
6094 // We've proven all lanes safe to speculate, fall through.
6095 [[fallthrough]];
6096 case Instruction::Add:
6097 case Instruction::Sub: {
6098 auto Info = Legal->getHistogramInfo(I);
6099 if (Info && VF.isVector()) {
6100 const HistogramInfo *HGram = Info.value();
6101 // Assume that a non-constant update value (or a constant != 1) requires
6102 // a multiply, and add that into the cost.
6104 ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
6105 if (!RHS || RHS->getZExtValue() != 1)
6106 MulCost =
6107 TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
6108
6109 // Find the cost of the histogram operation itself.
6110 Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
6111 Type *ScalarTy = I->getType();
6112 Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
6113 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
6114 Type::getVoidTy(I->getContext()),
6115 {PtrTy, ScalarTy, MaskTy});
6116
6117 // Add the costs together with the add/sub operation.
6118 return TTI.getIntrinsicInstrCost(ICA, CostKind) + MulCost +
6119 TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy, CostKind);
6120 }
6121 [[fallthrough]];
6122 }
6123 case Instruction::FAdd:
6124 case Instruction::FSub:
6125 case Instruction::Mul:
6126 case Instruction::FMul:
6127 case Instruction::FDiv:
6128 case Instruction::FRem:
6129 case Instruction::Shl:
6130 case Instruction::LShr:
6131 case Instruction::AShr:
6132 case Instruction::And:
6133 case Instruction::Or:
6134 case Instruction::Xor: {
6135 // If we're speculating on the stride being 1, the multiplication may
6136 // fold away. We can generalize this for all operations using the notion
6137 // of neutral elements. (TODO)
6138 if (I->getOpcode() == Instruction::Mul &&
6139 ((TheLoop->isLoopInvariant(I->getOperand(0)) &&
6140 PSE.getSCEV(I->getOperand(0))->isOne()) ||
6141 (TheLoop->isLoopInvariant(I->getOperand(1)) &&
6142 PSE.getSCEV(I->getOperand(1))->isOne())))
6143 return 0;
6144
6145 // Detect reduction patterns
6146 if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
6147 return *RedCost;
6148
6149 // Certain instructions can be cheaper to vectorize if they have a constant
6150 // second vector operand. One example of this are shifts on x86.
6151 Value *Op2 = I->getOperand(1);
6152 if (!isa<Constant>(Op2) && TheLoop->isLoopInvariant(Op2) &&
6153 PSE.getSE()->isSCEVable(Op2->getType()) &&
6154 isa<SCEVConstant>(PSE.getSCEV(Op2))) {
6155 Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
6156 }
6157 auto Op2Info = TTI.getOperandInfo(Op2);
6158 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
6161
6162 SmallVector<const Value *, 4> Operands(I->operand_values());
6163 return TTI.getArithmeticInstrCost(
6164 I->getOpcode(), VectorTy, CostKind,
6165 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6166 Op2Info, Operands, I, TLI);
6167 }
6168 case Instruction::FNeg: {
6169 return TTI.getArithmeticInstrCost(
6170 I->getOpcode(), VectorTy, CostKind,
6171 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6172 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
6173 I->getOperand(0), I);
6174 }
6175 case Instruction::Select: {
6177 const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
6178 bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
6179
6180 const Value *Op0, *Op1;
6181 using namespace llvm::PatternMatch;
6182 if (!ScalarCond && (match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1))) ||
6183 match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))) {
6184 // select x, y, false --> x & y
6185 // select x, true, y --> x | y
6186 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(Op0);
6187 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(Op1);
6188 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
6189 Op1->getType()->getScalarSizeInBits() == 1);
6190
6191 return TTI.getArithmeticInstrCost(
6192 match(I, m_LogicalOr()) ? Instruction::Or : Instruction::And,
6193 VectorTy, CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, {Op0, Op1}, I);
6194 }
6195
6196 Type *CondTy = SI->getCondition()->getType();
6197 if (!ScalarCond)
6198 CondTy = VectorType::get(CondTy, VF);
6199
6201 if (auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
6202 Pred = Cmp->getPredicate();
6203 return TTI.getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy, Pred,
6204 CostKind, {TTI::OK_AnyValue, TTI::OP_None},
6205 {TTI::OK_AnyValue, TTI::OP_None}, I);
6206 }
6207 case Instruction::ICmp:
6208 case Instruction::FCmp: {
6209 Type *ValTy = I->getOperand(0)->getType();
6210
6212 [[maybe_unused]] Instruction *Op0AsInstruction =
6213 dyn_cast<Instruction>(I->getOperand(0));
6214 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
6215 MinBWs[I] == MinBWs[Op0AsInstruction]) &&
6216 "if both the operand and the compare are marked for "
6217 "truncation, they must have the same bitwidth");
6218 ValTy = IntegerType::get(ValTy->getContext(), MinBWs[I]);
6219 }
6220
6221 VectorTy = toVectorTy(ValTy, VF);
6222 return TTI.getCmpSelInstrCost(
6223 I->getOpcode(), VectorTy, CmpInst::makeCmpResultType(VectorTy),
6224 cast<CmpInst>(I)->getPredicate(), CostKind,
6225 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, I);
6226 }
6227 case Instruction::Store:
6228 case Instruction::Load: {
6229 ElementCount Width = VF;
6230 if (Width.isVector()) {
6231 InstWidening Decision = getWideningDecision(I, Width);
6232 assert(Decision != CM_Unknown &&
6233 "CM decision should be taken at this point");
6236 if (Decision == CM_Scalarize)
6237 Width = ElementCount::getFixed(1);
6238 }
6239 VectorTy = toVectorTy(getLoadStoreType(I), Width);
6240 return getMemoryInstructionCost(I, VF);
6241 }
6242 case Instruction::BitCast:
6243 if (I->getType()->isPointerTy())
6244 return 0;
6245 [[fallthrough]];
6246 case Instruction::ZExt:
6247 case Instruction::SExt:
6248 case Instruction::FPToUI:
6249 case Instruction::FPToSI:
6250 case Instruction::FPExt:
6251 case Instruction::PtrToInt:
6252 case Instruction::IntToPtr:
6253 case Instruction::SIToFP:
6254 case Instruction::UIToFP:
6255 case Instruction::Trunc:
6256 case Instruction::FPTrunc: {
6257 // Computes the CastContextHint from a Load/Store instruction.
6258 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
6260 "Expected a load or a store!");
6261
6262 if (VF.isScalar() || !TheLoop->contains(I))
6264
6265 switch (getWideningDecision(I, VF)) {
6277 llvm_unreachable("Instr did not go through cost modelling?");
6280 llvm_unreachable_internal("Instr has invalid widening decision");
6281 }
6282
6283 llvm_unreachable("Unhandled case!");
6284 };
6285
6286 unsigned Opcode = I->getOpcode();
6288 // For Trunc, the context is the only user, which must be a StoreInst.
6289 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
6290 if (I->hasOneUse())
6291 if (StoreInst *Store = dyn_cast<StoreInst>(*I->user_begin()))
6292 CCH = ComputeCCH(Store);
6293 }
6294 // For Z/Sext, the context is the operand, which must be a LoadInst.
6295 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
6296 Opcode == Instruction::FPExt) {
6297 if (LoadInst *Load = dyn_cast<LoadInst>(I->getOperand(0)))
6298 CCH = ComputeCCH(Load);
6299 }
6300
6301 // We optimize the truncation of induction variables having constant
6302 // integer steps. The cost of these truncations is the same as the scalar
6303 // operation.
6304 if (isOptimizableIVTruncate(I, VF)) {
6305 auto *Trunc = cast<TruncInst>(I);
6306 return TTI.getCastInstrCost(Instruction::Trunc, Trunc->getDestTy(),
6307 Trunc->getSrcTy(), CCH, CostKind, Trunc);
6308 }
6309
6310 // Detect reduction patterns
6311 if (auto RedCost = getReductionPatternCost(I, VF, VectorTy))
6312 return *RedCost;
6313
6314 Type *SrcScalarTy = I->getOperand(0)->getType();
6315 Instruction *Op0AsInstruction = dyn_cast<Instruction>(I->getOperand(0));
6316 if (canTruncateToMinimalBitwidth(Op0AsInstruction, VF))
6317 SrcScalarTy =
6318 IntegerType::get(SrcScalarTy->getContext(), MinBWs[Op0AsInstruction]);
6319 Type *SrcVecTy =
6320 VectorTy->isVectorTy() ? toVectorTy(SrcScalarTy, VF) : SrcScalarTy;
6321
6323 // If the result type is <= the source type, there will be no extend
6324 // after truncating the users to the minimal required bitwidth.
6325 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
6326 (I->getOpcode() == Instruction::ZExt ||
6327 I->getOpcode() == Instruction::SExt))
6328 return 0;
6329 }
6330
6331 return TTI.getCastInstrCost(Opcode, VectorTy, SrcVecTy, CCH, CostKind, I);
6332 }
6333 case Instruction::Call:
6334 return getVectorCallCost(cast<CallInst>(I), VF);
6335 case Instruction::ExtractValue:
6336 return TTI.getInstructionCost(I, CostKind);
6337 case Instruction::Alloca:
6338 // We cannot easily widen alloca to a scalable alloca, as
6339 // the result would need to be a vector of pointers.
6340 if (VF.isScalable())
6342 [[fallthrough]];
6343 default:
6344 // This opcode is unknown. Assume that it is the same as 'mul'.
6345 return TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
6346 } // end of switch.
6347}
6348
6350 // Ignore ephemeral values.
6352
6353 SmallVector<Value *, 4> DeadInterleavePointerOps;
6355
6356 // If a scalar epilogue is required, users outside the loop won't use
6357 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
6358 // that is the case.
6359 bool RequiresScalarEpilogue = requiresScalarEpilogue(true);
6360 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
6361 return RequiresScalarEpilogue &&
6362 !TheLoop->contains(cast<Instruction>(U)->getParent());
6363 };
6364
6366 DFS.perform(LI);
6367 for (BasicBlock *BB : reverse(make_range(DFS.beginRPO(), DFS.endRPO())))
6368 for (Instruction &I : reverse(*BB)) {
6369 if (VecValuesToIgnore.contains(&I) || ValuesToIgnore.contains(&I))
6370 continue;
6371
6372 // Add instructions that would be trivially dead and are only used by
6373 // values already ignored to DeadOps to seed worklist.
6375 all_of(I.users(), [this, IsLiveOutDead](User *U) {
6376 return VecValuesToIgnore.contains(U) ||
6377 ValuesToIgnore.contains(U) || IsLiveOutDead(U);
6378 }))
6379 DeadOps.push_back(&I);
6380
6381 // For interleave groups, we only create a pointer for the start of the
6382 // interleave group. Queue up addresses of group members except the insert
6383 // position for further processing.
6384 if (isAccessInterleaved(&I)) {
6385 auto *Group = getInterleavedAccessGroup(&I);
6386 if (Group->getInsertPos() == &I)
6387 continue;
6388 Value *PointerOp = getLoadStorePointerOperand(&I);
6389 DeadInterleavePointerOps.push_back(PointerOp);
6390 }
6391
6392 // Queue branches for analysis. They are dead, if their successors only
6393 // contain dead instructions.
6394 if (auto *Br = dyn_cast<BranchInst>(&I)) {
6395 if (Br->isConditional())
6396 DeadOps.push_back(&I);
6397 }
6398 }
6399
6400 // Mark ops feeding interleave group members as free, if they are only used
6401 // by other dead computations.
6402 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
6403 auto *Op = dyn_cast<Instruction>(DeadInterleavePointerOps[I]);
6404 if (!Op || !TheLoop->contains(Op) || any_of(Op->users(), [this](User *U) {
6405 Instruction *UI = cast<Instruction>(U);
6406 return !VecValuesToIgnore.contains(U) &&
6407 (!isAccessInterleaved(UI) ||
6408 getInterleavedAccessGroup(UI)->getInsertPos() == UI);
6409 }))
6410 continue;
6411 VecValuesToIgnore.insert(Op);
6412 append_range(DeadInterleavePointerOps, Op->operands());
6413 }
6414
6415 // Mark ops that would be trivially dead and are only used by ignored
6416 // instructions as free.
6417 BasicBlock *Header = TheLoop->getHeader();
6418
6419 // Returns true if the block contains only dead instructions. Such blocks will
6420 // be removed by VPlan-to-VPlan transforms and won't be considered by the
6421 // VPlan-based cost model, so skip them in the legacy cost-model as well.
6422 auto IsEmptyBlock = [this](BasicBlock *BB) {
6423 return all_of(*BB, [this](Instruction &I) {
6424 return ValuesToIgnore.contains(&I) || VecValuesToIgnore.contains(&I) ||
6425 (isa<BranchInst>(&I) && !cast<BranchInst>(&I)->isConditional());
6426 });
6427 };
6428 for (unsigned I = 0; I != DeadOps.size(); ++I) {
6429 auto *Op = dyn_cast<Instruction>(DeadOps[I]);
6430
6431 // Check if the branch should be considered dead.
6432 if (auto *Br = dyn_cast_or_null<BranchInst>(Op)) {
6433 BasicBlock *ThenBB = Br->getSuccessor(0);
6434 BasicBlock *ElseBB = Br->getSuccessor(1);
6435 // Don't considers branches leaving the loop for simplification.
6436 if (!TheLoop->contains(ThenBB) || !TheLoop->contains(ElseBB))
6437 continue;
6438 bool ThenEmpty = IsEmptyBlock(ThenBB);
6439 bool ElseEmpty = IsEmptyBlock(ElseBB);
6440 if ((ThenEmpty && ElseEmpty) ||
6441 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
6442 ElseBB->phis().empty()) ||
6443 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
6444 ThenBB->phis().empty())) {
6445 VecValuesToIgnore.insert(Br);
6446 DeadOps.push_back(Br->getCondition());
6447 }
6448 continue;
6449 }
6450
6451 // Skip any op that shouldn't be considered dead.
6452 if (!Op || !TheLoop->contains(Op) ||
6453 (isa<PHINode>(Op) && Op->getParent() == Header) ||
6455 any_of(Op->users(), [this, IsLiveOutDead](User *U) {
6456 return !VecValuesToIgnore.contains(U) &&
6457 !ValuesToIgnore.contains(U) && !IsLiveOutDead(U);
6458 }))
6459 continue;
6460
6461 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
6462 // which applies for both scalar and vector versions. Otherwise it is only
6463 // dead in vector versions, so only add it to VecValuesToIgnore.
6464 if (all_of(Op->users(),
6465 [this](User *U) { return ValuesToIgnore.contains(U); }))
6466 ValuesToIgnore.insert(Op);
6467
6468 VecValuesToIgnore.insert(Op);
6469 append_range(DeadOps, Op->operands());
6470 }
6471
6472 // Ignore type-promoting instructions we identified during reduction
6473 // detection.
6474 for (const auto &Reduction : Legal->getReductionVars()) {
6475 const RecurrenceDescriptor &RedDes = Reduction.second;
6476 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
6477 VecValuesToIgnore.insert_range(Casts);
6478 }
6479 // Ignore type-casting instructions we identified during induction
6480 // detection.
6481 for (const auto &Induction : Legal->getInductionVars()) {
6482 const InductionDescriptor &IndDes = Induction.second;
6483 const SmallVectorImpl<Instruction *> &Casts = IndDes.getCastInsts();
6484 VecValuesToIgnore.insert_range(Casts);
6485 }
6486}
6487
6489 // Avoid duplicating work finding in-loop reductions.
6490 if (!InLoopReductions.empty())
6491 return;
6492
6493 for (const auto &Reduction : Legal->getReductionVars()) {
6494 PHINode *Phi = Reduction.first;
6495 const RecurrenceDescriptor &RdxDesc = Reduction.second;
6496
6497 // We don't collect reductions that are type promoted (yet).
6498 if (RdxDesc.getRecurrenceType() != Phi->getType())
6499 continue;
6500
6501 // If the target would prefer this reduction to happen "in-loop", then we
6502 // want to record it as such.
6503 RecurKind Kind = RdxDesc.getRecurrenceKind();
6504 if (!PreferInLoopReductions && !useOrderedReductions(RdxDesc) &&
6505 !TTI.preferInLoopReduction(Kind, Phi->getType()))
6506 continue;
6507
6508 // Check that we can correctly put the reductions into the loop, by
6509 // finding the chain of operations that leads from the phi to the loop
6510 // exit value.
6511 SmallVector<Instruction *, 4> ReductionOperations =
6512 RdxDesc.getReductionOpChain(Phi, TheLoop);
6513 bool InLoop = !ReductionOperations.empty();
6514
6515 if (InLoop) {
6516 InLoopReductions.insert(Phi);
6517 // Add the elements to InLoopReductionImmediateChains for cost modelling.
6518 Instruction *LastChain = Phi;
6519 for (auto *I : ReductionOperations) {
6520 InLoopReductionImmediateChains[I] = LastChain;
6521 LastChain = I;
6522 }
6523 }
6524 LLVM_DEBUG(dbgs() << "LV: Using " << (InLoop ? "inloop" : "out of loop")
6525 << " reduction for phi: " << *Phi << "\n");
6526 }
6527}
6528
6529// This function will select a scalable VF if the target supports scalable
6530// vectors and a fixed one otherwise.
6531// TODO: we could return a pair of values that specify the max VF and
6532// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
6533// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
6534// doesn't have a cost model that can choose which plan to execute if
6535// more than one is generated.
6538 unsigned WidestType;
6539 std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
6540
6542 TTI.enableScalableVectorization()
6545
6546 TypeSize RegSize = TTI.getRegisterBitWidth(RegKind);
6547 unsigned N = RegSize.getKnownMinValue() / WidestType;
6548 return ElementCount::get(N, RegSize.isScalable());
6549}
6550
6553 ElementCount VF = UserVF;
6554 // Outer loop handling: They may require CFG and instruction level
6555 // transformations before even evaluating whether vectorization is profitable.
6556 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
6557 // the vectorization pipeline.
6558 if (!OrigLoop->isInnermost()) {
6559 // If the user doesn't provide a vectorization factor, determine a
6560 // reasonable one.
6561 if (UserVF.isZero()) {
6562 VF = determineVPlanVF(TTI, CM);
6563 LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
6564
6565 // Make sure we have a VF > 1 for stress testing.
6566 if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) {
6567 LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: "
6568 << "overriding computed VF.\n");
6569 VF = ElementCount::getFixed(4);
6570 }
6571 } else if (UserVF.isScalable() && !TTI.supportsScalableVectors() &&
6573 LLVM_DEBUG(dbgs() << "LV: Not vectorizing. Scalable VF requested, but "
6574 << "not supported by the target.\n");
6576 "Scalable vectorization requested but not supported by the target",
6577 "the scalable user-specified vectorization width for outer-loop "
6578 "vectorization cannot be used because the target does not support "
6579 "scalable vectors.",
6580 "ScalableVFUnfeasible", ORE, OrigLoop);
6582 }
6583 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
6585 "VF needs to be a power of two");
6586 LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")
6587 << "VF " << VF << " to build VPlans.\n");
6588 buildVPlans(VF, VF);
6589
6590 if (VPlans.empty())
6592
6593 // For VPlan build stress testing, we bail out after VPlan construction.
6596
6597 return {VF, 0 /*Cost*/, 0 /* ScalarCost */};
6598 }
6599
6600 LLVM_DEBUG(
6601 dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
6602 "VPlan-native path.\n");
6604}
6605
6606void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
6607 assert(OrigLoop->isInnermost() && "Inner loop expected.");
6608 CM.collectValuesToIgnore();
6609 CM.collectElementTypesForWidening();
6610
6611 FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
6612 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
6613 return;
6614
6615 // Invalidate interleave groups if all blocks of loop will be predicated.
6616 if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
6618 LLVM_DEBUG(
6619 dbgs()
6620 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
6621 "which requires masked-interleaved support.\n");
6622 if (CM.InterleaveInfo.invalidateGroups())
6623 // Invalidating interleave groups also requires invalidating all decisions
6624 // based on them, which includes widening decisions and uniform and scalar
6625 // values.
6626 CM.invalidateCostModelingDecisions();
6627 }
6628
6629 if (CM.foldTailByMasking())
6630 Legal->prepareToFoldTailByMasking();
6631
6632 ElementCount MaxUserVF =
6633 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
6634 if (UserVF) {
6635 if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
6637 "UserVF ignored because it may be larger than the maximal safe VF",
6638 "InvalidUserVF", ORE, OrigLoop);
6639 } else {
6641 "VF needs to be a power of two");
6642 // Collect the instructions (and their associated costs) that will be more
6643 // profitable to scalarize.
6644 CM.collectInLoopReductions();
6645 if (CM.selectUserVectorizationFactor(UserVF)) {
6646 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
6647 buildVPlansWithVPRecipes(UserVF, UserVF);
6649 return;
6650 }
6651 reportVectorizationInfo("UserVF ignored because of invalid costs.",
6652 "InvalidCost", ORE, OrigLoop);
6653 }
6654 }
6655
6656 // Collect the Vectorization Factor Candidates.
6657 SmallVector<ElementCount> VFCandidates;
6658 for (auto VF = ElementCount::getFixed(1);
6659 ElementCount::isKnownLE(VF, MaxFactors.FixedVF); VF *= 2)
6660 VFCandidates.push_back(VF);
6661 for (auto VF = ElementCount::getScalable(1);
6662 ElementCount::isKnownLE(VF, MaxFactors.ScalableVF); VF *= 2)
6663 VFCandidates.push_back(VF);
6664
6665 CM.collectInLoopReductions();
6666 for (const auto &VF : VFCandidates) {
6667 // Collect Uniform and Scalar instructions after vectorization with VF.
6668 CM.collectNonVectorizedAndSetWideningDecisions(VF);
6669 }
6670
6671 buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);
6672 buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF);
6673
6675}
6676
6678 ElementCount VF) const {
6679 InstructionCost Cost = CM.getInstructionCost(UI, VF);
6680 if (Cost.isValid() && ForceTargetInstructionCost.getNumOccurrences())
6682 return Cost;
6683}
6684
6686 ElementCount VF) const {
6687 return CM.isUniformAfterVectorization(I, VF);
6688}
6689
6690bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
6691 return CM.ValuesToIgnore.contains(UI) ||
6692 (IsVector && CM.VecValuesToIgnore.contains(UI)) ||
6693 SkipCostComputation.contains(UI);
6694}
6695
6697LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
6698 VPCostContext &CostCtx) const {
6700 // Cost modeling for inductions is inaccurate in the legacy cost model
6701 // compared to the recipes that are generated. To match here initially during
6702 // VPlan cost model bring up directly use the induction costs from the legacy
6703 // cost model. Note that we do this as pre-processing; the VPlan may not have
6704 // any recipes associated with the original induction increment instruction
6705 // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
6706 // the cost of induction phis and increments (both that are represented by
6707 // recipes and those that are not), to avoid distinguishing between them here,
6708 // and skip all recipes that represent induction phis and increments (the
6709 // former case) later on, if they exist, to avoid counting them twice.
6710 // Similarly we pre-compute the cost of any optimized truncates.
6711 // TODO: Switch to more accurate costing based on VPlan.
6712 for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
6714 IV->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
6715 SmallVector<Instruction *> IVInsts = {IVInc};
6716 for (unsigned I = 0; I != IVInsts.size(); I++) {
6717 for (Value *Op : IVInsts[I]->operands()) {
6718 auto *OpI = dyn_cast<Instruction>(Op);
6719 if (Op == IV || !OpI || !OrigLoop->contains(OpI) || !Op->hasOneUse())
6720 continue;
6721 IVInsts.push_back(OpI);
6722 }
6723 }
6724 IVInsts.push_back(IV);
6725 for (User *U : IV->users()) {
6726 auto *CI = cast<Instruction>(U);
6727 if (!CostCtx.CM.isOptimizableIVTruncate(CI, VF))
6728 continue;
6729 IVInsts.push_back(CI);
6730 }
6731
6732 // If the vector loop gets executed exactly once with the given VF, ignore
6733 // the costs of comparison and induction instructions, as they'll get
6734 // simplified away.
6735 // TODO: Remove this code after stepping away from the legacy cost model and
6736 // adding code to simplify VPlans before calculating their costs.
6737 auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
6738 if (TC == VF && !CM.foldTailByMasking())
6739 addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
6740 CostCtx.SkipCostComputation);
6741
6742 for (Instruction *IVInst : IVInsts) {
6743 if (CostCtx.skipCostComputation(IVInst, VF.isVector()))
6744 continue;
6745 InstructionCost InductionCost = CostCtx.getLegacyCost(IVInst, VF);
6746 LLVM_DEBUG({
6747 dbgs() << "Cost of " << InductionCost << " for VF " << VF
6748 << ": induction instruction " << *IVInst << "\n";
6749 });
6750 Cost += InductionCost;
6751 CostCtx.SkipCostComputation.insert(IVInst);
6752 }
6753 }
6754
6755 /// Compute the cost of all exiting conditions of the loop using the legacy
6756 /// cost model. This is to match the legacy behavior, which adds the cost of
6757 /// all exit conditions. Note that this over-estimates the cost, as there will
6758 /// be a single condition to control the vector loop.
6760 CM.TheLoop->getExitingBlocks(Exiting);
6761 SetVector<Instruction *> ExitInstrs;
6762 // Collect all exit conditions.
6763 for (BasicBlock *EB : Exiting) {
6764 auto *Term = dyn_cast<BranchInst>(EB->getTerminator());
6765 if (!Term || CostCtx.skipCostComputation(Term, VF.isVector()))
6766 continue;
6767 if (auto *CondI = dyn_cast<Instruction>(Term->getOperand(0))) {
6768 ExitInstrs.insert(CondI);
6769 }
6770 }
6771 // Compute the cost of all instructions only feeding the exit conditions.
6772 for (unsigned I = 0; I != ExitInstrs.size(); ++I) {
6773 Instruction *CondI = ExitInstrs[I];
6774 if (!OrigLoop->contains(CondI) ||
6775 !CostCtx.SkipCostComputation.insert(CondI).second)
6776 continue;
6777 InstructionCost CondICost = CostCtx.getLegacyCost(CondI, VF);
6778 LLVM_DEBUG({
6779 dbgs() << "Cost of " << CondICost << " for VF " << VF
6780 << ": exit condition instruction " << *CondI << "\n";
6781 });
6782 Cost += CondICost;
6783 for (Value *Op : CondI->operands()) {
6784 auto *OpI = dyn_cast<Instruction>(Op);
6785 if (!OpI || CostCtx.skipCostComputation(OpI, VF.isVector()) ||
6786 any_of(OpI->users(), [&ExitInstrs, this](User *U) {
6787 return OrigLoop->contains(cast<Instruction>(U)->getParent()) &&
6788 !ExitInstrs.contains(cast<Instruction>(U));
6789 }))
6790 continue;
6791 ExitInstrs.insert(OpI);
6792 }
6793 }
6794
6795 // Pre-compute the costs for branches except for the backedge, as the number
6796 // of replicate regions in a VPlan may not directly match the number of
6797 // branches, which would lead to different decisions.
6798 // TODO: Compute cost of branches for each replicate region in the VPlan,
6799 // which is more accurate than the legacy cost model.
6800 for (BasicBlock *BB : OrigLoop->blocks()) {
6801 if (CostCtx.skipCostComputation(BB->getTerminator(), VF.isVector()))
6802 continue;
6803 CostCtx.SkipCostComputation.insert(BB->getTerminator());
6804 if (BB == OrigLoop->getLoopLatch())
6805 continue;
6806 auto BranchCost = CostCtx.getLegacyCost(BB->getTerminator(), VF);
6807 Cost += BranchCost;
6808 }
6809
6810 // Pre-compute costs for instructions that are forced-scalar or profitable to
6811 // scalarize. Their costs will be computed separately in the legacy cost
6812 // model.
6813 for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
6814 if (CostCtx.skipCostComputation(ForcedScalar, VF.isVector()))
6815 continue;
6816 CostCtx.SkipCostComputation.insert(ForcedScalar);
6817 InstructionCost ForcedCost = CostCtx.getLegacyCost(ForcedScalar, VF);
6818 LLVM_DEBUG({
6819 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
6820 << ": forced scalar " << *ForcedScalar << "\n";
6821 });
6822 Cost += ForcedCost;
6823 }
6824 for (const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
6825 if (CostCtx.skipCostComputation(Scalarized, VF.isVector()))
6826 continue;
6827 CostCtx.SkipCostComputation.insert(Scalarized);
6828 LLVM_DEBUG({
6829 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
6830 << ": profitable to scalarize " << *Scalarized << "\n";
6831 });
6832 Cost += ScalarCost;
6833 }
6834
6835 return Cost;
6836}
6837
6838InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
6839 ElementCount VF) const {
6840 VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, CM.CostKind, *PSE.getSE());
6841 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
6842
6843 // Now compute and add the VPlan-based cost.
6844 Cost += Plan.cost(VF, CostCtx);
6845#ifndef NDEBUG
6846 unsigned EstimatedWidth = estimateElementCount(VF, CM.getVScaleForTuning());
6847 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
6848 << " (Estimated cost per lane: ");
6849 if (Cost.isValid()) {
6850 double CostPerLane = double(Cost.getValue()) / EstimatedWidth;
6851 LLVM_DEBUG(dbgs() << format("%.1f", CostPerLane));
6852 } else /* No point dividing an invalid cost - it will still be invalid */
6853 LLVM_DEBUG(dbgs() << "Invalid");
6854 LLVM_DEBUG(dbgs() << ")\n");
6855#endif
6856 return Cost;
6857}
6858
6859#ifndef NDEBUG
6860/// Return true if the original loop \ TheLoop contains any instructions that do
6861/// not have corresponding recipes in \p Plan and are not marked to be ignored
6862/// in \p CostCtx. This means the VPlan contains simplification that the legacy
6863/// cost-model did not account for.
6865 VPCostContext &CostCtx,
6866 Loop *TheLoop,
6867 ElementCount VF) {
6868 // First collect all instructions for the recipes in Plan.
6869 auto GetInstructionForCost = [](const VPRecipeBase *R) -> Instruction * {
6870 if (auto *S = dyn_cast<VPSingleDefRecipe>(R))
6871 return dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
6872 if (auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(R))
6873 return &WidenMem->getIngredient();
6874 return nullptr;
6875 };
6876
6877 // Check if a select for a safe divisor was hoisted to the pre-header. If so,
6878 // the select doesn't need to be considered for the vector loop cost; go with
6879 // the more accurate VPlan-based cost model.
6880 for (VPRecipeBase &R : *Plan.getVectorPreheader()) {
6881 auto *VPI = dyn_cast<VPInstruction>(&R);
6882 if (!VPI || VPI->getOpcode() != Instruction::Select ||
6883 VPI->getNumUsers() != 1)
6884 continue;
6885
6886 if (auto *WR = dyn_cast<VPWidenRecipe>(*VPI->user_begin())) {
6887 switch (WR->getOpcode()) {
6888 case Instruction::UDiv:
6889 case Instruction::SDiv:
6890 case Instruction::URem:
6891 case Instruction::SRem:
6892 return true;
6893 default:
6894 break;
6895 }
6896 }
6897 }
6898
6899 DenseSet<Instruction *> SeenInstrs;
6900 auto Iter = vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry());
6902 for (VPRecipeBase &R : *VPBB) {
6903 if (auto *IR = dyn_cast<VPInterleaveRecipe>(&R)) {
6904 auto *IG = IR->getInterleaveGroup();
6905 unsigned NumMembers = IG->getNumMembers();
6906 for (unsigned I = 0; I != NumMembers; ++I) {
6907 if (Instruction *M = IG->getMember(I))
6908 SeenInstrs.insert(M);
6909 }
6910 continue;
6911 }
6912 // Unused FOR splices are removed by VPlan transforms, so the VPlan-based
6913 // cost model won't cost it whilst the legacy will.
6914 if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R)) {
6915 using namespace VPlanPatternMatch;
6916 if (none_of(FOR->users(),
6917 match_fn(m_VPInstruction<
6919 return true;
6920 }
6921 // The VPlan-based cost model is more accurate for partial reduction and
6922 // comparing against the legacy cost isn't desirable.
6924 return true;
6925
6926 // The VPlan-based cost model can analyze if recipes are scalar
6927 // recursively, but the legacy cost model cannot.
6928 if (auto *WidenMemR = dyn_cast<VPWidenMemoryRecipe>(&R)) {
6929 auto *AddrI = dyn_cast<Instruction>(
6930 getLoadStorePointerOperand(&WidenMemR->getIngredient()));
6931 if (AddrI && vputils::isSingleScalar(WidenMemR->getAddr()) !=
6932 CostCtx.isLegacyUniformAfterVectorization(AddrI, VF))
6933 return true;
6934 }
6935
6936 /// If a VPlan transform folded a recipe to one producing a single-scalar,
6937 /// but the original instruction wasn't uniform-after-vectorization in the
6938 /// legacy cost model, the legacy cost overestimates the actual cost.
6939 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
6940 if (RepR->isSingleScalar() &&
6942 RepR->getUnderlyingInstr(), VF))
6943 return true;
6944 }
6945 if (Instruction *UI = GetInstructionForCost(&R)) {
6946 // If we adjusted the predicate of the recipe, the cost in the legacy
6947 // cost model may be different.
6948 using namespace VPlanPatternMatch;
6949 CmpPredicate Pred;
6950 if (match(&R, m_Cmp(Pred, m_VPValue(), m_VPValue())) &&
6951 cast<VPRecipeWithIRFlags>(R).getPredicate() !=
6952 cast<CmpInst>(UI)->getPredicate())
6953 return true;
6954 SeenInstrs.insert(UI);
6955 }
6956 }
6957 }
6958
6959 // Return true if the loop contains any instructions that are not also part of
6960 // the VPlan or are skipped for VPlan-based cost computations. This indicates
6961 // that the VPlan contains extra simplifications.
6962 return any_of(TheLoop->blocks(), [&SeenInstrs, &CostCtx,
6963 TheLoop](BasicBlock *BB) {
6964 return any_of(*BB, [&SeenInstrs, &CostCtx, TheLoop, BB](Instruction &I) {
6965 // Skip induction phis when checking for simplifications, as they may not
6966 // be lowered directly be lowered to a corresponding PHI recipe.
6967 if (isa<PHINode>(&I) && BB == TheLoop->getHeader() &&
6968 CostCtx.CM.Legal->isInductionPhi(cast<PHINode>(&I)))
6969 return false;
6970 return !SeenInstrs.contains(&I) && !CostCtx.skipCostComputation(&I, true);
6971 });
6972 });
6973}
6974#endif
6975
6977 if (VPlans.empty())
6979 // If there is a single VPlan with a single VF, return it directly.
6980 VPlan &FirstPlan = *VPlans[0];
6981 if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
6982 return {*FirstPlan.vectorFactors().begin(), 0, 0};
6983
6984 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
6985 << (CM.CostKind == TTI::TCK_RecipThroughput
6986 ? "Reciprocal Throughput\n"
6987 : CM.CostKind == TTI::TCK_Latency
6988 ? "Instruction Latency\n"
6989 : CM.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
6990 : CM.CostKind == TTI::TCK_SizeAndLatency
6991 ? "Code Size and Latency\n"
6992 : "Unknown\n"));
6993
6995 assert(hasPlanWithVF(ScalarVF) &&
6996 "More than a single plan/VF w/o any plan having scalar VF");
6997
6998 // TODO: Compute scalar cost using VPlan-based cost model.
6999 InstructionCost ScalarCost = CM.expectedCost(ScalarVF);
7000 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
7001 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
7002 VectorizationFactor BestFactor = ScalarFactor;
7003
7004 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
7005 if (ForceVectorization) {
7006 // Ignore scalar width, because the user explicitly wants vectorization.
7007 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
7008 // evaluation.
7009 BestFactor.Cost = InstructionCost::getMax();
7010 }
7011
7012 for (auto &P : VPlans) {
7013 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
7014 P->vectorFactors().end());
7015
7017 if (any_of(VFs, [this](ElementCount VF) {
7018 return CM.shouldConsiderRegPressureForVF(VF);
7019 }))
7020 RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore);
7021
7022 for (unsigned I = 0; I < VFs.size(); I++) {
7023 ElementCount VF = VFs[I];
7024 if (VF.isScalar())
7025 continue;
7026 if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) {
7027 LLVM_DEBUG(
7028 dbgs()
7029 << "LV: Not considering vector loop of width " << VF
7030 << " because it will not generate any vector instructions.\n");
7031 continue;
7032 }
7033 if (CM.OptForSize && !ForceVectorization && hasReplicatorRegion(*P)) {
7034 LLVM_DEBUG(
7035 dbgs()
7036 << "LV: Not considering vector loop of width " << VF
7037 << " because it would cause replicated blocks to be generated,"
7038 << " which isn't allowed when optimizing for size.\n");
7039 continue;
7040 }
7041
7042 InstructionCost Cost = cost(*P, VF);
7043 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
7044
7045 if (CM.shouldConsiderRegPressureForVF(VF) &&
7046 RUs[I].exceedsMaxNumRegs(TTI, ForceTargetNumVectorRegs)) {
7047 LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width "
7048 << VF << " because it uses too many registers\n");
7049 continue;
7050 }
7051
7052 if (isMoreProfitable(CurrentFactor, BestFactor, P->hasScalarTail()))
7053 BestFactor = CurrentFactor;
7054
7055 // If profitable add it to ProfitableVF list.
7056 if (isMoreProfitable(CurrentFactor, ScalarFactor, P->hasScalarTail()))
7057 ProfitableVFs.push_back(CurrentFactor);
7058 }
7059 }
7060
7061#ifndef NDEBUG
7062 // Select the optimal vectorization factor according to the legacy cost-model.
7063 // This is now only used to verify the decisions by the new VPlan-based
7064 // cost-model and will be retired once the VPlan-based cost-model is
7065 // stabilized.
7066 VectorizationFactor LegacyVF = selectVectorizationFactor();
7067 VPlan &BestPlan = getPlanFor(BestFactor.Width);
7068
7069 // Pre-compute the cost and use it to check if BestPlan contains any
7070 // simplifications not accounted for in the legacy cost model. If that's the
7071 // case, don't trigger the assertion, as the extra simplifications may cause a
7072 // different VF to be picked by the VPlan-based cost model.
7073 VPCostContext CostCtx(CM.TTI, *CM.TLI, BestPlan, CM, CM.CostKind,
7074 *CM.PSE.getSE());
7075 precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
7076 // Verify that the VPlan-based and legacy cost models agree, except for VPlans
7077 // with early exits and plans with additional VPlan simplifications. The
7078 // legacy cost model doesn't properly model costs for such loops.
7079 assert((BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() ||
7081 CostCtx, OrigLoop,
7082 BestFactor.Width) ||
7084 getPlanFor(LegacyVF.Width), CostCtx, OrigLoop, LegacyVF.Width)) &&
7085 " VPlan cost model and legacy cost model disagreed");
7086 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
7087 "when vectorizing, the scalar cost must be computed.");
7088#endif
7089
7090 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
7091 return BestFactor;
7092}
7093
7095 using namespace VPlanPatternMatch;
7097 "RdxResult must be ComputeFindIVResult");
7098 VPValue *StartVPV = RdxResult->getOperand(1);
7099 match(StartVPV, m_Freeze(m_VPValue(StartVPV)));
7100 return StartVPV->getLiveInIRValue();
7101}
7102
7103// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
7104// epilog loop, fix the reduction's scalar PHI node by adding the incoming value
7105// from the main vector loop.
7107 VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock) {
7108 // Get the VPInstruction computing the reduction result in the middle block.
7109 // The first operand may not be from the middle block if it is not connected
7110 // to the scalar preheader. In that case, there's nothing to fix.
7111 VPValue *Incoming = EpiResumePhiR->getOperand(0);
7114 auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
7115 if (!EpiRedResult ||
7116 (EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
7117 EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
7118 EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
7119 return;
7120
7121 auto *EpiRedHeaderPhi =
7122 cast<VPReductionPHIRecipe>(EpiRedResult->getOperand(0));
7123 RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind();
7124 Value *MainResumeValue;
7125 if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
7126 assert((VPI->getOpcode() == VPInstruction::Broadcast ||
7127 VPI->getOpcode() == VPInstruction::ReductionStartVector) &&
7128 "unexpected start recipe");
7129 MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
7130 } else
7131 MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
7133 [[maybe_unused]] Value *StartV =
7134 EpiRedResult->getOperand(1)->getLiveInIRValue();
7135 auto *Cmp = cast<ICmpInst>(MainResumeValue);
7136 assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
7137 "AnyOf expected to start with ICMP_NE");
7138 assert(Cmp->getOperand(1) == StartV &&
7139 "AnyOf expected to start by comparing main resume value to original "
7140 "start value");
7141 MainResumeValue = Cmp->getOperand(0);
7143 Value *StartV = getStartValueFromReductionResult(EpiRedResult);
7144 Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue();
7145 using namespace llvm::PatternMatch;
7146 Value *Cmp, *OrigResumeV, *CmpOp;
7147 [[maybe_unused]] bool IsExpectedPattern =
7148 match(MainResumeValue,
7149 m_Select(m_OneUse(m_Value(Cmp)), m_Specific(SentinelV),
7150 m_Value(OrigResumeV))) &&
7152 m_Value(CmpOp))) &&
7153 ((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
7154 assert(IsExpectedPattern && "Unexpected reduction resume pattern");
7155 MainResumeValue = OrigResumeV;
7156 }
7157 PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);
7158
7159 // When fixing reductions in the epilogue loop we should already have
7160 // created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
7161 // over the incoming values correctly.
7162 EpiResumePhi.setIncomingValueForBlock(
7163 BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
7164}
7165
7167 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
7168 InnerLoopVectorizer &ILV, DominatorTree *DT, bool VectorizingEpilogue) {
7169 assert(BestVPlan.hasVF(BestVF) &&
7170 "Trying to execute plan with unsupported VF");
7171 assert(BestVPlan.hasUF(BestUF) &&
7172 "Trying to execute plan with unsupported UF");
7173 if (BestVPlan.hasEarlyExit())
7174 ++LoopsEarlyExitVectorized;
7175 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
7176 // cost model is complete for better cost estimates.
7181 bool HasBranchWeights =
7182 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator());
7183 if (HasBranchWeights) {
7184 std::optional<unsigned> VScale = CM.getVScaleForTuning();
7186 BestVPlan, BestVF, VScale);
7187 }
7188
7189 // Checks are the same for all VPlans, added to BestVPlan only for
7190 // compactness.
7191 attachRuntimeChecks(BestVPlan, ILV.RTChecks, HasBranchWeights);
7192
7193 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
7194 VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
7195
7196 VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
7199 if (BestVPlan.getEntry()->getSingleSuccessor() ==
7200 BestVPlan.getScalarPreheader()) {
7201 // TODO: The vector loop would be dead, should not even try to vectorize.
7202 ORE->emit([&]() {
7203 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
7204 OrigLoop->getStartLoc(),
7205 OrigLoop->getHeader())
7206 << "Created vector loop never executes due to insufficient trip "
7207 "count.";
7208 });
7210 }
7211
7213 BestVPlan, BestVF,
7214 TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector));
7216
7218 // Regions are dissolved after optimizing for VF and UF, which completely
7219 // removes unneeded loop regions first.
7221 // Canonicalize EVL loops after regions are dissolved.
7225 BestVPlan, VectorPH, CM.foldTailByMasking(),
7226 CM.requiresScalarEpilogue(BestVF.isVector()));
7227 VPlanTransforms::materializeVFAndVFxUF(BestVPlan, VectorPH, BestVF);
7228 VPlanTransforms::cse(BestVPlan);
7230
7231 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
7232 // making any changes to the CFG.
7233 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
7234 VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
7235 if (!ILV.getTripCount())
7236 ILV.setTripCount(BestVPlan.getTripCount()->getLiveInIRValue());
7237 else
7238 assert(VectorizingEpilogue && "should only re-use the existing trip "
7239 "count during epilogue vectorization");
7240
7241 // Perform the actual loop transformation.
7242 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
7243 OrigLoop->getParentLoop(),
7244 Legal->getWidestInductionType());
7245
7246#ifdef EXPENSIVE_CHECKS
7247 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
7248#endif
7249
7250 // 1. Set up the skeleton for vectorization, including vector pre-header and
7251 // middle block. The vector loop is created during VPlan execution.
7252 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
7254 State.CFG.PrevBB->getSingleSuccessor(), &BestVPlan);
7256
7257 assert(verifyVPlanIsValid(BestVPlan, true /*VerifyLate*/) &&
7258 "final VPlan is invalid");
7259
7260 // After vectorization, the exit blocks of the original loop will have
7261 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
7262 // looked through single-entry phis.
7263 ScalarEvolution &SE = *PSE.getSE();
7264 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
7265 if (!Exit->hasPredecessors())
7266 continue;
7267 for (VPRecipeBase &PhiR : Exit->phis())
7269 OrigLoop, cast<PHINode>(&cast<VPIRPhi>(PhiR).getInstruction()));
7270 }
7271 // Forget the original loop and block dispositions.
7272 SE.forgetLoop(OrigLoop);
7274
7276
7277 //===------------------------------------------------===//
7278 //
7279 // Notice: any optimization or new instruction that go
7280 // into the code below should also be implemented in
7281 // the cost-model.
7282 //
7283 //===------------------------------------------------===//
7284
7285 // Retrieve loop information before executing the plan, which may remove the
7286 // original loop, if it becomes unreachable.
7287 MDNode *LID = OrigLoop->getLoopID();
7288 unsigned OrigLoopInvocationWeight = 0;
7289 std::optional<unsigned> OrigAverageTripCount =
7290 getLoopEstimatedTripCount(OrigLoop, &OrigLoopInvocationWeight);
7291
7292 BestVPlan.execute(&State);
7293
7294 // 2.6. Maintain Loop Hints
7295 // Keep all loop hints from the original loop on the vector loop (we'll
7296 // replace the vectorizer-specific hints below).
7297 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(BestVPlan, State.VPDT);
7298 // Add metadata to disable runtime unrolling a scalar loop when there
7299 // are no runtime checks about strides and memory. A scalar loop that is
7300 // rarely used is not worth unrolling.
7301 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
7303 HeaderVPBB ? LI->getLoopFor(State.CFG.VPBB2IRBB.lookup(HeaderVPBB))
7304 : nullptr,
7305 HeaderVPBB, BestVPlan, VectorizingEpilogue, LID, OrigAverageTripCount,
7306 OrigLoopInvocationWeight,
7307 estimateElementCount(BestVF * BestUF, CM.getVScaleForTuning()),
7308 DisableRuntimeUnroll);
7309
7310 // 3. Fix the vectorized code: take care of header phi's, live-outs,
7311 // predication, updating analyses.
7312 ILV.fixVectorizedLoop(State);
7313
7315
7316 return ExpandedSCEVs;
7317}
7318
7319//===--------------------------------------------------------------------===//
7320// EpilogueVectorizerMainLoop
7321//===--------------------------------------------------------------------===//
7322
7323/// This function is partially responsible for generating the control flow
7324/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
7326 BasicBlock *ScalarPH = createScalarPreheader("");
7327 BasicBlock *VectorPH = ScalarPH->getSinglePredecessor();
7328
7329 // Generate the code to check the minimum iteration count of the vector
7330 // epilogue (see below).
7331 EPI.EpilogueIterationCountCheck =
7332 emitIterationCountCheck(VectorPH, ScalarPH, true);
7333 EPI.EpilogueIterationCountCheck->setName("iter.check");
7334
7335 VectorPH = cast<BranchInst>(EPI.EpilogueIterationCountCheck->getTerminator())
7336 ->getSuccessor(1);
7337 // Generate the iteration count check for the main loop, *after* the check
7338 // for the epilogue loop, so that the path-length is shorter for the case
7339 // that goes directly through the vector epilogue. The longer-path length for
7340 // the main loop is compensated for, by the gain from vectorizing the larger
7341 // trip count. Note: the branch will get updated later on when we vectorize
7342 // the epilogue.
7343 EPI.MainLoopIterationCountCheck =
7344 emitIterationCountCheck(VectorPH, ScalarPH, false);
7345
7346 return cast<BranchInst>(EPI.MainLoopIterationCountCheck->getTerminator())
7347 ->getSuccessor(1);
7348}
7349
7351 LLVM_DEBUG({
7352 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7353 << "Main Loop VF:" << EPI.MainLoopVF
7354 << ", Main Loop UF:" << EPI.MainLoopUF
7355 << ", Epilogue Loop VF:" << EPI.EpilogueVF
7356 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7357 });
7358}
7359
7362 dbgs() << "intermediate fn:\n"
7363 << *OrigLoop->getHeader()->getParent() << "\n";
7364 });
7365}
7366
7368 BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue) {
7369 assert(Bypass && "Expected valid bypass basic block.");
7372 Value *CheckMinIters = createIterationCountCheck(
7373 VectorPH, ForEpilogue ? EPI.EpilogueVF : EPI.MainLoopVF,
7374 ForEpilogue ? EPI.EpilogueUF : EPI.MainLoopUF);
7375
7376 BasicBlock *const TCCheckBlock = VectorPH;
7377 if (!ForEpilogue)
7378 TCCheckBlock->setName("vector.main.loop.iter.check");
7379
7380 // Create new preheader for vector loop.
7381 VectorPH = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
7382 static_cast<DominatorTree *>(nullptr), LI, nullptr,
7383 "vector.ph");
7384 if (ForEpilogue) {
7385 // Save the trip count so we don't have to regenerate it in the
7386 // vec.epilog.iter.check. This is safe to do because the trip count
7387 // generated here dominates the vector epilog iter check.
7388 EPI.TripCount = Count;
7389 } else {
7391 }
7392
7393 BranchInst &BI = *BranchInst::Create(Bypass, VectorPH, CheckMinIters);
7394 if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator()))
7395 setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
7396 ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
7397
7398 // When vectorizing the main loop, its trip-count check is placed in a new
7399 // block, whereas the overall trip-count check is placed in the VPlan entry
7400 // block. When vectorizing the epilogue loop, its trip-count check is placed
7401 // in the VPlan entry block.
7402 if (!ForEpilogue)
7403 introduceCheckBlockInVPlan(TCCheckBlock);
7404 return TCCheckBlock;
7405}
7406
7407//===--------------------------------------------------------------------===//
7408// EpilogueVectorizerEpilogueLoop
7409//===--------------------------------------------------------------------===//
7410
7411/// This function creates a new scalar preheader, using the previous one as
7412/// entry block to the epilogue VPlan. The minimum iteration check is being
7413/// represented in VPlan.
7415 BasicBlock *NewScalarPH = createScalarPreheader("vec.epilog.");
7416 BasicBlock *OriginalScalarPH = NewScalarPH->getSinglePredecessor();
7417 OriginalScalarPH->setName("vec.epilog.iter.check");
7418 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(OriginalScalarPH);
7419 VPBasicBlock *OldEntry = Plan.getEntry();
7420 for (auto &R : make_early_inc_range(*OldEntry)) {
7421 // Skip moving VPIRInstructions (including VPIRPhis), which are unmovable by
7422 // defining.
7423 if (isa<VPIRInstruction>(&R))
7424 continue;
7425 R.moveBefore(*NewEntry, NewEntry->end());
7426 }
7427
7428 VPBlockUtils::reassociateBlocks(OldEntry, NewEntry);
7429 Plan.setEntry(NewEntry);
7430 // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
7431
7432 return OriginalScalarPH;
7433}
7434
7436 LLVM_DEBUG({
7437 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
7438 << "Epilogue Loop VF:" << EPI.EpilogueVF
7439 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
7440 });
7441}
7442
7445 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n";
7446 });
7447}
7448
7450VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
7451 VFRange &Range) {
7453 "Must be called with either a load or store");
7454
7455 auto WillWiden = [&](ElementCount VF) -> bool {
7457 CM.getWideningDecision(I, VF);
7459 "CM decision should be taken at this point.");
7461 return true;
7462 if (CM.isScalarAfterVectorization(I, VF) ||
7463 CM.isProfitableToScalarize(I, VF))
7464 return false;
7466 };
7467
7469 return nullptr;
7470
7471 VPValue *Mask = nullptr;
7472 if (Legal->isMaskRequired(I))
7473 Mask = getBlockInMask(Builder.getInsertBlock());
7474
7475 // Determine if the pointer operand of the access is either consecutive or
7476 // reverse consecutive.
7478 CM.getWideningDecision(I, Range.Start);
7480 bool Consecutive =
7482
7484 if (Consecutive) {
7486 Ptr->getUnderlyingValue()->stripPointerCasts());
7487 VPSingleDefRecipe *VectorPtr;
7488 if (Reverse) {
7489 // When folding the tail, we may compute an address that we don't in the
7490 // original scalar loop: drop the GEP no-wrap flags in this case.
7491 // Otherwise preserve existing flags without no-unsigned-wrap, as we will
7492 // emit negative indices.
7493 GEPNoWrapFlags Flags =
7494 CM.foldTailByMasking() || !GEP
7496 : GEP->getNoWrapFlags().withoutNoUnsignedWrap();
7497 VectorPtr =
7499 /*Stride*/ -1, Flags, I->getDebugLoc());
7500 } else {
7501 VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
7502 GEP ? GEP->getNoWrapFlags()
7504 I->getDebugLoc());
7505 }
7506 Builder.insert(VectorPtr);
7507 Ptr = VectorPtr;
7508 }
7509 if (LoadInst *Load = dyn_cast<LoadInst>(I))
7510 return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
7511 VPIRMetadata(*Load, LVer), I->getDebugLoc());
7512
7513 StoreInst *Store = cast<StoreInst>(I);
7514 return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
7515 Reverse, VPIRMetadata(*Store, LVer),
7516 I->getDebugLoc());
7517}
7518
7519/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
7520/// insert a recipe to expand the step for the induction recipe.
7521static VPWidenIntOrFpInductionRecipe *
7523 VPValue *Start, const InductionDescriptor &IndDesc,
7524 VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop) {
7525 assert(IndDesc.getStartValue() ==
7526 Phi->getIncomingValueForBlock(OrigLoop.getLoopPreheader()));
7527 assert(SE.isLoopInvariant(IndDesc.getStep(), &OrigLoop) &&
7528 "step must be loop invariant");
7529
7530 VPValue *Step =
7532 if (auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
7533 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
7534 IndDesc, TruncI,
7535 TruncI->getDebugLoc());
7536 }
7537 assert(isa<PHINode>(PhiOrTrunc) && "must be a phi node here");
7538 return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
7539 IndDesc, Phi->getDebugLoc());
7540}
7541
7542VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
7543 PHINode *Phi, ArrayRef<VPValue *> Operands, VFRange &Range) {
7544
7545 // Check if this is an integer or fp induction. If so, build the recipe that
7546 // produces its scalar and vector values.
7547 if (auto *II = Legal->getIntOrFpInductionDescriptor(Phi))
7548 return createWidenInductionRecipes(Phi, Phi, Operands[0], *II, Plan,
7549 *PSE.getSE(), *OrigLoop);
7550
7551 // Check if this is pointer induction. If so, build the recipe for it.
7552 if (auto *II = Legal->getPointerInductionDescriptor(Phi)) {
7553 VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
7554 return new VPWidenPointerInductionRecipe(
7555 Phi, Operands[0], Step, &Plan.getVFxUF(), *II,
7557 [&](ElementCount VF) {
7558 return CM.isScalarAfterVectorization(Phi, VF);
7559 },
7560 Range),
7561 Phi->getDebugLoc());
7562 }
7563 return nullptr;
7564}
7565
7566VPWidenIntOrFpInductionRecipe *VPRecipeBuilder::tryToOptimizeInductionTruncate(
7567 TruncInst *I, ArrayRef<VPValue *> Operands, VFRange &Range) {
7568 // Optimize the special case where the source is a constant integer
7569 // induction variable. Notice that we can only optimize the 'trunc' case
7570 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
7571 // (c) other casts depend on pointer size.
7572
7573 // Determine whether \p K is a truncation based on an induction variable that
7574 // can be optimized.
7575 auto IsOptimizableIVTruncate =
7576 [&](Instruction *K) -> std::function<bool(ElementCount)> {
7577 return [=](ElementCount VF) -> bool {
7578 return CM.isOptimizableIVTruncate(K, VF);
7579 };
7580 };
7581
7583 IsOptimizableIVTruncate(I), Range)) {
7584
7585 auto *Phi = cast<PHINode>(I->getOperand(0));
7586 const InductionDescriptor &II = *Legal->getIntOrFpInductionDescriptor(Phi);
7587 VPValue *Start = Plan.getOrAddLiveIn(II.getStartValue());
7588 return createWidenInductionRecipes(Phi, I, Start, II, Plan, *PSE.getSE(),
7589 *OrigLoop);
7590 }
7591 return nullptr;
7592}
7593
7594VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
7596 VFRange &Range) {
7598 [this, CI](ElementCount VF) {
7599 return CM.isScalarWithPredication(CI, VF);
7600 },
7601 Range);
7602
7603 if (IsPredicated)
7604 return nullptr;
7605
7607 if (ID && (ID == Intrinsic::assume || ID == Intrinsic::lifetime_end ||
7608 ID == Intrinsic::lifetime_start || ID == Intrinsic::sideeffect ||
7609 ID == Intrinsic::pseudoprobe ||
7610 ID == Intrinsic::experimental_noalias_scope_decl))
7611 return nullptr;
7612
7614
7615 // Is it beneficial to perform intrinsic call compared to lib call?
7616 bool ShouldUseVectorIntrinsic =
7618 [&](ElementCount VF) -> bool {
7619 return CM.getCallWideningDecision(CI, VF).Kind ==
7621 },
7622 Range);
7623 if (ShouldUseVectorIntrinsic)
7624 return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
7625 CI->getDebugLoc());
7626
7627 Function *Variant = nullptr;
7628 std::optional<unsigned> MaskPos;
7629 // Is better to call a vectorized version of the function than to to scalarize
7630 // the call?
7631 auto ShouldUseVectorCall = LoopVectorizationPlanner::getDecisionAndClampRange(
7632 [&](ElementCount VF) -> bool {
7633 // The following case may be scalarized depending on the VF.
7634 // The flag shows whether we can use a usual Call for vectorized
7635 // version of the instruction.
7636
7637 // If we've found a variant at a previous VF, then stop looking. A
7638 // vectorized variant of a function expects input in a certain shape
7639 // -- basically the number of input registers, the number of lanes
7640 // per register, and whether there's a mask required.
7641 // We store a pointer to the variant in the VPWidenCallRecipe, so
7642 // once we have an appropriate variant it's only valid for that VF.
7643 // This will force a different vplan to be generated for each VF that
7644 // finds a valid variant.
7645 if (Variant)
7646 return false;
7647 LoopVectorizationCostModel::CallWideningDecision Decision =
7648 CM.getCallWideningDecision(CI, VF);
7650 Variant = Decision.Variant;
7651 MaskPos = Decision.MaskPos;
7652 return true;
7653 }
7654
7655 return false;
7656 },
7657 Range);
7658 if (ShouldUseVectorCall) {
7659 if (MaskPos.has_value()) {
7660 // We have 2 cases that would require a mask:
7661 // 1) The block needs to be predicated, either due to a conditional
7662 // in the scalar loop or use of an active lane mask with
7663 // tail-folding, and we use the appropriate mask for the block.
7664 // 2) No mask is required for the block, but the only available
7665 // vector variant at this VF requires a mask, so we synthesize an
7666 // all-true mask.
7667 VPValue *Mask = nullptr;
7668 if (Legal->isMaskRequired(CI))
7669 Mask = getBlockInMask(Builder.getInsertBlock());
7670 else
7671 Mask = Plan.getOrAddLiveIn(
7672 ConstantInt::getTrue(IntegerType::getInt1Ty(CI->getContext())));
7673
7674 Ops.insert(Ops.begin() + *MaskPos, Mask);
7675 }
7676
7677 Ops.push_back(Operands.back());
7678 return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc());
7679 }
7680
7681 return nullptr;
7682}
7683
7684bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
7686 !isa<StoreInst>(I) && "Instruction should have been handled earlier");
7687 // Instruction should be widened, unless it is scalar after vectorization,
7688 // scalarization is profitable or it is predicated.
7689 auto WillScalarize = [this, I](ElementCount VF) -> bool {
7690 return CM.isScalarAfterVectorization(I, VF) ||
7691 CM.isProfitableToScalarize(I, VF) ||
7692 CM.isScalarWithPredication(I, VF);
7693 };
7695 Range);
7696}
7697
7698VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
7700 switch (I->getOpcode()) {
7701 default:
7702 return nullptr;
7703 case Instruction::SDiv:
7704 case Instruction::UDiv:
7705 case Instruction::SRem:
7706 case Instruction::URem: {
7707 // If not provably safe, use a select to form a safe divisor before widening the
7708 // div/rem operation itself. Otherwise fall through to general handling below.
7709 if (CM.isPredicatedInst(I)) {
7711 VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
7712 VPValue *One =
7713 Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
7714 auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
7715 Ops[1] = SafeRHS;
7716 return new VPWidenRecipe(*I, Ops);
7717 }
7718 [[fallthrough]];
7719 }
7720 case Instruction::Add:
7721 case Instruction::And:
7722 case Instruction::AShr:
7723 case Instruction::FAdd:
7724 case Instruction::FCmp:
7725 case Instruction::FDiv:
7726 case Instruction::FMul:
7727 case Instruction::FNeg:
7728 case Instruction::FRem:
7729 case Instruction::FSub:
7730 case Instruction::ICmp:
7731 case Instruction::LShr:
7732 case Instruction::Mul:
7733 case Instruction::Or:
7734 case Instruction::Select:
7735 case Instruction::Shl:
7736 case Instruction::Sub:
7737 case Instruction::Xor:
7738 case Instruction::Freeze: {
7740 if (Instruction::isBinaryOp(I->getOpcode())) {
7741 // The legacy cost model uses SCEV to check if some of the operands are
7742 // constants. To match the legacy cost model's behavior, use SCEV to try
7743 // to replace operands with constants.
7744 ScalarEvolution &SE = *PSE.getSE();
7745 auto GetConstantViaSCEV = [this, &SE](VPValue *Op) {
7746 if (!Op->isLiveIn())
7747 return Op;
7748 Value *V = Op->getUnderlyingValue();
7749 if (isa<Constant>(V) || !SE.isSCEVable(V->getType()))
7750 return Op;
7751 auto *C = dyn_cast<SCEVConstant>(SE.getSCEV(V));
7752 if (!C)
7753 return Op;
7754 return Plan.getOrAddLiveIn(C->getValue());
7755 };
7756 // For Mul, the legacy cost model checks both operands.
7757 if (I->getOpcode() == Instruction::Mul)
7758 NewOps[0] = GetConstantViaSCEV(NewOps[0]);
7759 // For other binops, the legacy cost model only checks the second operand.
7760 NewOps[1] = GetConstantViaSCEV(NewOps[1]);
7761 }
7762 return new VPWidenRecipe(*I, NewOps);
7763 }
7764 case Instruction::ExtractValue: {
7766 Type *I32Ty = IntegerType::getInt32Ty(I->getContext());
7767 auto *EVI = cast<ExtractValueInst>(I);
7768 assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
7769 unsigned Idx = EVI->getIndices()[0];
7770 NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
7771 return new VPWidenRecipe(*I, NewOps);
7772 }
7773 };
7774}
7775
7776VPHistogramRecipe *
7777VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
7779 // FIXME: Support other operations.
7780 unsigned Opcode = HI->Update->getOpcode();
7781 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
7782 "Histogram update operation must be an Add or Sub");
7783
7785 // Bucket address.
7786 HGramOps.push_back(Operands[1]);
7787 // Increment value.
7788 HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1)));
7789
7790 // In case of predicated execution (due to tail-folding, or conditional
7791 // execution, or both), pass the relevant mask.
7792 if (Legal->isMaskRequired(HI->Store))
7793 HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
7794
7795 return new VPHistogramRecipe(Opcode, HGramOps, HI->Store->getDebugLoc());
7796}
7797
7798VPReplicateRecipe *
7800 VFRange &Range) {
7802 [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
7803 Range);
7804
7805 bool IsPredicated = CM.isPredicatedInst(I);
7806
7807 // Even if the instruction is not marked as uniform, there are certain
7808 // intrinsic calls that can be effectively treated as such, so we check for
7809 // them here. Conservatively, we only do this for scalable vectors, since
7810 // for fixed-width VFs we can always fall back on full scalarization.
7811 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(I)) {
7812 switch (cast<IntrinsicInst>(I)->getIntrinsicID()) {
7813 case Intrinsic::assume:
7814 case Intrinsic::lifetime_start:
7815 case Intrinsic::lifetime_end:
7816 // For scalable vectors if one of the operands is variant then we still
7817 // want to mark as uniform, which will generate one instruction for just
7818 // the first lane of the vector. We can't scalarize the call in the same
7819 // way as for fixed-width vectors because we don't know how many lanes
7820 // there are.
7821 //
7822 // The reasons for doing it this way for scalable vectors are:
7823 // 1. For the assume intrinsic generating the instruction for the first
7824 // lane is still be better than not generating any at all. For
7825 // example, the input may be a splat across all lanes.
7826 // 2. For the lifetime start/end intrinsics the pointer operand only
7827 // does anything useful when the input comes from a stack object,
7828 // which suggests it should always be uniform. For non-stack objects
7829 // the effect is to poison the object, which still allows us to
7830 // remove the call.
7831 IsUniform = true;
7832 break;
7833 default:
7834 break;
7835 }
7836 }
7837 VPValue *BlockInMask = nullptr;
7838 if (!IsPredicated) {
7839 // Finalize the recipe for Instr, first if it is not predicated.
7840 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
7841 } else {
7842 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
7843 // Instructions marked for predication are replicated and a mask operand is
7844 // added initially. Masked replicate recipes will later be placed under an
7845 // if-then construct to prevent side-effects. Generate recipes to compute
7846 // the block mask for this region.
7847 BlockInMask = getBlockInMask(Builder.getInsertBlock());
7848 }
7849
7850 // Note that there is some custom logic to mark some intrinsics as uniform
7851 // manually above for scalable vectors, which this assert needs to account for
7852 // as well.
7853 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
7854 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
7855 "Should not predicate a uniform recipe");
7856 auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
7857 VPIRMetadata(*I, LVer));
7858 return Recipe;
7859}
7860
7861/// Find all possible partial reductions in the loop and track all of those that
7862/// are valid so recipes can be formed later.
7864 // Find all possible partial reductions.
7866 PartialReductionChains;
7867 for (const auto &[Phi, RdxDesc] : Legal->getReductionVars()) {
7868 getScaledReductions(Phi, RdxDesc.getLoopExitInstr(), Range,
7869 PartialReductionChains);
7870 }
7871
7872 // A partial reduction is invalid if any of its extends are used by
7873 // something that isn't another partial reduction. This is because the
7874 // extends are intended to be lowered along with the reduction itself.
7875
7876 // Build up a set of partial reduction ops for efficient use checking.
7877 SmallPtrSet<User *, 4> PartialReductionOps;
7878 for (const auto &[PartialRdx, _] : PartialReductionChains)
7879 PartialReductionOps.insert(PartialRdx.ExtendUser);
7880
7881 auto ExtendIsOnlyUsedByPartialReductions =
7882 [&PartialReductionOps](Instruction *Extend) {
7883 return all_of(Extend->users(), [&](const User *U) {
7884 return PartialReductionOps.contains(U);
7885 });
7886 };
7887
7888 // Check if each use of a chain's two extends is a partial reduction
7889 // and only add those that don't have non-partial reduction users.
7890 for (auto Pair : PartialReductionChains) {
7891 PartialReductionChain Chain = Pair.first;
7892 if (ExtendIsOnlyUsedByPartialReductions(Chain.ExtendA) &&
7893 (!Chain.ExtendB || ExtendIsOnlyUsedByPartialReductions(Chain.ExtendB)))
7894 ScaledReductionMap.try_emplace(Chain.Reduction, Pair.second);
7895 }
7896}
7897
7898bool VPRecipeBuilder::getScaledReductions(
7899 Instruction *PHI, Instruction *RdxExitInstr, VFRange &Range,
7900 SmallVectorImpl<std::pair<PartialReductionChain, unsigned>> &Chains) {
7901 if (!CM.TheLoop->contains(RdxExitInstr))
7902 return false;
7903
7904 auto *Update = dyn_cast<BinaryOperator>(RdxExitInstr);
7905 if (!Update)
7906 return false;
7907
7908 Value *Op = Update->getOperand(0);
7909 Value *PhiOp = Update->getOperand(1);
7910 if (Op == PHI)
7911 std::swap(Op, PhiOp);
7912
7913 // Try and get a scaled reduction from the first non-phi operand.
7914 // If one is found, we use the discovered reduction instruction in
7915 // place of the accumulator for costing.
7916 if (auto *OpInst = dyn_cast<Instruction>(Op)) {
7917 if (getScaledReductions(PHI, OpInst, Range, Chains)) {
7918 PHI = Chains.rbegin()->first.Reduction;
7919
7920 Op = Update->getOperand(0);
7921 PhiOp = Update->getOperand(1);
7922 if (Op == PHI)
7923 std::swap(Op, PhiOp);
7924 }
7925 }
7926 if (PhiOp != PHI)
7927 return false;
7928
7929 using namespace llvm::PatternMatch;
7930
7931 // If the update is a binary operator, check both of its operands to see if
7932 // they are extends. Otherwise, see if the update comes directly from an
7933 // extend.
7934 Instruction *Exts[2] = {nullptr};
7935 BinaryOperator *ExtendUser = dyn_cast<BinaryOperator>(Op);
7936 std::optional<unsigned> BinOpc;
7937 Type *ExtOpTypes[2] = {nullptr};
7939
7940 auto CollectExtInfo = [this, &Exts, &ExtOpTypes,
7941 &ExtKinds](SmallVectorImpl<Value *> &Ops) -> bool {
7942 for (const auto &[I, OpI] : enumerate(Ops)) {
7943 Value *ExtOp;
7944 if (!match(OpI, m_ZExtOrSExt(m_Value(ExtOp))))
7945 return false;
7946 Exts[I] = cast<Instruction>(OpI);
7947
7948 // TODO: We should be able to support live-ins.
7949 if (!CM.TheLoop->contains(Exts[I]))
7950 return false;
7951
7952 ExtOpTypes[I] = ExtOp->getType();
7953 ExtKinds[I] = TTI::getPartialReductionExtendKind(Exts[I]);
7954 }
7955 return true;
7956 };
7957
7958 if (ExtendUser) {
7959 if (!ExtendUser->hasOneUse())
7960 return false;
7961
7962 // Use the side-effect of match to replace BinOp only if the pattern is
7963 // matched, we don't care at this point whether it actually matched.
7964 match(ExtendUser, m_Neg(m_BinOp(ExtendUser)));
7965
7966 SmallVector<Value *> Ops(ExtendUser->operands());
7967 if (!CollectExtInfo(Ops))
7968 return false;
7969
7970 BinOpc = std::make_optional(ExtendUser->getOpcode());
7971 } else if (match(Update, m_Add(m_Value(), m_Value()))) {
7972 // We already know the operands for Update are Op and PhiOp.
7974 if (!CollectExtInfo(Ops))
7975 return false;
7976
7977 ExtendUser = Update;
7978 BinOpc = std::nullopt;
7979 } else
7980 return false;
7981
7982 PartialReductionChain Chain(RdxExitInstr, Exts[0], Exts[1], ExtendUser);
7983
7984 TypeSize PHISize = PHI->getType()->getPrimitiveSizeInBits();
7985 TypeSize ASize = ExtOpTypes[0]->getPrimitiveSizeInBits();
7986 if (!PHISize.hasKnownScalarFactor(ASize))
7987 return false;
7988 unsigned TargetScaleFactor = PHISize.getKnownScalarFactor(ASize);
7989
7991 [&](ElementCount VF) {
7993 Update->getOpcode(), ExtOpTypes[0], ExtOpTypes[1],
7994 PHI->getType(), VF, ExtKinds[0], ExtKinds[1], BinOpc,
7995 CM.CostKind);
7996 return Cost.isValid();
7997 },
7998 Range)) {
7999 Chains.emplace_back(Chain, TargetScaleFactor);
8000 return true;
8001 }
8002
8003 return false;
8004}
8005
8007 VFRange &Range) {
8008 // First, check for specific widening recipes that deal with inductions, Phi
8009 // nodes, calls and memory operations.
8010 VPRecipeBase *Recipe;
8011 Instruction *Instr = R->getUnderlyingInstr();
8012 SmallVector<VPValue *, 4> Operands(R->operands());
8013 if (auto *PhiR = dyn_cast<VPPhi>(R)) {
8014 VPBasicBlock *Parent = PhiR->getParent();
8015 [[maybe_unused]] VPRegionBlock *LoopRegionOf =
8016 Parent->getEnclosingLoopRegion();
8017 assert(LoopRegionOf && LoopRegionOf->getEntry() == Parent &&
8018 "Non-header phis should have been handled during predication");
8019 auto *Phi = cast<PHINode>(R->getUnderlyingInstr());
8020 assert(Operands.size() == 2 && "Must have 2 operands for header phis");
8021 if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range)))
8022 return Recipe;
8023
8024 VPHeaderPHIRecipe *PhiRecipe = nullptr;
8025 assert((Legal->isReductionVariable(Phi) ||
8026 Legal->isFixedOrderRecurrence(Phi)) &&
8027 "can only widen reductions and fixed-order recurrences here");
8028 VPValue *StartV = Operands[0];
8029 if (Legal->isReductionVariable(Phi)) {
8030 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(Phi);
8031 assert(RdxDesc.getRecurrenceStartValue() ==
8032 Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
8033
8034 // If the PHI is used by a partial reduction, set the scale factor.
8035 unsigned ScaleFactor =
8036 getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1);
8037 PhiRecipe = new VPReductionPHIRecipe(
8038 Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi),
8039 CM.useOrderedReductions(RdxDesc), ScaleFactor);
8040 } else {
8041 // TODO: Currently fixed-order recurrences are modeled as chains of
8042 // first-order recurrences. If there are no users of the intermediate
8043 // recurrences in the chain, the fixed order recurrence should be modeled
8044 // directly, enabling more efficient codegen.
8045 PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV);
8046 }
8047 // Add backedge value.
8048 PhiRecipe->addOperand(Operands[1]);
8049 return PhiRecipe;
8050 }
8051 assert(!R->isPhi() && "only VPPhi nodes expected at this point");
8052
8053 if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
8054 cast<TruncInst>(Instr), Operands, Range)))
8055 return Recipe;
8056
8057 // All widen recipes below deal only with VF > 1.
8059 [&](ElementCount VF) { return VF.isScalar(); }, Range))
8060 return nullptr;
8061
8062 if (auto *CI = dyn_cast<CallInst>(Instr))
8063 return tryToWidenCall(CI, Operands, Range);
8064
8065 if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
8066 if (auto HistInfo = Legal->getHistogramInfo(SI))
8067 return tryToWidenHistogram(*HistInfo, Operands);
8068
8069 if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
8070 return tryToWidenMemory(Instr, Operands, Range);
8071
8072 if (std::optional<unsigned> ScaleFactor = getScalingForReduction(Instr)) {
8073 if (auto PartialRed =
8074 tryToCreatePartialReduction(Instr, Operands, ScaleFactor.value()))
8075 return PartialRed;
8076 }
8077
8078 if (!shouldWiden(Instr, Range))
8079 return nullptr;
8080
8081 if (auto *GEP = dyn_cast<GetElementPtrInst>(Instr))
8082 return new VPWidenGEPRecipe(GEP, Operands);
8083
8084 if (auto *SI = dyn_cast<SelectInst>(Instr)) {
8085 return new VPWidenSelectRecipe(*SI, Operands);
8086 }
8087
8088 if (auto *CI = dyn_cast<CastInst>(Instr)) {
8089 return new VPWidenCastRecipe(CI->getOpcode(), Operands[0], CI->getType(),
8090 *CI);
8091 }
8092
8093 return tryToWiden(Instr, Operands);
8094}
8095
8099 unsigned ScaleFactor) {
8100 assert(Operands.size() == 2 &&
8101 "Unexpected number of operands for partial reduction");
8102
8103 VPValue *BinOp = Operands[0];
8105 VPRecipeBase *BinOpRecipe = BinOp->getDefiningRecipe();
8106 if (isa<VPReductionPHIRecipe>(BinOpRecipe) ||
8107 isa<VPPartialReductionRecipe>(BinOpRecipe))
8108 std::swap(BinOp, Accumulator);
8109
8110 if (ScaleFactor !=
8111 vputils::getVFScaleFactor(Accumulator->getDefiningRecipe()))
8112 return nullptr;
8113
8114 unsigned ReductionOpcode = Reduction->getOpcode();
8115 if (ReductionOpcode == Instruction::Sub) {
8116 auto *const Zero = ConstantInt::get(Reduction->getType(), 0);
8118 Ops.push_back(Plan.getOrAddLiveIn(Zero));
8119 Ops.push_back(BinOp);
8120 BinOp = new VPWidenRecipe(*Reduction, Ops);
8121 Builder.insert(BinOp->getDefiningRecipe());
8122 ReductionOpcode = Instruction::Add;
8123 }
8124
8125 VPValue *Cond = nullptr;
8126 if (CM.blockNeedsPredicationForAnyReason(Reduction->getParent())) {
8127 assert((ReductionOpcode == Instruction::Add ||
8128 ReductionOpcode == Instruction::Sub) &&
8129 "Expected an ADD or SUB operation for predicated partial "
8130 "reductions (because the neutral element in the mask is zero)!");
8131 Cond = getBlockInMask(Builder.getInsertBlock());
8132 VPValue *Zero =
8133 Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
8134 BinOp = Builder.createSelect(Cond, BinOp, Zero, Reduction->getDebugLoc());
8135 }
8136 return new VPPartialReductionRecipe(ReductionOpcode, Accumulator, BinOp, Cond,
8137 ScaleFactor, Reduction);
8138}
8139
8140void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
8141 ElementCount MaxVF) {
8142 if (ElementCount::isKnownGT(MinVF, MaxVF))
8143 return;
8144
8145 assert(OrigLoop->isInnermost() && "Inner loop expected.");
8146
8147 const LoopAccessInfo *LAI = Legal->getLAI();
8149 OrigLoop, LI, DT, PSE.getSE());
8150 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
8152 // Only use noalias metadata when using memory checks guaranteeing no
8153 // overlap across all iterations.
8154 LVer.prepareNoAliasMetadata();
8155 }
8156
8157 // Create initial base VPlan0, to serve as common starting point for all
8158 // candidates built later for specific VF ranges.
8159 auto VPlan0 = VPlanTransforms::buildVPlan0(
8160 OrigLoop, *LI, Legal->getWidestInductionType(),
8161 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8162
8163 auto MaxVFTimes2 = MaxVF * 2;
8164 for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
8165 VFRange SubRange = {VF, MaxVFTimes2};
8166 if (auto Plan = tryToBuildVPlanWithVPRecipes(
8167 std::unique_ptr<VPlan>(VPlan0->duplicate()), SubRange, &LVer)) {
8168 bool HasScalarVF = Plan->hasScalarVFOnly();
8169 // Now optimize the initial VPlan.
8170 if (!HasScalarVF)
8172 *Plan, CM.getMinimalBitwidths());
8174 // TODO: try to put it close to addActiveLaneMask().
8175 if (CM.foldTailWithEVL() && !HasScalarVF)
8177 *Plan, CM.getMaxSafeElements());
8178 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8179 VPlans.push_back(std::move(Plan));
8180 }
8181 VF = SubRange.End;
8182 }
8183}
8184
8185/// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the
8186/// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute
8187/// the end value of the induction.
8189 VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder,
8190 VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) {
8191 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
8192 // Truncated wide inductions resume from the last lane of their vector value
8193 // in the last vector iteration which is handled elsewhere.
8194 if (WideIntOrFp && WideIntOrFp->getTruncInst())
8195 return nullptr;
8196
8197 VPValue *Start = WideIV->getStartValue();
8198 VPValue *Step = WideIV->getStepValue();
8200 VPValue *EndValue = VectorTC;
8201 if (!WideIntOrFp || !WideIntOrFp->isCanonical()) {
8202 EndValue = VectorPHBuilder.createDerivedIV(
8203 ID.getKind(), dyn_cast_or_null<FPMathOperator>(ID.getInductionBinOp()),
8204 Start, VectorTC, Step);
8205 }
8206
8207 // EndValue is derived from the vector trip count (which has the same type as
8208 // the widest induction) and thus may be wider than the induction here.
8209 Type *ScalarTypeOfWideIV = TypeInfo.inferScalarType(WideIV);
8210 if (ScalarTypeOfWideIV != TypeInfo.inferScalarType(EndValue)) {
8211 EndValue = VectorPHBuilder.createScalarCast(Instruction::Trunc, EndValue,
8212 ScalarTypeOfWideIV,
8213 WideIV->getDebugLoc());
8214 }
8215
8216 auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi(
8217 {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val");
8218 return ResumePhiRecipe;
8219}
8220
8221/// Create resume phis in the scalar preheader for first-order recurrences,
8222/// reductions and inductions, and update the VPIRInstructions wrapping the
8223/// original phis in the scalar header. End values for inductions are added to
8224/// \p IVEndValues.
8225static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
8226 DenseMap<VPValue *, VPValue *> &IVEndValues) {
8227 VPTypeAnalysis TypeInfo(Plan);
8228 auto *ScalarPH = Plan.getScalarPreheader();
8229 auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
8230 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8231 VPBuilder VectorPHBuilder(
8232 cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
8233 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8234 VPBuilder ScalarPHBuilder(ScalarPH);
8235 for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) {
8236 auto *ScalarPhiIRI = cast<VPIRPhi>(&ScalarPhiR);
8237
8238 // TODO: Extract final value from induction recipe initially, optimize to
8239 // pre-computed end value together in optimizeInductionExitUsers.
8240 auto *VectorPhiR =
8241 cast<VPHeaderPHIRecipe>(Builder.getRecipe(&ScalarPhiIRI->getIRPhi()));
8242 if (auto *WideIVR = dyn_cast<VPWidenInductionRecipe>(VectorPhiR)) {
8244 WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo,
8245 &Plan.getVectorTripCount())) {
8246 assert(isa<VPPhi>(ResumePhi) && "Expected a phi");
8247 IVEndValues[WideIVR] = ResumePhi->getOperand(0);
8248 ScalarPhiIRI->addOperand(ResumePhi);
8249 continue;
8250 }
8251 // TODO: Also handle truncated inductions here. Computing end-values
8252 // separately should be done as VPlan-to-VPlan optimization, after
8253 // legalizing all resume values to use the last lane from the loop.
8254 assert(cast<VPWidenIntOrFpInductionRecipe>(VectorPhiR)->getTruncInst() &&
8255 "should only skip truncated wide inductions");
8256 continue;
8257 }
8258
8259 // The backedge value provides the value to resume coming out of a loop,
8260 // which for FORs is a vector whose last element needs to be extracted. The
8261 // start value provides the value if the loop is bypassed.
8262 bool IsFOR = isa<VPFirstOrderRecurrencePHIRecipe>(VectorPhiR);
8263 auto *ResumeFromVectorLoop = VectorPhiR->getBackedgeValue();
8264 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
8265 "Cannot handle loops with uncountable early exits");
8266 if (IsFOR)
8267 ResumeFromVectorLoop = MiddleBuilder.createNaryOp(
8268 VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {},
8269 "vector.recur.extract");
8270 StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx";
8271 auto *ResumePhiR = ScalarPHBuilder.createScalarPhi(
8272 {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name);
8273 ScalarPhiIRI->addOperand(ResumePhiR);
8274 }
8275}
8276
8277/// Handle users in the exit block for first order reductions in the original
8278/// exit block. The penultimate value of recurrences is fed to their LCSSA phi
8279/// users in the original exit block using the VPIRInstruction wrapping to the
8280/// LCSSA phi.
8282 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
8283 auto *ScalarPHVPBB = Plan.getScalarPreheader();
8284 auto *MiddleVPBB = Plan.getMiddleBlock();
8285 VPBuilder ScalarPHBuilder(ScalarPHVPBB);
8286 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
8287
8288 auto IsScalableOne = [](ElementCount VF) -> bool {
8289 return VF == ElementCount::getScalable(1);
8290 };
8291
8292 for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
8293 auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
8294 if (!FOR)
8295 continue;
8296
8297 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
8298 "Cannot handle loops with uncountable early exits");
8299
8300 // This is the second phase of vectorizing first-order recurrences, creating
8301 // extract for users outside the loop. An overview of the transformation is
8302 // described below. Suppose we have the following loop with some use after
8303 // the loop of the last a[i-1],
8304 //
8305 // for (int i = 0; i < n; ++i) {
8306 // t = a[i - 1];
8307 // b[i] = a[i] - t;
8308 // }
8309 // use t;
8310 //
8311 // There is a first-order recurrence on "a". For this loop, the shorthand
8312 // scalar IR looks like:
8313 //
8314 // scalar.ph:
8315 // s.init = a[-1]
8316 // br scalar.body
8317 //
8318 // scalar.body:
8319 // i = phi [0, scalar.ph], [i+1, scalar.body]
8320 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
8321 // s2 = a[i]
8322 // b[i] = s2 - s1
8323 // br cond, scalar.body, exit.block
8324 //
8325 // exit.block:
8326 // use = lcssa.phi [s1, scalar.body]
8327 //
8328 // In this example, s1 is a recurrence because it's value depends on the
8329 // previous iteration. In the first phase of vectorization, we created a
8330 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
8331 // for users in the scalar preheader and exit block.
8332 //
8333 // vector.ph:
8334 // v_init = vector(..., ..., ..., a[-1])
8335 // br vector.body
8336 //
8337 // vector.body
8338 // i = phi [0, vector.ph], [i+4, vector.body]
8339 // v1 = phi [v_init, vector.ph], [v2, vector.body]
8340 // v2 = a[i, i+1, i+2, i+3]
8341 // b[i] = v2 - v1
8342 // // Next, third phase will introduce v1' = splice(v1(3), v2(0, 1, 2))
8343 // b[i, i+1, i+2, i+3] = v2 - v1
8344 // br cond, vector.body, middle.block
8345 //
8346 // middle.block:
8347 // vector.recur.extract.for.phi = v2(2)
8348 // vector.recur.extract = v2(3)
8349 // br cond, scalar.ph, exit.block
8350 //
8351 // scalar.ph:
8352 // scalar.recur.init = phi [vector.recur.extract, middle.block],
8353 // [s.init, otherwise]
8354 // br scalar.body
8355 //
8356 // scalar.body:
8357 // i = phi [0, scalar.ph], [i+1, scalar.body]
8358 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
8359 // s2 = a[i]
8360 // b[i] = s2 - s1
8361 // br cond, scalar.body, exit.block
8362 //
8363 // exit.block:
8364 // lo = lcssa.phi [s1, scalar.body],
8365 // [vector.recur.extract.for.phi, middle.block]
8366 //
8367 // Now update VPIRInstructions modeling LCSSA phis in the exit block.
8368 // Extract the penultimate value of the recurrence and use it as operand for
8369 // the VPIRInstruction modeling the phi.
8370 for (VPUser *U : FOR->users()) {
8371 using namespace llvm::VPlanPatternMatch;
8372 if (!match(U, m_ExtractLastElement(m_Specific(FOR))))
8373 continue;
8374 // For VF vscale x 1, if vscale = 1, we are unable to extract the
8375 // penultimate value of the recurrence. Instead we rely on the existing
8376 // extract of the last element from the result of
8377 // VPInstruction::FirstOrderRecurrenceSplice.
8378 // TODO: Consider vscale_range info and UF.
8380 Range))
8381 return;
8382 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
8383 VPInstruction::ExtractPenultimateElement, {FOR->getBackedgeValue()},
8384 {}, "vector.recur.extract.for.phi");
8385 cast<VPInstruction>(U)->replaceAllUsesWith(PenultimateElement);
8386 }
8387 }
8388}
8389
8390VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8391 VPlanPtr Plan, VFRange &Range, LoopVersioning *LVer) {
8392
8393 using namespace llvm::VPlanPatternMatch;
8394 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
8395
8396 // ---------------------------------------------------------------------------
8397 // Build initial VPlan: Scan the body of the loop in a topological order to
8398 // visit each basic block after having visited its predecessor basic blocks.
8399 // ---------------------------------------------------------------------------
8400
8401 bool RequiresScalarEpilogueCheck =
8403 [this](ElementCount VF) {
8404 return !CM.requiresScalarEpilogue(VF.isVector());
8405 },
8406 Range);
8407 VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
8408 VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
8409 CM.foldTailByMasking());
8410
8412
8413 // Don't use getDecisionAndClampRange here, because we don't know the UF
8414 // so this function is better to be conservative, rather than to split
8415 // it up into different VPlans.
8416 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
8417 bool IVUpdateMayOverflow = false;
8418 for (ElementCount VF : Range)
8419 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(&CM, VF);
8420
8421 TailFoldingStyle Style = CM.getTailFoldingStyle(IVUpdateMayOverflow);
8422 // Use NUW for the induction increment if we proved that it won't overflow in
8423 // the vector loop or when not folding the tail. In the later case, we know
8424 // that the canonical induction increment will not overflow as the vector trip
8425 // count is >= increment and a multiple of the increment.
8426 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
8427 if (!HasNUW) {
8428 auto *IVInc = Plan->getVectorLoopRegion()
8429 ->getExitingBasicBlock()
8430 ->getTerminator()
8431 ->getOperand(0);
8432 assert(match(IVInc, m_VPInstruction<Instruction::Add>(
8433 m_Specific(Plan->getCanonicalIV()), m_VPValue())) &&
8434 "Did not find the canonical IV increment");
8435 cast<VPRecipeWithIRFlags>(IVInc)->dropPoisonGeneratingFlags();
8436 }
8437
8438 // ---------------------------------------------------------------------------
8439 // Pre-construction: record ingredients whose recipes we'll need to further
8440 // process after constructing the initial VPlan.
8441 // ---------------------------------------------------------------------------
8442
8443 // For each interleave group which is relevant for this (possibly trimmed)
8444 // Range, add it to the set of groups to be later applied to the VPlan and add
8445 // placeholders for its members' Recipes which we'll be replacing with a
8446 // single VPInterleaveRecipe.
8447 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
8448 auto ApplyIG = [IG, this](ElementCount VF) -> bool {
8449 bool Result = (VF.isVector() && // Query is illegal for VF == 1
8450 CM.getWideningDecision(IG->getInsertPos(), VF) ==
8452 // For scalable vectors, the interleave factors must be <= 8 since we
8453 // require the (de)interleaveN intrinsics instead of shufflevectors.
8454 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
8455 "Unsupported interleave factor for scalable vectors");
8456 return Result;
8457 };
8458 if (!getDecisionAndClampRange(ApplyIG, Range))
8459 continue;
8460 InterleaveGroups.insert(IG);
8461 }
8462
8463 // ---------------------------------------------------------------------------
8464 // Predicate and linearize the top-level loop region.
8465 // ---------------------------------------------------------------------------
8466 auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(
8467 *Plan, CM.foldTailByMasking());
8468
8469 // ---------------------------------------------------------------------------
8470 // Construct wide recipes and apply predication for original scalar
8471 // VPInstructions in the loop.
8472 // ---------------------------------------------------------------------------
8473 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8474 Builder, BlockMaskCache, LVer);
8475 RecipeBuilder.collectScaledReductions(Range);
8476
8477 // Scan the body of the loop in a topological order to visit each basic block
8478 // after having visited its predecessor basic blocks.
8479 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8480 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
8481 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
8482 HeaderVPBB);
8483
8484 auto *MiddleVPBB = Plan->getMiddleBlock();
8485 VPBasicBlock::iterator MBIP = MiddleVPBB->getFirstNonPhi();
8486 // Mapping from VPValues in the initial plan to their widened VPValues. Needed
8487 // temporarily to update created block masks.
8488 DenseMap<VPValue *, VPValue *> Old2New;
8489 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
8490 // Convert input VPInstructions to widened recipes.
8491 for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
8492 auto *SingleDef = cast<VPSingleDefRecipe>(&R);
8493 auto *UnderlyingValue = SingleDef->getUnderlyingValue();
8494 // Skip recipes that do not need transforming, including canonical IV,
8495 // wide canonical IV and VPInstructions without underlying values. The
8496 // latter are added above for masking.
8497 // FIXME: Migrate code relying on the underlying instruction from VPlan0
8498 // to construct recipes below to not use the underlying instruction.
8500 &R) ||
8501 (isa<VPInstruction>(&R) && !UnderlyingValue))
8502 continue;
8503
8504 // FIXME: VPlan0, which models a copy of the original scalar loop, should
8505 // not use VPWidenPHIRecipe to model the phis.
8507 UnderlyingValue && "unsupported recipe");
8508
8509 // TODO: Gradually replace uses of underlying instruction by analyses on
8510 // VPlan.
8511 Instruction *Instr = cast<Instruction>(UnderlyingValue);
8512 Builder.setInsertPoint(SingleDef);
8513
8514 // The stores with invariant address inside the loop will be deleted, and
8515 // in the exit block, a uniform store recipe will be created for the final
8516 // invariant store of the reduction.
8517 StoreInst *SI;
8518 if ((SI = dyn_cast<StoreInst>(Instr)) &&
8519 Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
8520 // Only create recipe for the final invariant store of the reduction.
8521 if (Legal->isInvariantStoreOfReduction(SI)) {
8522 auto *Recipe =
8523 new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
8524 nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
8525 Recipe->insertBefore(*MiddleVPBB, MBIP);
8526 }
8527 R.eraseFromParent();
8528 continue;
8529 }
8530
8531 VPRecipeBase *Recipe =
8532 RecipeBuilder.tryToCreateWidenRecipe(SingleDef, Range);
8533 if (!Recipe)
8534 Recipe = RecipeBuilder.handleReplication(Instr, R.operands(), Range);
8535
8536 RecipeBuilder.setRecipe(Instr, Recipe);
8537 if (isa<VPWidenIntOrFpInductionRecipe>(Recipe) && isa<TruncInst>(Instr)) {
8538 // Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
8539 // moved to the phi section in the header.
8540 Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi());
8541 } else {
8542 Builder.insert(Recipe);
8543 }
8544 if (Recipe->getNumDefinedValues() == 1) {
8545 SingleDef->replaceAllUsesWith(Recipe->getVPSingleValue());
8546 Old2New[SingleDef] = Recipe->getVPSingleValue();
8547 } else {
8548 assert(Recipe->getNumDefinedValues() == 0 &&
8549 "Unexpected multidef recipe");
8550 R.eraseFromParent();
8551 }
8552 }
8553 }
8554
8555 // replaceAllUsesWith above may invalidate the block masks. Update them here.
8556 // TODO: Include the masks as operands in the predicated VPlan directly
8557 // to remove the need to keep a map of masks beyond the predication
8558 // transform.
8559 RecipeBuilder.updateBlockMaskCache(Old2New);
8560 for (VPValue *Old : Old2New.keys())
8561 Old->getDefiningRecipe()->eraseFromParent();
8562
8563 assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
8564 !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8565 "entry block must be set to a VPRegionBlock having a non-empty entry "
8566 "VPBasicBlock");
8567
8568 // Update wide induction increments to use the same step as the corresponding
8569 // wide induction. This enables detecting induction increments directly in
8570 // VPlan and removes redundant splats.
8571 for (const auto &[Phi, ID] : Legal->getInductionVars()) {
8572 auto *IVInc = cast<Instruction>(
8573 Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch()));
8574 if (IVInc->getOperand(0) != Phi || IVInc->getOpcode() != Instruction::Add)
8575 continue;
8576 VPWidenInductionRecipe *WideIV =
8577 cast<VPWidenInductionRecipe>(RecipeBuilder.getRecipe(Phi));
8578 VPRecipeBase *R = RecipeBuilder.getRecipe(IVInc);
8579 R->setOperand(1, WideIV->getStepValue());
8580 }
8581
8583 DenseMap<VPValue *, VPValue *> IVEndValues;
8584 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
8585
8586 // ---------------------------------------------------------------------------
8587 // Transform initial VPlan: Apply previously taken decisions, in order, to
8588 // bring the VPlan to its final state.
8589 // ---------------------------------------------------------------------------
8590
8591 // Adjust the recipes for any inloop reductions.
8592 adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
8593
8594 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
8595 // NaNs if possible, bail out otherwise.
8597 *Plan))
8598 return nullptr;
8599
8600 // Transform recipes to abstract recipes if it is legal and beneficial and
8601 // clamp the range for better cost estimation.
8602 // TODO: Enable following transform when the EVL-version of extended-reduction
8603 // and mulacc-reduction are implemented.
8604 if (!CM.foldTailWithEVL()) {
8605 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind,
8606 *CM.PSE.getSE());
8608 CostCtx, Range);
8609 }
8610
8611 for (ElementCount VF : Range)
8612 Plan->addVF(VF);
8613 Plan->setName("Initial VPlan");
8614
8615 // Interleave memory: for each Interleave Group we marked earlier as relevant
8616 // for this VPlan, replace the Recipes widening its memory instructions with a
8617 // single VPInterleaveRecipe at its insertion point.
8619 InterleaveGroups, RecipeBuilder,
8620 CM.isScalarEpilogueAllowed());
8621
8622 // Replace VPValues for known constant strides.
8624 Legal->getLAI()->getSymbolicStrides());
8625
8626 auto BlockNeedsPredication = [this](BasicBlock *BB) {
8627 return Legal->blockNeedsPredication(BB);
8628 };
8630 BlockNeedsPredication);
8631
8632 // Sink users of fixed-order recurrence past the recipe defining the previous
8633 // value and introduce FirstOrderRecurrenceSplice VPInstructions.
8635 *Plan, Builder))
8636 return nullptr;
8637
8638 if (useActiveLaneMask(Style)) {
8639 // TODO: Move checks to VPlanTransforms::addActiveLaneMask once
8640 // TailFoldingStyle is visible there.
8641 bool ForControlFlow = useActiveLaneMaskForControlFlow(Style);
8642 bool WithoutRuntimeCheck =
8643 Style == TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck;
8644 VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
8645 WithoutRuntimeCheck);
8646 }
8647 VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues, *PSE.getSE());
8648
8649 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8650 return Plan;
8651}
8652
8653VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
8654 // Outer loop handling: They may require CFG and instruction level
8655 // transformations before even evaluating whether vectorization is profitable.
8656 // Since we cannot modify the incoming IR, we need to build VPlan upfront in
8657 // the vectorization pipeline.
8658 assert(!OrigLoop->isInnermost());
8659 assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
8660
8661 auto Plan = VPlanTransforms::buildVPlan0(
8662 OrigLoop, *LI, Legal->getWidestInductionType(),
8663 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
8665 /*HasUncountableExit*/ false);
8666 VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
8667 /*TailFolded*/ false);
8668
8670
8671 for (ElementCount VF : Range)
8672 Plan->addVF(VF);
8673
8675 Plan,
8676 [this](PHINode *P) {
8677 return Legal->getIntOrFpInductionDescriptor(P);
8678 },
8679 *TLI))
8680 return nullptr;
8681
8682 // Collect mapping of IR header phis to header phi recipes, to be used in
8683 // addScalarResumePhis.
8684 DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
8685 VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
8686 Builder, BlockMaskCache, nullptr /*LVer*/);
8687 for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8689 continue;
8690 auto *HeaderR = cast<VPHeaderPHIRecipe>(&R);
8691 RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR);
8692 }
8693 DenseMap<VPValue *, VPValue *> IVEndValues;
8694 // TODO: IVEndValues are not used yet in the native path, to optimize exit
8695 // values.
8696 addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
8697
8698 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
8699 return Plan;
8700}
8701
8702// Adjust the recipes for reductions. For in-loop reductions the chain of
8703// instructions leading from the loop exit instr to the phi need to be converted
8704// to reductions, with one operand being vector and the other being the scalar
8705// reduction chain. For other reductions, a select is introduced between the phi
8706// and users outside the vector region when folding the tail.
8707//
8708// A ComputeReductionResult recipe is added to the middle block, also for
8709// in-loop reductions which compute their result in-loop, because generating
8710// the subsequent bc.merge.rdx phi is driven by ComputeReductionResult recipes.
8711//
8712// Adjust AnyOf reductions; replace the reduction phi for the selected value
8713// with a boolean reduction phi node to check if the condition is true in any
8714// iteration. The final value is selected by the final ComputeReductionResult.
8715void LoopVectorizationPlanner::adjustRecipesForReductions(
8716 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
8717 using namespace VPlanPatternMatch;
8718 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
8719 VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
8720 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
8722
8723 for (VPRecipeBase &R : Header->phis()) {
8724 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8725 if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
8726 continue;
8727
8728 RecurKind Kind = PhiR->getRecurrenceKind();
8729 assert(
8732 "AnyOf and FindIV reductions are not allowed for in-loop reductions");
8733
8734 // Collect the chain of "link" recipes for the reduction starting at PhiR.
8735 SetVector<VPSingleDefRecipe *> Worklist;
8736 Worklist.insert(PhiR);
8737 for (unsigned I = 0; I != Worklist.size(); ++I) {
8738 VPSingleDefRecipe *Cur = Worklist[I];
8739 for (VPUser *U : Cur->users()) {
8740 auto *UserRecipe = cast<VPSingleDefRecipe>(U);
8741 if (!UserRecipe->getParent()->getEnclosingLoopRegion()) {
8742 assert((UserRecipe->getParent() == MiddleVPBB ||
8743 UserRecipe->getParent() == Plan->getScalarPreheader()) &&
8744 "U must be either in the loop region, the middle block or the "
8745 "scalar preheader.");
8746 continue;
8747 }
8748 Worklist.insert(UserRecipe);
8749 }
8750 }
8751
8752 // Visit operation "Links" along the reduction chain top-down starting from
8753 // the phi until LoopExitValue. We keep track of the previous item
8754 // (PreviousLink) to tell which of the two operands of a Link will remain
8755 // scalar and which will be reduced. For minmax by select(cmp), Link will be
8756 // the select instructions. Blend recipes of in-loop reduction phi's will
8757 // get folded to their non-phi operand, as the reduction recipe handles the
8758 // condition directly.
8759 VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
8760 for (VPSingleDefRecipe *CurrentLink : drop_begin(Worklist)) {
8761 if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentLink)) {
8762 assert(Blend->getNumIncomingValues() == 2 &&
8763 "Blend must have 2 incoming values");
8764 if (Blend->getIncomingValue(0) == PhiR) {
8765 Blend->replaceAllUsesWith(Blend->getIncomingValue(1));
8766 } else {
8767 assert(Blend->getIncomingValue(1) == PhiR &&
8768 "PhiR must be an operand of the blend");
8769 Blend->replaceAllUsesWith(Blend->getIncomingValue(0));
8770 }
8771 continue;
8772 }
8773
8774 Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
8775
8776 // Index of the first operand which holds a non-mask vector operand.
8777 unsigned IndexOfFirstOperand;
8778 // Recognize a call to the llvm.fmuladd intrinsic.
8779 bool IsFMulAdd = (Kind == RecurKind::FMulAdd);
8780 VPValue *VecOp;
8781 VPBasicBlock *LinkVPBB = CurrentLink->getParent();
8782 if (IsFMulAdd) {
8783 assert(
8785 "Expected instruction to be a call to the llvm.fmuladd intrinsic");
8786 assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
8787 isa<VPWidenIntrinsicRecipe>(CurrentLink)) &&
8788 CurrentLink->getOperand(2) == PreviousLink &&
8789 "expected a call where the previous link is the added operand");
8790
8791 // If the instruction is a call to the llvm.fmuladd intrinsic then we
8792 // need to create an fmul recipe (multiplying the first two operands of
8793 // the fmuladd together) to use as the vector operand for the fadd
8794 // reduction.
8795 VPInstruction *FMulRecipe = new VPInstruction(
8796 Instruction::FMul,
8797 {CurrentLink->getOperand(0), CurrentLink->getOperand(1)},
8798 CurrentLinkI->getFastMathFlags());
8799 LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
8800 VecOp = FMulRecipe;
8801 } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
8802 CurrentLinkI->getOpcode() == Instruction::Sub) {
8803 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8804 auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
8805 VPWidenRecipe *Sub = new VPWidenRecipe(
8806 Instruction::Sub, {Zero, CurrentLink->getOperand(1)}, {},
8807 VPIRMetadata(), CurrentLinkI->getDebugLoc());
8808 Sub->setUnderlyingValue(CurrentLinkI);
8809 LinkVPBB->insert(Sub, CurrentLink->getIterator());
8810 VecOp = Sub;
8811 } else {
8813 if (isa<VPWidenRecipe>(CurrentLink)) {
8814 assert(isa<CmpInst>(CurrentLinkI) &&
8815 "need to have the compare of the select");
8816 continue;
8817 }
8818 assert(isa<VPWidenSelectRecipe>(CurrentLink) &&
8819 "must be a select recipe");
8820 IndexOfFirstOperand = 1;
8821 } else {
8822 assert((MinVF.isScalar() || isa<VPWidenRecipe>(CurrentLink)) &&
8823 "Expected to replace a VPWidenSC");
8824 IndexOfFirstOperand = 0;
8825 }
8826 // Note that for non-commutable operands (cmp-selects), the semantics of
8827 // the cmp-select are captured in the recurrence kind.
8828 unsigned VecOpId =
8829 CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
8830 ? IndexOfFirstOperand + 1
8831 : IndexOfFirstOperand;
8832 VecOp = CurrentLink->getOperand(VecOpId);
8833 assert(VecOp != PreviousLink &&
8834 CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
8835 (VecOpId - IndexOfFirstOperand)) ==
8836 PreviousLink &&
8837 "PreviousLink must be the operand other than VecOp");
8838 }
8839
8840 VPValue *CondOp = nullptr;
8841 if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
8842 CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
8843
8844 // TODO: Retrieve FMFs from recipes directly.
8845 RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor(
8846 cast<PHINode>(PhiR->getUnderlyingInstr()));
8847 // Non-FP RdxDescs will have all fast math flags set, so clear them.
8848 FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
8849 ? RdxDesc.getFastMathFlags()
8850 : FastMathFlags();
8851 auto *RedRecipe = new VPReductionRecipe(
8852 Kind, FMFs, CurrentLinkI, PreviousLink, VecOp, CondOp,
8853 PhiR->isOrdered(), CurrentLinkI->getDebugLoc());
8854 // Append the recipe to the end of the VPBasicBlock because we need to
8855 // ensure that it comes after all of it's inputs, including CondOp.
8856 // Delete CurrentLink as it will be invalid if its operand is replaced
8857 // with a reduction defined at the bottom of the block in the next link.
8858 if (LinkVPBB->getNumSuccessors() == 0)
8859 RedRecipe->insertBefore(&*std::prev(std::prev(LinkVPBB->end())));
8860 else
8861 LinkVPBB->appendRecipe(RedRecipe);
8862
8863 CurrentLink->replaceAllUsesWith(RedRecipe);
8864 ToDelete.push_back(CurrentLink);
8865 PreviousLink = RedRecipe;
8866 }
8867 }
8868 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
8869 Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
8870 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
8871 for (VPRecipeBase &R :
8872 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
8873 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
8874 if (!PhiR)
8875 continue;
8876
8877 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
8879 Type *PhiTy = PhiR->getUnderlyingValue()->getType();
8880 // If tail is folded by masking, introduce selects between the phi
8881 // and the users outside the vector region of each reduction, at the
8882 // beginning of the dedicated latch block.
8883 auto *OrigExitingVPV = PhiR->getBackedgeValue();
8884 auto *NewExitingVPV = PhiR->getBackedgeValue();
8885 // Don't output selects for partial reductions because they have an output
8886 // with fewer lanes than the VF. So the operands of the select would have
8887 // different numbers of lanes. Partial reductions mask the input instead.
8888 if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
8889 !isa<VPPartialReductionRecipe>(OrigExitingVPV->getDefiningRecipe())) {
8890 VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
8891 std::optional<FastMathFlags> FMFs =
8892 PhiTy->isFloatingPointTy()
8893 ? std::make_optional(RdxDesc.getFastMathFlags())
8894 : std::nullopt;
8895 NewExitingVPV =
8896 Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", FMFs);
8897 OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
8898 return isa<VPInstruction>(&U) &&
8899 (cast<VPInstruction>(&U)->getOpcode() ==
8901 cast<VPInstruction>(&U)->getOpcode() ==
8903 cast<VPInstruction>(&U)->getOpcode() ==
8905 });
8906 if (CM.usePredicatedReductionSelect())
8907 PhiR->setOperand(1, NewExitingVPV);
8908 }
8909
8910 // We want code in the middle block to appear to execute on the location of
8911 // the scalar loop's latch terminator because: (a) it is all compiler
8912 // generated, (b) these instructions are always executed after evaluating
8913 // the latch conditional branch, and (c) other passes may add new
8914 // predecessors which terminate on this line. This is the easiest way to
8915 // ensure we don't accidentally cause an extra step back into the loop while
8916 // debugging.
8917 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
8918
8919 // TODO: At the moment ComputeReductionResult also drives creation of the
8920 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
8921 // even for in-loop reductions, until the reduction resume value handling is
8922 // also modeled in VPlan.
8923 VPInstruction *FinalReductionResult;
8924 VPBuilder::InsertPointGuard Guard(Builder);
8925 Builder.setInsertPoint(MiddleVPBB, IP);
8926 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
8928 VPValue *Start = PhiR->getStartValue();
8929 VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue());
8930 FinalReductionResult =
8931 Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
8932 {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL);
8933 } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
8934 VPValue *Start = PhiR->getStartValue();
8935 FinalReductionResult =
8936 Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
8937 {PhiR, Start, NewExitingVPV}, ExitDL);
8938 } else {
8939 VPIRFlags Flags =
8941 ? VPIRFlags(RdxDesc.getFastMathFlags())
8942 : VPIRFlags();
8943 FinalReductionResult =
8944 Builder.createNaryOp(VPInstruction::ComputeReductionResult,
8945 {PhiR, NewExitingVPV}, Flags, ExitDL);
8946 }
8947 // If the vector reduction can be performed in a smaller type, we truncate
8948 // then extend the loop exit value to enable InstCombine to evaluate the
8949 // entire expression in the smaller type.
8950 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
8952 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
8954 "Unexpected truncated min-max recurrence!");
8955 Type *RdxTy = RdxDesc.getRecurrenceType();
8956 auto *Trunc =
8957 new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
8958 Instruction::CastOps ExtendOpc =
8959 RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
8960 auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
8961 Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
8962 Extnd->insertAfter(Trunc);
8963 if (PhiR->getOperand(1) == NewExitingVPV)
8964 PhiR->setOperand(1, Extnd->getVPSingleValue());
8965
8966 // Update ComputeReductionResult with the truncated exiting value and
8967 // extend its result.
8968 FinalReductionResult->setOperand(1, Trunc);
8969 FinalReductionResult =
8970 Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
8971 }
8972
8973 // Update all users outside the vector region. Also replace redundant
8974 // ExtractLastElement.
8975 for (auto *U : to_vector(OrigExitingVPV->users())) {
8976 auto *Parent = cast<VPRecipeBase>(U)->getParent();
8977 if (FinalReductionResult == U || Parent->getParent())
8978 continue;
8979 U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
8981 cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
8982 }
8983
8984 // Adjust AnyOf reductions; replace the reduction phi for the selected value
8985 // with a boolean reduction phi node to check if the condition is true in
8986 // any iteration. The final value is selected by the final
8987 // ComputeReductionResult.
8988 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
8989 auto *Select = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
8990 return isa<VPWidenSelectRecipe>(U) ||
8991 (isa<VPReplicateRecipe>(U) &&
8992 cast<VPReplicateRecipe>(U)->getUnderlyingInstr()->getOpcode() ==
8993 Instruction::Select);
8994 }));
8995 VPValue *Cmp = Select->getOperand(0);
8996 // If the compare is checking the reduction PHI node, adjust it to check
8997 // the start value.
8998 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
8999 CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
9000 Builder.setInsertPoint(Select);
9001
9002 // If the true value of the select is the reduction phi, the new value is
9003 // selected if the negated condition is true in any iteration.
9004 if (Select->getOperand(1) == PhiR)
9005 Cmp = Builder.createNot(Cmp);
9006 VPValue *Or = Builder.createOr(PhiR, Cmp);
9007 Select->getVPSingleValue()->replaceAllUsesWith(Or);
9008 // Delete Select now that it has invalid types.
9009 ToDelete.push_back(Select);
9010
9011 // Convert the reduction phi to operate on bools.
9012 PhiR->setOperand(0, Plan->getOrAddLiveIn(ConstantInt::getFalse(
9013 OrigLoop->getHeader()->getContext())));
9014 continue;
9015 }
9016
9018 RdxDesc.getRecurrenceKind())) {
9019 // Adjust the start value for FindFirstIV/FindLastIV recurrences to use
9020 // the sentinel value after generating the ResumePhi recipe, which uses
9021 // the original start value.
9022 PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
9023 }
9024 RecurKind RK = RdxDesc.getRecurrenceKind();
9028 VPBuilder PHBuilder(Plan->getVectorPreheader());
9029 VPValue *Iden = Plan->getOrAddLiveIn(
9030 getRecurrenceIdentity(RK, PhiTy, RdxDesc.getFastMathFlags()));
9031 // If the PHI is used by a partial reduction, set the scale factor.
9032 unsigned ScaleFactor =
9033 RecipeBuilder.getScalingForReduction(RdxDesc.getLoopExitInstr())
9034 .value_or(1);
9035 Type *I32Ty = IntegerType::getInt32Ty(PhiTy->getContext());
9036 auto *ScaleFactorVPV =
9037 Plan->getOrAddLiveIn(ConstantInt::get(I32Ty, ScaleFactor));
9038 VPValue *StartV = PHBuilder.createNaryOp(
9040 {PhiR->getStartValue(), Iden, ScaleFactorVPV},
9041 PhiTy->isFloatingPointTy() ? RdxDesc.getFastMathFlags()
9042 : FastMathFlags());
9043 PhiR->setOperand(0, StartV);
9044 }
9045 }
9046 for (VPRecipeBase *R : ToDelete)
9047 R->eraseFromParent();
9048
9050}
9051
9052void LoopVectorizationPlanner::attachRuntimeChecks(
9053 VPlan &Plan, GeneratedRTChecks &RTChecks, bool HasBranchWeights) const {
9054 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
9055 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(0)) {
9056 assert((!CM.OptForSize ||
9057 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
9058 "Cannot SCEV check stride or overflow when optimizing for size");
9059 VPlanTransforms::attachCheckBlock(Plan, SCEVCheckCond, SCEVCheckBlock,
9060 HasBranchWeights);
9061 }
9062 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
9063 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
9064 // VPlan-native path does not do any analysis for runtime checks
9065 // currently.
9066 assert((!EnableVPlanNativePath || OrigLoop->isInnermost()) &&
9067 "Runtime checks are not supported for outer loops yet");
9068
9069 if (CM.OptForSize) {
9070 assert(
9071 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
9072 "Cannot emit memory checks when optimizing for size, unless forced "
9073 "to vectorize.");
9074 ORE->emit([&]() {
9075 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
9076 OrigLoop->getStartLoc(),
9077 OrigLoop->getHeader())
9078 << "Code-size may be reduced by not forcing "
9079 "vectorization, or by source-code modifications "
9080 "eliminating the need for runtime checks "
9081 "(e.g., adding 'restrict').";
9082 });
9083 }
9084 VPlanTransforms::attachCheckBlock(Plan, MemCheckCond, MemCheckBlock,
9085 HasBranchWeights);
9086 }
9087}
9088
9090 VPlan &Plan, ElementCount VF, unsigned UF,
9091 ElementCount MinProfitableTripCount) const {
9092 // vscale is not necessarily a power-of-2, which means we cannot guarantee
9093 // an overflow to zero when updating induction variables and so an
9094 // additional overflow check is required before entering the vector loop.
9095 bool IsIndvarOverflowCheckNeededForVF =
9096 VF.isScalable() && !TTI.isVScaleKnownToBeAPowerOfTwo() &&
9097 !isIndvarOverflowCheckKnownFalse(&CM, VF, UF) &&
9098 CM.getTailFoldingStyle() !=
9100 const uint32_t *BranchWeigths =
9101 hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())
9103 : nullptr;
9105 Plan, VF, UF, MinProfitableTripCount,
9106 CM.requiresScalarEpilogue(VF.isVector()), CM.foldTailByMasking(),
9107 IsIndvarOverflowCheckNeededForVF, OrigLoop, BranchWeigths,
9108 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
9109 *PSE.getSE());
9110}
9111
9113 assert(!State.Lane && "VPDerivedIVRecipe being replicated.");
9114
9115 // Fast-math-flags propagate from the original induction instruction.
9116 IRBuilder<>::FastMathFlagGuard FMFG(State.Builder);
9117 if (FPBinOp)
9118 State.Builder.setFastMathFlags(FPBinOp->getFastMathFlags());
9119
9120 Value *Step = State.get(getStepValue(), VPLane(0));
9121 Value *Index = State.get(getOperand(1), VPLane(0));
9122 Value *DerivedIV = emitTransformedIndex(
9123 State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
9125 DerivedIV->setName(Name);
9126 State.set(this, DerivedIV, VPLane(0));
9127}
9128
9129// Determine how to lower the scalar epilogue, which depends on 1) optimising
9130// for minimum code-size, 2) predicate compiler options, 3) loop hints forcing
9131// predication, and 4) a TTI hook that analyses whether the loop is suitable
9132// for predication.
9137 // 1) OptSize takes precedence over all other options, i.e. if this is set,
9138 // don't look at hints or options, and don't request a scalar epilogue.
9139 // (For PGSO, as shouldOptimizeForSize isn't currently accessible from
9140 // LoopAccessInfo (due to code dependency and not being able to reliably get
9141 // PSI/BFI from a loop analysis under NPM), we cannot suppress the collection
9142 // of strides in LoopAccessInfo::analyzeLoop() and vectorize without
9143 // versioning when the vectorization is forced, unlike hasOptSize. So revert
9144 // back to the old way and vectorize with versioning when forced. See D81345.)
9145 if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
9149
9150 // 2) If set, obey the directives
9151 if (PreferPredicateOverEpilogue.getNumOccurrences()) {
9159 };
9160 }
9161
9162 // 3) If set, obey the hints
9163 switch (Hints.getPredicate()) {
9168 };
9169
9170 // 4) if the TTI hook indicates this is profitable, request predication.
9171 TailFoldingInfo TFI(TLI, &LVL, IAI);
9172 if (TTI->preferPredicateOverEpilogue(&TFI))
9174
9176}
9177
9178// Process the loop in the VPlan-native vectorization path. This path builds
9179// VPlan upfront in the vectorization pipeline, which allows to apply
9180// VPlan-to-VPlan transformations from the very beginning without modifying the
9181// input LLVM IR.
9188 LoopVectorizationRequirements &Requirements) {
9189
9191 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
9192 return false;
9193 }
9194 assert(EnableVPlanNativePath && "VPlan-native path is disabled.");
9195 Function *F = L->getHeader()->getParent();
9196 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
9197
9199 getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, *LVL, &IAI);
9200
9201 LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F,
9202 &Hints, IAI, PSI, BFI);
9203 // Use the planner for outer loop vectorization.
9204 // TODO: CM is not used at this point inside the planner. Turn CM into an
9205 // optional argument if we don't need it in the future.
9206 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, LVL, CM, IAI, PSE, Hints,
9207 ORE);
9208
9209 // Get user vectorization factor.
9210 ElementCount UserVF = Hints.getWidth();
9211
9213
9214 // Plan how to best vectorize, return the best VF and its cost.
9215 const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);
9216
9217 // If we are stress testing VPlan builds, do not attempt to generate vector
9218 // code. Masked vector code generation support will follow soon.
9219 // Also, do not attempt to vectorize if no vector code will be produced.
9221 return false;
9222
9223 VPlan &BestPlan = LVP.getPlanFor(VF.Width);
9224
9225 {
9226 GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
9227 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, /*UF=*/1, &CM,
9228 BFI, PSI, Checks, BestPlan);
9229 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
9230 << L->getHeader()->getParent()->getName() << "\"\n");
9231 LVP.addMinimumIterationCheck(BestPlan, VF.Width, /*UF=*/1,
9233
9234 LVP.executePlan(VF.Width, /*UF=*/1, BestPlan, LB, DT, false);
9235 }
9236
9237 reportVectorization(ORE, L, VF, 1);
9238
9239 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
9240 return true;
9241}
9242
9243// Emit a remark if there are stores to floats that required a floating point
9244// extension. If the vectorized loop was generated with floating point there
9245// will be a performance penalty from the conversion overhead and the change in
9246// the vector width.
9249 for (BasicBlock *BB : L->getBlocks()) {
9250 for (Instruction &Inst : *BB) {
9251 if (auto *S = dyn_cast<StoreInst>(&Inst)) {
9252 if (S->getValueOperand()->getType()->isFloatTy())
9253 Worklist.push_back(S);
9254 }
9255 }
9256 }
9257
9258 // Traverse the floating point stores upwards searching, for floating point
9259 // conversions.
9262 while (!Worklist.empty()) {
9263 auto *I = Worklist.pop_back_val();
9264 if (!L->contains(I))
9265 continue;
9266 if (!Visited.insert(I).second)
9267 continue;
9268
9269 // Emit a remark if the floating point store required a floating
9270 // point conversion.
9271 // TODO: More work could be done to identify the root cause such as a
9272 // constant or a function return type and point the user to it.
9273 if (isa<FPExtInst>(I) && EmittedRemark.insert(I).second)
9274 ORE->emit([&]() {
9275 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
9276 I->getDebugLoc(), L->getHeader())
9277 << "floating point conversion changes vector width. "
9278 << "Mixed floating point precision requires an up/down "
9279 << "cast that will negatively impact performance.";
9280 });
9281
9282 for (Use &Op : I->operands())
9283 if (auto *OpI = dyn_cast<Instruction>(Op))
9284 Worklist.push_back(OpI);
9285 }
9286}
9287
9288/// For loops with uncountable early exits, find the cost of doing work when
9289/// exiting the loop early, such as calculating the final exit values of
9290/// variables used outside the loop.
9291/// TODO: This is currently overly pessimistic because the loop may not take
9292/// the early exit, but better to keep this conservative for now. In future,
9293/// it might be possible to relax this by using branch probabilities.
9295 VPlan &Plan, ElementCount VF) {
9296 InstructionCost Cost = 0;
9297 for (auto *ExitVPBB : Plan.getExitBlocks()) {
9298 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
9299 // If the predecessor is not the middle.block, then it must be the
9300 // vector.early.exit block, which may contain work to calculate the exit
9301 // values of variables used outside the loop.
9302 if (PredVPBB != Plan.getMiddleBlock()) {
9303 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
9304 << PredVPBB->getName() << ":\n");
9305 Cost += PredVPBB->cost(VF, CostCtx);
9306 }
9307 }
9308 }
9309 return Cost;
9310}
9311
9312/// This function determines whether or not it's still profitable to vectorize
9313/// the loop given the extra work we have to do outside of the loop:
9314/// 1. Perform the runtime checks before entering the loop to ensure it's safe
9315/// to vectorize.
9316/// 2. In the case of loops with uncountable early exits, we may have to do
9317/// extra work when exiting the loop early, such as calculating the final
9318/// exit values of variables used outside the loop.
9319static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
9320 VectorizationFactor &VF, Loop *L,
9322 VPCostContext &CostCtx, VPlan &Plan,
9324 std::optional<unsigned> VScale) {
9325 InstructionCost TotalCost = Checks.getCost();
9326 if (!TotalCost.isValid())
9327 return false;
9328
9329 // Add on the cost of any work required in the vector early exit block, if
9330 // one exists.
9331 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF.Width);
9332
9333 // When interleaving only scalar and vector cost will be equal, which in turn
9334 // would lead to a divide by 0. Fall back to hard threshold.
9335 if (VF.Width.isScalar()) {
9336 // TODO: Should we rename VectorizeMemoryCheckThreshold?
9337 if (TotalCost > VectorizeMemoryCheckThreshold) {
9338 LLVM_DEBUG(
9339 dbgs()
9340 << "LV: Interleaving only is not profitable due to runtime checks\n");
9341 return false;
9342 }
9343 return true;
9344 }
9345
9346 // The scalar cost should only be 0 when vectorizing with a user specified
9347 // VF/IC. In those cases, runtime checks should always be generated.
9348 uint64_t ScalarC = VF.ScalarCost.getValue();
9349 if (ScalarC == 0)
9350 return true;
9351
9352 // First, compute the minimum iteration count required so that the vector
9353 // loop outperforms the scalar loop.
9354 // The total cost of the scalar loop is
9355 // ScalarC * TC
9356 // where
9357 // * TC is the actual trip count of the loop.
9358 // * ScalarC is the cost of a single scalar iteration.
9359 //
9360 // The total cost of the vector loop is
9361 // RtC + VecC * (TC / VF) + EpiC
9362 // where
9363 // * RtC is the cost of the generated runtime checks plus the cost of
9364 // performing any additional work in the vector.early.exit block for loops
9365 // with uncountable early exits.
9366 // * VecC is the cost of a single vector iteration.
9367 // * TC is the actual trip count of the loop
9368 // * VF is the vectorization factor
9369 // * EpiCost is the cost of the generated epilogue, including the cost
9370 // of the remaining scalar operations.
9371 //
9372 // Vectorization is profitable once the total vector cost is less than the
9373 // total scalar cost:
9374 // RtC + VecC * (TC / VF) + EpiC < ScalarC * TC
9375 //
9376 // Now we can compute the minimum required trip count TC as
9377 // VF * (RtC + EpiC) / (ScalarC * VF - VecC) < TC
9378 //
9379 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
9380 // the computations are performed on doubles, not integers and the result
9381 // is rounded up, hence we get an upper estimate of the TC.
9382 unsigned IntVF = estimateElementCount(VF.Width, VScale);
9383 uint64_t RtC = TotalCost.getValue();
9384 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
9385 uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
9386
9387 // Second, compute a minimum iteration count so that the cost of the
9388 // runtime checks is only a fraction of the total scalar loop cost. This
9389 // adds a loop-dependent bound on the overhead incurred if the runtime
9390 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
9391 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
9392 // cost, compute
9393 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
9394 uint64_t MinTC2 = divideCeil(RtC * 10, ScalarC);
9395
9396 // Now pick the larger minimum. If it is not a multiple of VF and a scalar
9397 // epilogue is allowed, choose the next closest multiple of VF. This should
9398 // partly compensate for ignoring the epilogue cost.
9399 uint64_t MinTC = std::max(MinTC1, MinTC2);
9400 if (SEL == CM_ScalarEpilogueAllowed)
9401 MinTC = alignTo(MinTC, IntVF);
9403
9404 LLVM_DEBUG(
9405 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
9406 << VF.MinProfitableTripCount << "\n");
9407
9408 // Skip vectorization if the expected trip count is less than the minimum
9409 // required trip count.
9410 if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
9411 if (ElementCount::isKnownLT(*ExpectedTC, VF.MinProfitableTripCount)) {
9412 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
9413 "trip count < minimum profitable VF ("
9414 << *ExpectedTC << " < " << VF.MinProfitableTripCount
9415 << ")\n");
9416
9417 return false;
9418 }
9419 }
9420 return true;
9421}
9422
9424 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
9426 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
9428
9429/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
9430/// vectorization. Remove ResumePhis from \p MainPlan for inductions that
9431/// don't have a corresponding wide induction in \p EpiPlan.
9432static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
9433 // Collect PHI nodes of widened phis in the VPlan for the epilogue. Those
9434 // will need their resume-values computed in the main vector loop. Others
9435 // can be removed from the main VPlan.
9436 SmallPtrSet<PHINode *, 2> EpiWidenedPhis;
9437 for (VPRecipeBase &R :
9440 continue;
9441 EpiWidenedPhis.insert(
9442 cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
9443 }
9444 for (VPRecipeBase &R :
9445 make_early_inc_range(MainPlan.getScalarHeader()->phis())) {
9446 auto *VPIRInst = cast<VPIRPhi>(&R);
9447 if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
9448 continue;
9449 // There is no corresponding wide induction in the epilogue plan that would
9450 // need a resume value. Remove the VPIRInst wrapping the scalar header phi
9451 // together with the corresponding ResumePhi. The resume values for the
9452 // scalar loop will be created during execution of EpiPlan.
9453 VPRecipeBase *ResumePhi = VPIRInst->getOperand(0)->getDefiningRecipe();
9454 VPIRInst->eraseFromParent();
9455 ResumePhi->eraseFromParent();
9456 }
9458
9459 using namespace VPlanPatternMatch;
9460 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
9461 // introduce multiple uses of undef/poison. If the reduction start value may
9462 // be undef or poison it needs to be frozen and the frozen start has to be
9463 // used when computing the reduction result. We also need to use the frozen
9464 // value in the resume phi generated by the main vector loop, as this is also
9465 // used to compute the reduction result after the epilogue vector loop.
9466 auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
9467 bool UpdateResumePhis) {
9468 VPBuilder Builder(Plan.getEntry());
9469 for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
9470 auto *VPI = dyn_cast<VPInstruction>(&R);
9471 if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
9472 continue;
9473 VPValue *OrigStart = VPI->getOperand(1);
9475 continue;
9476 VPInstruction *Freeze =
9477 Builder.createNaryOp(Instruction::Freeze, {OrigStart}, {}, "fr");
9478 VPI->setOperand(1, Freeze);
9479 if (UpdateResumePhis)
9480 OrigStart->replaceUsesWithIf(Freeze, [Freeze](VPUser &U, unsigned) {
9481 return Freeze != &U && isa<VPPhi>(&U);
9482 });
9483 }
9484 };
9485 AddFreezeForFindLastIVReductions(MainPlan, true);
9486 AddFreezeForFindLastIVReductions(EpiPlan, false);
9487
9488 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
9489 VPValue *VectorTC = &MainPlan.getVectorTripCount();
9490 // If there is a suitable resume value for the canonical induction in the
9491 // scalar (which will become vector) epilogue loop, use it and move it to the
9492 // beginning of the scalar preheader. Otherwise create it below.
9493 auto ResumePhiIter =
9494 find_if(MainScalarPH->phis(), [VectorTC](VPRecipeBase &R) {
9495 return match(&R, m_VPInstruction<Instruction::PHI>(m_Specific(VectorTC),
9496 m_ZeroInt()));
9497 });
9498 VPPhi *ResumePhi = nullptr;
9499 if (ResumePhiIter == MainScalarPH->phis().end()) {
9500 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
9501 ResumePhi = ScalarPHBuilder.createScalarPhi(
9502 {VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
9503 "vec.epilog.resume.val");
9504 } else {
9505 ResumePhi = cast<VPPhi>(&*ResumePhiIter);
9506 if (MainScalarPH->begin() == MainScalarPH->end())
9507 ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->end());
9508 else if (&*MainScalarPH->begin() != ResumePhi)
9509 ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->begin());
9510 }
9511 // Add a user to to make sure the resume phi won't get removed.
9512 VPBuilder(MainScalarPH)
9514}
9515
9516/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
9517/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. Some
9518/// reductions require creating new instructions to compute the resume values.
9519/// They are collected in a vector and returned. They must be moved to the
9520/// preheader of the vector epilogue loop, after created by the execution of \p
9521/// Plan.
9523 VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
9525 ScalarEvolution &SE) {
9526 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
9527 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
9528 Header->setName("vec.epilog.vector.body");
9529
9531 SmallVector<Instruction *> InstsToMove;
9532 // Ensure that the start values for all header phi recipes are updated before
9533 // vectorizing the epilogue loop.
9534 for (VPRecipeBase &R : Header->phis()) {
9535 if (auto *IV = dyn_cast<VPCanonicalIVPHIRecipe>(&R)) {
9536 // When vectorizing the epilogue loop, the canonical induction start
9537 // value needs to be changed from zero to the value after the main
9538 // vector loop. Find the resume value created during execution of the main
9539 // VPlan. It must be the first phi in the loop preheader.
9540 // FIXME: Improve modeling for canonical IV start values in the epilogue
9541 // loop.
9542 using namespace llvm::PatternMatch;
9543 PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
9544 for (Value *Inc : EPResumeVal->incoming_values()) {
9545 if (match(Inc, m_SpecificInt(0)))
9546 continue;
9547 assert(!EPI.VectorTripCount &&
9548 "Must only have a single non-zero incoming value");
9549 EPI.VectorTripCount = Inc;
9550 }
9551 // If we didn't find a non-zero vector trip count, all incoming values
9552 // must be zero, which also means the vector trip count is zero. Pick the
9553 // first zero as vector trip count.
9554 // TODO: We should not choose VF * UF so the main vector loop is known to
9555 // be dead.
9556 if (!EPI.VectorTripCount) {
9557 assert(
9558 EPResumeVal->getNumIncomingValues() > 0 &&
9559 all_of(EPResumeVal->incoming_values(),
9560 [](Value *Inc) { return match(Inc, m_SpecificInt(0)); }) &&
9561 "all incoming values must be 0");
9562 EPI.VectorTripCount = EPResumeVal->getOperand(0);
9563 }
9564 VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
9565 assert(all_of(IV->users(),
9566 [](const VPUser *U) {
9567 return isa<VPScalarIVStepsRecipe>(U) ||
9568 isa<VPDerivedIVRecipe>(U) ||
9569 cast<VPRecipeBase>(U)->isScalarCast() ||
9570 cast<VPInstruction>(U)->getOpcode() ==
9571 Instruction::Add;
9572 }) &&
9573 "the canonical IV should only be used by its increment or "
9574 "ScalarIVSteps when resetting the start value");
9575 IV->setOperand(0, VPV);
9576 continue;
9577 }
9578
9579 Value *ResumeV = nullptr;
9580 // TODO: Move setting of resume values to prepareToExecute.
9581 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
9582 auto *RdxResult =
9583 cast<VPInstruction>(*find_if(ReductionPhi->users(), [](VPUser *U) {
9584 auto *VPI = dyn_cast<VPInstruction>(U);
9585 return VPI &&
9586 (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
9587 VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
9588 VPI->getOpcode() == VPInstruction::ComputeFindIVResult);
9589 }));
9590 ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
9591 ->getIncomingValueForBlock(L->getLoopPreheader());
9592 RecurKind RK = ReductionPhi->getRecurrenceKind();
9594 Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue();
9595 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
9596 // start value; compare the final value from the main vector loop
9597 // to the start value.
9598 BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
9599 IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
9600 ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
9601 if (auto *I = dyn_cast<Instruction>(ResumeV))
9602 InstsToMove.push_back(I);
9604 Value *StartV = getStartValueFromReductionResult(RdxResult);
9605 ToFrozen[StartV] = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
9607
9608 // VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
9609 // an adjustment to the resume value. The resume value is adjusted to
9610 // the sentinel value when the final value from the main vector loop
9611 // equals the start value. This ensures correctness when the start value
9612 // might not be less than the minimum value of a monotonically
9613 // increasing induction variable.
9614 BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
9615 IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
9616 Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
9617 if (auto *I = dyn_cast<Instruction>(Cmp))
9618 InstsToMove.push_back(I);
9619 Value *Sentinel = RdxResult->getOperand(2)->getLiveInIRValue();
9620 ResumeV = Builder.CreateSelect(Cmp, Sentinel, ResumeV);
9621 if (auto *I = dyn_cast<Instruction>(ResumeV))
9622 InstsToMove.push_back(I);
9623 } else {
9624 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
9625 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
9626 if (auto *VPI = dyn_cast<VPInstruction>(PhiR->getStartValue())) {
9627 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
9628 "unexpected start value");
9629 VPI->setOperand(0, StartVal);
9630 continue;
9631 }
9632 }
9633 } else {
9634 // Retrieve the induction resume values for wide inductions from
9635 // their original phi nodes in the scalar loop.
9636 PHINode *IndPhi = cast<VPWidenInductionRecipe>(&R)->getPHINode();
9637 // Hook up to the PHINode generated by a ResumePhi recipe of main
9638 // loop VPlan, which feeds the scalar loop.
9639 ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());
9640 }
9641 assert(ResumeV && "Must have a resume value");
9642 VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
9643 cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
9644 }
9645
9646 // For some VPValues in the epilogue plan we must re-use the generated IR
9647 // values from the main plan. Replace them with live-in VPValues.
9648 // TODO: This is a workaround needed for epilogue vectorization and it
9649 // should be removed once induction resume value creation is done
9650 // directly in VPlan.
9651 for (auto &R : make_early_inc_range(*Plan.getEntry())) {
9652 // Re-use frozen values from the main plan for Freeze VPInstructions in the
9653 // epilogue plan. This ensures all users use the same frozen value.
9654 auto *VPI = dyn_cast<VPInstruction>(&R);
9655 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
9656 VPI->replaceAllUsesWith(Plan.getOrAddLiveIn(
9657 ToFrozen.lookup(VPI->getOperand(0)->getLiveInIRValue())));
9658 continue;
9659 }
9660
9661 // Re-use the trip count and steps expanded for the main loop, as
9662 // skeleton creation needs it as a value that dominates both the scalar
9663 // and vector epilogue loops
9664 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
9665 if (!ExpandR)
9666 continue;
9667 VPValue *ExpandedVal =
9668 Plan.getOrAddLiveIn(ExpandedSCEVs.lookup(ExpandR->getSCEV()));
9669 ExpandR->replaceAllUsesWith(ExpandedVal);
9670 if (Plan.getTripCount() == ExpandR)
9671 Plan.resetTripCount(ExpandedVal);
9672 ExpandR->eraseFromParent();
9673 }
9674
9675 auto VScale = CM.getVScaleForTuning();
9676 unsigned MainLoopStep =
9677 estimateElementCount(EPI.MainLoopVF * EPI.MainLoopUF, VScale);
9678 unsigned EpilogueLoopStep =
9679 estimateElementCount(EPI.EpilogueVF * EPI.EpilogueUF, VScale);
9681 Plan, EPI.TripCount, EPI.VectorTripCount,
9683 EPI.EpilogueUF, MainLoopStep, EpilogueLoopStep, SE);
9684
9685 return InstsToMove;
9686}
9687
9688// Generate bypass values from the additional bypass block. Note that when the
9689// vectorized epilogue is skipped due to iteration count check, then the
9690// resume value for the induction variable comes from the trip count of the
9691// main vector loop, passed as the second argument.
9693 PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder,
9694 const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount,
9695 Instruction *OldInduction) {
9696 Value *Step = getExpandedStep(II, ExpandedSCEVs);
9697 // For the primary induction the additional bypass end value is known.
9698 // Otherwise it is computed.
9699 Value *EndValueFromAdditionalBypass = MainVectorTripCount;
9700 if (OrigPhi != OldInduction) {
9701 auto *BinOp = II.getInductionBinOp();
9702 // Fast-math-flags propagate from the original induction instruction.
9704 BypassBuilder.setFastMathFlags(BinOp->getFastMathFlags());
9705
9706 // Compute the end value for the additional bypass.
9707 EndValueFromAdditionalBypass =
9708 emitTransformedIndex(BypassBuilder, MainVectorTripCount,
9709 II.getStartValue(), Step, II.getKind(), BinOp);
9710 EndValueFromAdditionalBypass->setName("ind.end");
9711 }
9712 return EndValueFromAdditionalBypass;
9713}
9714
9716 VPlan &BestEpiPlan,
9718 const SCEV2ValueTy &ExpandedSCEVs,
9719 Value *MainVectorTripCount) {
9720 // Fix reduction resume values from the additional bypass block.
9721 BasicBlock *PH = L->getLoopPreheader();
9722 for (auto *Pred : predecessors(PH)) {
9723 for (PHINode &Phi : PH->phis()) {
9724 if (Phi.getBasicBlockIndex(Pred) != -1)
9725 continue;
9726 Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
9727 }
9728 }
9729 auto *ScalarPH = cast<VPIRBasicBlock>(BestEpiPlan.getScalarPreheader());
9730 if (ScalarPH->hasPredecessors()) {
9731 // If ScalarPH has predecessors, we may need to update its reduction
9732 // resume values.
9733 for (const auto &[R, IRPhi] :
9734 zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
9736 BypassBlock);
9737 }
9738 }
9739
9740 // Fix induction resume values from the additional bypass block.
9741 IRBuilder<> BypassBuilder(BypassBlock, BypassBlock->getFirstInsertionPt());
9742 for (const auto &[IVPhi, II] : LVL.getInductionVars()) {
9743 auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
9745 IVPhi, II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
9746 LVL.getPrimaryInduction());
9747 // TODO: Directly add as extra operand to the VPResumePHI recipe.
9748 Inc->setIncomingValueForBlock(BypassBlock, V);
9749 }
9750}
9751
9752/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
9753// loop, after both plans have executed, updating branches from the iteration
9754// and runtime checks of the main loop, as well as updating various phis. \p
9755// InstsToMove contains instructions that need to be moved to the preheader of
9756// the epilogue vector loop.
9758 VPlan &EpiPlan, Loop *L, EpilogueLoopVectorizationInfo &EPI,
9760 DenseMap<const SCEV *, Value *> &ExpandedSCEVs, GeneratedRTChecks &Checks,
9761 ArrayRef<Instruction *> InstsToMove) {
9762 BasicBlock *VecEpilogueIterationCountCheck =
9763 cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();
9764
9765 BasicBlock *VecEpiloguePreHeader =
9766 cast<BranchInst>(VecEpilogueIterationCountCheck->getTerminator())
9767 ->getSuccessor(1);
9768 // Adjust the control flow taking the state info from the main loop
9769 // vectorization into account.
9771 "expected this to be saved from the previous pass.");
9772 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
9774 VecEpilogueIterationCountCheck, VecEpiloguePreHeader);
9775
9777 VecEpilogueIterationCountCheck},
9779 VecEpiloguePreHeader}});
9780
9781 BasicBlock *ScalarPH =
9782 cast<VPIRBasicBlock>(EpiPlan.getScalarPreheader())->getIRBasicBlock();
9784 VecEpilogueIterationCountCheck, ScalarPH);
9785 DTU.applyUpdates(
9787 VecEpilogueIterationCountCheck},
9789
9790 // Adjust the terminators of runtime check blocks and phis using them.
9791 BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
9792 BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
9793 if (SCEVCheckBlock) {
9794 SCEVCheckBlock->getTerminator()->replaceUsesOfWith(
9795 VecEpilogueIterationCountCheck, ScalarPH);
9796 DTU.applyUpdates({{DominatorTree::Delete, SCEVCheckBlock,
9797 VecEpilogueIterationCountCheck},
9798 {DominatorTree::Insert, SCEVCheckBlock, ScalarPH}});
9799 }
9800 if (MemCheckBlock) {
9801 MemCheckBlock->getTerminator()->replaceUsesOfWith(
9802 VecEpilogueIterationCountCheck, ScalarPH);
9803 DTU.applyUpdates(
9804 {{DominatorTree::Delete, MemCheckBlock, VecEpilogueIterationCountCheck},
9805 {DominatorTree::Insert, MemCheckBlock, ScalarPH}});
9806 }
9807
9808 // The vec.epilog.iter.check block may contain Phi nodes from inductions
9809 // or reductions which merge control-flow from the latch block and the
9810 // middle block. Update the incoming values here and move the Phi into the
9811 // preheader.
9812 SmallVector<PHINode *, 4> PhisInBlock(
9813 llvm::make_pointer_range(VecEpilogueIterationCountCheck->phis()));
9814
9815 for (PHINode *Phi : PhisInBlock) {
9816 Phi->moveBefore(VecEpiloguePreHeader->getFirstNonPHIIt());
9817 Phi->replaceIncomingBlockWith(
9818 VecEpilogueIterationCountCheck->getSinglePredecessor(),
9819 VecEpilogueIterationCountCheck);
9820
9821 // If the phi doesn't have an incoming value from the
9822 // EpilogueIterationCountCheck, we are done. Otherwise remove the
9823 // incoming value and also those from other check blocks. This is needed
9824 // for reduction phis only.
9825 if (none_of(Phi->blocks(), [&](BasicBlock *IncB) {
9826 return EPI.EpilogueIterationCountCheck == IncB;
9827 }))
9828 continue;
9829 Phi->removeIncomingValue(EPI.EpilogueIterationCountCheck);
9830 if (SCEVCheckBlock)
9831 Phi->removeIncomingValue(SCEVCheckBlock);
9832 if (MemCheckBlock)
9833 Phi->removeIncomingValue(MemCheckBlock);
9834 }
9835
9836 auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
9837 for (auto *I : InstsToMove)
9838 I->moveBefore(IP);
9839
9840 // VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
9841 // after executing the main loop. We need to update the resume values of
9842 // inductions and reductions during epilogue vectorization.
9843 fixScalarResumeValuesFromBypass(VecEpilogueIterationCountCheck, L, EpiPlan,
9844 LVL, ExpandedSCEVs, EPI.VectorTripCount);
9845}
9846
9848 assert((EnableVPlanNativePath || L->isInnermost()) &&
9849 "VPlan-native path is not enabled. Only process inner loops.");
9850
9851 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
9852 << L->getHeader()->getParent()->getName() << "' from "
9853 << L->getLocStr() << "\n");
9854
9855 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
9856
9857 LLVM_DEBUG(
9858 dbgs() << "LV: Loop hints:"
9859 << " force="
9861 ? "disabled"
9863 ? "enabled"
9864 : "?"))
9865 << " width=" << Hints.getWidth()
9866 << " interleave=" << Hints.getInterleave() << "\n");
9867
9868 // Function containing loop
9869 Function *F = L->getHeader()->getParent();
9870
9871 // Looking at the diagnostic output is the only way to determine if a loop
9872 // was vectorized (other than looking at the IR or machine code), so it
9873 // is important to generate an optimization remark for each loop. Most of
9874 // these messages are generated as OptimizationRemarkAnalysis. Remarks
9875 // generated as OptimizationRemark and OptimizationRemarkMissed are
9876 // less verbose reporting vectorized loops and unvectorized loops that may
9877 // benefit from vectorization, respectively.
9878
9879 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
9880 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
9881 return false;
9882 }
9883
9884 PredicatedScalarEvolution PSE(*SE, *L);
9885
9886 // Check if it is legal to vectorize the loop.
9887 LoopVectorizationRequirements Requirements;
9888 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
9889 &Requirements, &Hints, DB, AC, BFI, PSI, AA);
9891 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
9892 Hints.emitRemarkWithHints();
9893 return false;
9894 }
9895
9897 reportVectorizationFailure("Auto-vectorization of loops with uncountable "
9898 "early exit is not enabled",
9899 "UncountableEarlyExitLoopsDisabled", ORE, L);
9900 return false;
9901 }
9902
9903 if (!LVL.getPotentiallyFaultingLoads().empty()) {
9904 reportVectorizationFailure("Auto-vectorization of loops with potentially "
9905 "faulting load is not supported",
9906 "PotentiallyFaultingLoadsNotSupported", ORE, L);
9907 return false;
9908 }
9909
9910 // Entrance to the VPlan-native vectorization path. Outer loops are processed
9911 // here. They may require CFG and instruction level transformations before
9912 // even evaluating whether vectorization is profitable. Since we cannot modify
9913 // the incoming IR, we need to build VPlan upfront in the vectorization
9914 // pipeline.
9915 if (!L->isInnermost())
9916 return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
9917 ORE, BFI, PSI, Hints, Requirements);
9918
9919 assert(L->isInnermost() && "Inner loop expected.");
9920
9921 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());
9922 bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
9923
9924 // If an override option has been passed in for interleaved accesses, use it.
9925 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
9926 UseInterleaved = EnableInterleavedMemAccesses;
9927
9928 // Analyze interleaved memory accesses.
9929 if (UseInterleaved)
9931
9932 if (LVL.hasUncountableEarlyExit()) {
9933 BasicBlock *LoopLatch = L->getLoopLatch();
9934 if (IAI.requiresScalarEpilogue() ||
9936 [LoopLatch](BasicBlock *BB) { return BB != LoopLatch; })) {
9937 reportVectorizationFailure("Auto-vectorization of early exit loops "
9938 "requiring a scalar epilogue is unsupported",
9939 "UncountableEarlyExitUnsupported", ORE, L);
9940 return false;
9941 }
9942 }
9943
9944 // Check the function attributes and profiles to find out if this function
9945 // should be optimized for size.
9947 getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, LVL, &IAI);
9948
9949 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
9950 // count by optimizing for size, to minimize overheads.
9951 auto ExpectedTC = getSmallBestKnownTC(PSE, L);
9952 if (ExpectedTC && ExpectedTC->isFixed() &&
9953 ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
9954 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
9955 << "This loop is worth vectorizing only if no scalar "
9956 << "iteration overheads are incurred.");
9958 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
9959 else {
9960 LLVM_DEBUG(dbgs() << "\n");
9961 // Predicate tail-folded loops are efficient even when the loop
9962 // iteration count is low. However, setting the epilogue policy to
9963 // `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
9964 // with runtime checks. It's more effective to let
9965 // `isOutsideLoopWorkProfitable` determine if vectorization is
9966 // beneficial for the loop.
9969 }
9970 }
9971
9972 // Check the function attributes to see if implicit floats or vectors are
9973 // allowed.
9974 if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
9976 "Can't vectorize when the NoImplicitFloat attribute is used",
9977 "loop not vectorized due to NoImplicitFloat attribute",
9978 "NoImplicitFloat", ORE, L);
9979 Hints.emitRemarkWithHints();
9980 return false;
9981 }
9982
9983 // Check if the target supports potentially unsafe FP vectorization.
9984 // FIXME: Add a check for the type of safety issue (denormal, signaling)
9985 // for the target we're vectorizing for, to make sure none of the
9986 // additional fp-math flags can help.
9987 if (Hints.isPotentiallyUnsafe() &&
9988 TTI->isFPVectorizationPotentiallyUnsafe()) {
9990 "Potentially unsafe FP op prevents vectorization",
9991 "loop not vectorized due to unsafe FP support.",
9992 "UnsafeFP", ORE, L);
9993 Hints.emitRemarkWithHints();
9994 return false;
9995 }
9996
9997 bool AllowOrderedReductions;
9998 // If the flag is set, use that instead and override the TTI behaviour.
9999 if (ForceOrderedReductions.getNumOccurrences() > 0)
10000 AllowOrderedReductions = ForceOrderedReductions;
10001 else
10002 AllowOrderedReductions = TTI->enableOrderedReductions();
10003 if (!LVL.canVectorizeFPMath(AllowOrderedReductions)) {
10004 ORE->emit([&]() {
10005 auto *ExactFPMathInst = Requirements.getExactFPInst();
10006 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
10007 ExactFPMathInst->getDebugLoc(),
10008 ExactFPMathInst->getParent())
10009 << "loop not vectorized: cannot prove it is safe to reorder "
10010 "floating-point operations";
10011 });
10012 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
10013 "reorder floating-point operations\n");
10014 Hints.emitRemarkWithHints();
10015 return false;
10016 }
10017
10018 // Use the cost model.
10019 LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE,
10020 F, &Hints, IAI, PSI, BFI);
10021 // Use the planner for vectorization.
10022 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints,
10023 ORE);
10024
10025 // Get user vectorization factor and interleave count.
10026 ElementCount UserVF = Hints.getWidth();
10027 unsigned UserIC = Hints.getInterleave();
10028
10029 // Plan how to best vectorize.
10030 LVP.plan(UserVF, UserIC);
10032 unsigned IC = 1;
10033
10034 if (ORE->allowExtraAnalysis(LV_NAME))
10036
10037 GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
10038 if (LVP.hasPlanWithVF(VF.Width)) {
10039 // Select the interleave count.
10040 IC = LVP.selectInterleaveCount(LVP.getPlanFor(VF.Width), VF.Width, VF.Cost);
10041
10042 unsigned SelectedIC = std::max(IC, UserIC);
10043 // Optimistically generate runtime checks if they are needed. Drop them if
10044 // they turn out to not be profitable.
10045 if (VF.Width.isVector() || SelectedIC > 1) {
10046 Checks.create(L, *LVL.getLAI(), PSE.getPredicate(), VF.Width, SelectedIC);
10047
10048 // Bail out early if either the SCEV or memory runtime checks are known to
10049 // fail. In that case, the vector loop would never execute.
10050 using namespace llvm::PatternMatch;
10051 if (Checks.getSCEVChecks().first &&
10052 match(Checks.getSCEVChecks().first, m_One()))
10053 return false;
10054 if (Checks.getMemRuntimeChecks().first &&
10055 match(Checks.getMemRuntimeChecks().first, m_One()))
10056 return false;
10057 }
10058
10059 // Check if it is profitable to vectorize with runtime checks.
10060 bool ForceVectorization =
10062 VPCostContext CostCtx(CM.TTI, *CM.TLI, LVP.getPlanFor(VF.Width), CM,
10063 CM.CostKind, *CM.PSE.getSE());
10064 if (!ForceVectorization &&
10065 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
10066 LVP.getPlanFor(VF.Width), SEL,
10067 CM.getVScaleForTuning())) {
10068 ORE->emit([&]() {
10070 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
10071 L->getHeader())
10072 << "loop not vectorized: cannot prove it is safe to reorder "
10073 "memory operations";
10074 });
10075 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
10076 Hints.emitRemarkWithHints();
10077 return false;
10078 }
10079 }
10080
10081 // Identify the diagnostic messages that should be produced.
10082 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
10083 bool VectorizeLoop = true, InterleaveLoop = true;
10084 if (VF.Width.isScalar()) {
10085 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
10086 VecDiagMsg = {
10087 "VectorizationNotBeneficial",
10088 "the cost-model indicates that vectorization is not beneficial"};
10089 VectorizeLoop = false;
10090 }
10091
10092 if (!LVP.hasPlanWithVF(VF.Width) && UserIC > 1) {
10093 // Tell the user interleaving was avoided up-front, despite being explicitly
10094 // requested.
10095 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
10096 "interleaving should be avoided up front\n");
10097 IntDiagMsg = {"InterleavingAvoided",
10098 "Ignoring UserIC, because interleaving was avoided up front"};
10099 InterleaveLoop = false;
10100 } else if (IC == 1 && UserIC <= 1) {
10101 // Tell the user interleaving is not beneficial.
10102 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
10103 IntDiagMsg = {
10104 "InterleavingNotBeneficial",
10105 "the cost-model indicates that interleaving is not beneficial"};
10106 InterleaveLoop = false;
10107 if (UserIC == 1) {
10108 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
10109 IntDiagMsg.second +=
10110 " and is explicitly disabled or interleave count is set to 1";
10111 }
10112 } else if (IC > 1 && UserIC == 1) {
10113 // Tell the user interleaving is beneficial, but it explicitly disabled.
10114 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
10115 "disabled.\n");
10116 IntDiagMsg = {"InterleavingBeneficialButDisabled",
10117 "the cost-model indicates that interleaving is beneficial "
10118 "but is explicitly disabled or interleave count is set to 1"};
10119 InterleaveLoop = false;
10120 }
10121
10122 // If there is a histogram in the loop, do not just interleave without
10123 // vectorizing. The order of operations will be incorrect without the
10124 // histogram intrinsics, which are only used for recipes with VF > 1.
10125 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
10126 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
10127 << "to histogram operations.\n");
10128 IntDiagMsg = {
10129 "HistogramPreventsScalarInterleaving",
10130 "Unable to interleave without vectorization due to constraints on "
10131 "the order of histogram operations"};
10132 InterleaveLoop = false;
10133 }
10134
10135 // Override IC if user provided an interleave count.
10136 IC = UserIC > 0 ? UserIC : IC;
10137
10138 // Emit diagnostic messages, if any.
10139 const char *VAPassName = Hints.vectorizeAnalysisPassName();
10140 if (!VectorizeLoop && !InterleaveLoop) {
10141 // Do not vectorize or interleaving the loop.
10142 ORE->emit([&]() {
10143 return OptimizationRemarkMissed(VAPassName, VecDiagMsg.first,
10144 L->getStartLoc(), L->getHeader())
10145 << VecDiagMsg.second;
10146 });
10147 ORE->emit([&]() {
10148 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
10149 L->getStartLoc(), L->getHeader())
10150 << IntDiagMsg.second;
10151 });
10152 return false;
10153 }
10154
10155 if (!VectorizeLoop && InterleaveLoop) {
10156 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
10157 ORE->emit([&]() {
10158 return OptimizationRemarkAnalysis(VAPassName, VecDiagMsg.first,
10159 L->getStartLoc(), L->getHeader())
10160 << VecDiagMsg.second;
10161 });
10162 } else if (VectorizeLoop && !InterleaveLoop) {
10163 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
10164 << ") in " << L->getLocStr() << '\n');
10165 ORE->emit([&]() {
10166 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
10167 L->getStartLoc(), L->getHeader())
10168 << IntDiagMsg.second;
10169 });
10170 } else if (VectorizeLoop && InterleaveLoop) {
10171 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
10172 << ") in " << L->getLocStr() << '\n');
10173 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
10174 }
10175
10176 // Report the vectorization decision.
10177 if (VF.Width.isScalar()) {
10178 using namespace ore;
10179 assert(IC > 1);
10180 ORE->emit([&]() {
10181 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
10182 L->getHeader())
10183 << "interleaved loop (interleaved count: "
10184 << NV("InterleaveCount", IC) << ")";
10185 });
10186 } else {
10187 // Report the vectorization decision.
10188 reportVectorization(ORE, L, VF, IC);
10189 }
10190 if (ORE->allowExtraAnalysis(LV_NAME))
10192
10193 // If we decided that it is *legal* to interleave or vectorize the loop, then
10194 // do it.
10195
10196 VPlan &BestPlan = LVP.getPlanFor(VF.Width);
10197 // Consider vectorizing the epilogue too if it's profitable.
10198 VectorizationFactor EpilogueVF =
10200 if (EpilogueVF.Width.isVector()) {
10201 std::unique_ptr<VPlan> BestMainPlan(BestPlan.duplicate());
10202
10203 // The first pass vectorizes the main loop and creates a scalar epilogue
10204 // to be vectorized by executing the plan (potentially with a different
10205 // factor) again shortly afterwards.
10206 VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
10207 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
10208 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
10209 preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
10210 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
10211 BestEpiPlan);
10212 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM, BFI,
10213 PSI, Checks, *BestMainPlan);
10214 auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
10215 *BestMainPlan, MainILV, DT, false);
10216 ++LoopsVectorized;
10217
10218 // Second pass vectorizes the epilogue and adjusts the control flow
10219 // edges from the first pass.
10220 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
10221 BFI, PSI, Checks, BestEpiPlan);
10223 BestEpiPlan, L, ExpandedSCEVs, EPI, CM, *PSE.getSE());
10224 LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
10225 true);
10226 connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LVL, ExpandedSCEVs,
10227 Checks, InstsToMove);
10228 ++LoopsEpilogueVectorized;
10229 } else {
10230 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, &CM, BFI, PSI,
10231 Checks, BestPlan);
10232 // TODO: Move to general VPlan pipeline once epilogue loops are also
10233 // supported.
10236 IC, PSE);
10237 LVP.addMinimumIterationCheck(BestPlan, VF.Width, IC,
10239
10240 LVP.executePlan(VF.Width, IC, BestPlan, LB, DT, false);
10241 ++LoopsVectorized;
10242 }
10243
10244 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
10245 "DT not preserved correctly");
10246 assert(!verifyFunction(*F, &dbgs()));
10247
10248 return true;
10249}
10250
10252
10253 // Don't attempt if
10254 // 1. the target claims to have no vector registers, and
10255 // 2. interleaving won't help ILP.
10256 //
10257 // The second condition is necessary because, even if the target has no
10258 // vector registers, loop vectorization may still enable scalar
10259 // interleaving.
10260 if (!TTI->getNumberOfRegisters(TTI->getRegisterClassForType(true)) &&
10261 TTI->getMaxInterleaveFactor(ElementCount::getFixed(1)) < 2)
10262 return LoopVectorizeResult(false, false);
10263
10264 bool Changed = false, CFGChanged = false;
10265
10266 // The vectorizer requires loops to be in simplified form.
10267 // Since simplification may add new inner loops, it has to run before the
10268 // legality and profitability checks. This means running the loop vectorizer
10269 // will simplify all loops, regardless of whether anything end up being
10270 // vectorized.
10271 for (const auto &L : *LI)
10272 Changed |= CFGChanged |=
10273 simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
10274
10275 // Build up a worklist of inner-loops to vectorize. This is necessary as
10276 // the act of vectorizing or partially unrolling a loop creates new loops
10277 // and can invalidate iterators across the loops.
10278 SmallVector<Loop *, 8> Worklist;
10279
10280 for (Loop *L : *LI)
10281 collectSupportedLoops(*L, LI, ORE, Worklist);
10282
10283 LoopsAnalyzed += Worklist.size();
10284
10285 // Now walk the identified inner loops.
10286 while (!Worklist.empty()) {
10287 Loop *L = Worklist.pop_back_val();
10288
10289 // For the inner loops we actually process, form LCSSA to simplify the
10290 // transform.
10291 Changed |= formLCSSARecursively(*L, *DT, LI, SE);
10292
10293 Changed |= CFGChanged |= processLoop(L);
10294
10295 if (Changed) {
10296 LAIs->clear();
10297
10298#ifndef NDEBUG
10299 if (VerifySCEV)
10300 SE->verify();
10301#endif
10302 }
10303 }
10304
10305 // Process each loop nest in the function.
10306 return LoopVectorizeResult(Changed, CFGChanged);
10307}
10308
10311 LI = &AM.getResult<LoopAnalysis>(F);
10312 // There are no loops in the function. Return before computing other
10313 // expensive analyses.
10314 if (LI->empty())
10315 return PreservedAnalyses::all();
10324 AA = &AM.getResult<AAManager>(F);
10325
10326 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
10327 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
10328 BFI = nullptr;
10329 if (PSI && PSI->hasProfileSummary())
10331 LoopVectorizeResult Result = runImpl(F);
10332 if (!Result.MadeAnyChange)
10333 return PreservedAnalyses::all();
10335
10336 if (isAssignmentTrackingEnabled(*F.getParent())) {
10337 for (auto &BB : F)
10339 }
10340
10341 PA.preserve<LoopAnalysis>();
10345
10346 if (Result.MadeCFGChange) {
10347 // Making CFG changes likely means a loop got vectorized. Indicate that
10348 // extra simplification passes should be run.
10349 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
10350 // be run if runtime checks have been added.
10353 } else {
10355 }
10356 return PA;
10357}
10358
10360 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
10361 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
10362 OS, MapClassName2PassName);
10363
10364 OS << '<';
10365 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
10366 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
10367 OS << '>';
10368}
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
@ PostInc
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isEqual(const Function &Caller, const Function &Callee)
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
static bool IsEmptyBlock(MachineBasicBlock *MBB)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< OutputCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(OutputCostKind::RecipThroughput), cl::values(clEnumValN(OutputCostKind::RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(OutputCostKind::Latency, "latency", "Instruction latency"), clEnumValN(OutputCostKind::CodeSize, "code-size", "Code size"), clEnumValN(OutputCostKind::SizeAndLatency, "size-latency", "Code size and latency"), clEnumValN(OutputCostKind::All, "all", "Print all cost kinds")))
static cl::opt< IntrinsicCostStrategy > IntrinsicCost("intrinsic-cost-strategy", cl::desc("Costing strategy for intrinsic instructions"), cl::init(IntrinsicCostStrategy::InstructionCost), cl::values(clEnumValN(IntrinsicCostStrategy::InstructionCost, "instruction-cost", "Use TargetTransformInfo::getInstructionCost"), clEnumValN(IntrinsicCostStrategy::IntrinsicCost, "intrinsic-cost", "Use TargetTransformInfo::getIntrinsicInstrCost"), clEnumValN(IntrinsicCostStrategy::TypeBasedIntrinsicCost, "type-based-intrinsic-cost", "Calculate the intrinsic cost based only on argument types")))
static InstructionCost getCost(Instruction &Inst, TTI::TargetCostKind CostKind, TargetTransformInfo &TTI, TargetLibraryInfo &TLI)
Definition CostModel.cpp:74
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
#define _
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
This defines the Use class.
static bool hasNoUnsignedWrap(BinaryOperator &I)
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Legalize the Machine IR a function s Machine IR
Definition Legalizer.cpp:80
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
This header provides classes for managing per-loop analyses.
static const char * VerboseDebug
#define LV_NAME
This file defines the LoopVectorizationLegality class.
This file provides a LoopVectorizationPlanner class.
static void collectSupportedLoops(Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop * > &V)
static cl::opt< unsigned > EpilogueVectorizationMinVF("epilogue-vectorization-minimum-VF", cl::Hidden, cl::desc("Only loops with vectorization factor equal to or larger than " "the specified value are considered for epilogue vectorization."))
static cl::opt< unsigned > EpilogueVectorizationForceVF("epilogue-vectorization-force-VF", cl::init(1), cl::Hidden, cl::desc("When epilogue vectorization is enabled, and a value greater than " "1 is specified, forces the given VF for all applicable epilogue " "loops."))
static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)
Create resume phis in the scalar preheader for first-order recurrences, reductions and inductions,...
static Type * maybeVectorizeType(Type *Ty, ElementCount VF)
static ElementCount determineVPlanVF(const TargetTransformInfo &TTI, LoopVectorizationCostModel &CM)
static ElementCount getSmallConstantTripCount(ScalarEvolution *SE, const Loop *L)
A version of ScalarEvolution::getSmallConstantTripCount that returns an ElementCount to include loops...
static cl::opt< unsigned > VectorizeMemoryCheckThreshold("vectorize-memory-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum allowed number of runtime memory checks"))
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan)
Prepare MainPlan for vectorizing the main vector loop during epilogue vectorization.
static cl::opt< unsigned > TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
Loops with a known constant trip count below this number are vectorized only if no scalar iteration o...
static void debugVectorizationMessage(const StringRef Prefix, const StringRef DebugMsg, Instruction *I)
Write a DebugMsg about vectorization to the debug output stream.
static cl::opt< bool > EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static void legacyCSE(BasicBlock *BB)
FIXME: This legacy common-subexpression-elimination routine is scheduled for removal,...
static VPIRBasicBlock * replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB, VPlan *Plan=nullptr)
Replace VPBB with a VPIRBasicBlock wrapping IRBB.
static VPInstruction * addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC)
Create and return a ResumePhi for WideIV, unless it is truncated.
static Value * emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue, Value *Step, InductionDescriptor::InductionKind InductionKind, const BinaryOperator *InductionBinOp)
Compute the transformed value of Index at offset StartValue using step StepValue.
static DebugLoc getDebugLocFromInstOrOperands(Instruction *I)
Look for a meaningful debug location on the instruction or its operands.
static Value * createInductionAdditionalBypassValues(PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount, Instruction *OldInduction)
static void fixReductionScalarResumeWhenVectorizingEpilog(VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock)
static Value * getStartValueFromReductionResult(VPInstruction *RdxResult)
static cl::opt< bool > ForceTargetSupportsScalableVectors("force-target-supports-scalable-vectors", cl::init(false), cl::Hidden, cl::desc("Pretend that scalable vectors are supported, even if the target does " "not support them. This flag should only be used for testing."))
static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style)
static cl::opt< bool > EnableEarlyExitVectorization("enable-early-exit-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of early exit loops with uncountable exits."))
static cl::opt< bool > ConsiderRegPressure("vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden, cl::desc("Discard VFs if their register pressure is too high."))
static unsigned estimateElementCount(ElementCount VF, std::optional< unsigned > VScale)
This function attempts to return a value that represents the ElementCount at runtime.
static constexpr uint32_t MinItersBypassWeights[]
static cl::opt< unsigned > ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< bool > UseWiderVFIfCallVariantsPresent("vectorizer-maximize-bandwidth-for-vector-calls", cl::init(true), cl::Hidden, cl::desc("Try wider VFs if they enable the use of vector variants"))
static std::optional< unsigned > getMaxVScale(const Function &F, const TargetTransformInfo &TTI)
static cl::opt< unsigned > SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L, EpilogueLoopVectorizationInfo &EPI, DominatorTree *DT, LoopVectorizationLegality &LVL, DenseMap< const SCEV *, Value * > &ExpandedSCEVs, GeneratedRTChecks &Checks, ArrayRef< Instruction * > InstsToMove)
Connect the epilogue vector loop generated for EpiPlan to the main vector.
static bool planContainsAdditionalSimplifications(VPlan &Plan, VPCostContext &CostCtx, Loop *TheLoop, ElementCount VF)
Return true if the original loop \ TheLoop contains any instructions that do not have corresponding r...
static cl::opt< unsigned > ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static bool isExplicitVecOuterLoop(Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static cl::opt< bool > EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< TailFoldingStyle > ForceTailFoldingStyle("force-tail-folding-style", cl::desc("Force the tail folding style"), cl::init(TailFoldingStyle::None), cl::values(clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"), clEnumValN(TailFoldingStyle::Data, "data", "Create lane mask for data only, using active.lane.mask intrinsic"), clEnumValN(TailFoldingStyle::DataWithoutLaneMask, "data-without-lane-mask", "Create lane mask with compare/stepvector"), clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control", "Create lane mask using active.lane.mask intrinsic, and use " "it for both data and control flow"), clEnumValN(TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck, "data-and-control-without-rt-check", "Similar to data-and-control, but remove the runtime check"), clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl", "Use predicated EVL instructions for tail folding. If EVL " "is unsupported, fallback to data-without-lane-mask.")))
static cl::opt< bool > EnableEpilogueVectorization("enable-epilogue-vectorization", cl::init(true), cl::Hidden, cl::desc("Enable vectorization of epilogue loops."))
static ScalarEpilogueLowering getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL, InterleavedAccessInfo *IAI)
static cl::opt< bool > PreferPredicatedReductionSelect("prefer-predicated-reduction-select", cl::init(false), cl::Hidden, cl::desc("Prefer predicating a reduction operation over an after loop select."))
static VPWidenIntOrFpInductionRecipe * createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc, VPValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, ScalarEvolution &SE, Loop &OrigLoop)
Creates a VPWidenIntOrFpInductionRecpipe for Phi.
static cl::opt< bool > PreferInLoopReductions("prefer-inloop-reductions", cl::init(false), cl::Hidden, cl::desc("Prefer in-loop vector reductions, " "overriding the targets preference."))
static SmallVector< Instruction * > preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel &CM, ScalarEvolution &SE)
Prepare Plan for vectorizing the epilogue loop.
static cl::opt< bool > EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< bool > VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static bool hasIrregularType(Type *Ty, const DataLayout &DL)
A helper function that returns true if the given type is irregular.
static cl::opt< bool > LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static std::optional< ElementCount > getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax=true)
Returns "best known" trip count, which is either a valid positive trip count or std::nullopt when an ...
static Value * getExpandedStep(const InductionDescriptor &ID, const SCEV2ValueTy &ExpandedSCEVs)
Return the expanded step for ID using ExpandedSCEVs to look up SCEV expansion results.
static bool useActiveLaneMask(TailFoldingStyle Style)
static bool hasReplicatorRegion(VPlan &Plan)
static bool isIndvarOverflowCheckKnownFalse(const LoopVectorizationCostModel *Cost, ElementCount VF, std::optional< unsigned > UF=std::nullopt)
For the given VF and UF and maximum trip count computed for the loop, return whether the induction va...
static void addFullyUnrolledInstructionsToIgnore(Loop *L, const LoopVectorizationLegality::InductionList &IL, SmallPtrSetImpl< Instruction * > &InstsToIgnore)
Knowing that loop L executes a single vector iteration, add instructions that will get simplified and...
static cl::opt< PreferPredicateTy::Option > PreferPredicateOverEpilogue("prefer-predicate-over-epilogue", cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", "Don't tail-predicate loops, create scalar epilogue"), clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, "predicate-else-scalar-epilogue", "prefer tail-folding, create scalar epilogue if tail " "folding fails."), clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, "predicate-dont-vectorize", "prefers tail-folding, don't attempt vectorization if " "tail-folding fails.")))
static cl::opt< bool > EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool > EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
An interleave-group may need masking if it resides in a block that needs predication,...
static cl::opt< bool > ForceOrderedReductions("force-ordered-reductions", cl::init(false), cl::Hidden, cl::desc("Enable the vectorisation of loops with in-order (strict) " "FP reductions"))
static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...
static cl::opt< cl::boolOrDefault > ForceSafeDivisor("force-widen-divrem-via-safe-divisor", cl::Hidden, cl::desc("Override cost based safe divisor widening for div/rem instructions"))
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx, VPlan &Plan, ElementCount VF)
For loops with uncountable early exits, find the cost of doing work when exiting the loop early,...
static cl::opt< unsigned > ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static bool processLoopInVPlanNativePath(Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements)
static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI)
static cl::opt< unsigned > NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
The number of stores in a loop that are allowed to need predication.
static cl::opt< unsigned > MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
static cl::opt< unsigned > ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE)
static bool willGenerateVectors(VPlan &Plan, ElementCount VF, const TargetTransformInfo &TTI)
Check if any recipe of Plan will generate a vector value, which will be assigned a vector register.
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks, VectorizationFactor &VF, Loop *L, PredicatedScalarEvolution &PSE, VPCostContext &CostCtx, VPlan &Plan, ScalarEpilogueLowering SEL, std::optional< unsigned > VScale)
This function determines whether or not it's still profitable to vectorize the loop given the extra w...
static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range)
Handle users in the exit block for first order reductions in the original exit block.
static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L, VPlan &BestEpiPlan, LoopVectorizationLegality &LVL, const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount)
static cl::opt< bool > MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName, StringRef RemarkName, Loop *TheLoop, Instruction *I, DebugLoc DL={})
Create an analysis remark that explains why vectorization failed.
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
This file contains the declarations for profiling metadata utility functions.
const SmallVectorImpl< MachineOperand > & Cond
static BinaryOperator * CreateMul(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static BinaryOperator * CreateAdd(Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore, Value *FlagsOp)
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
static InstructionCost getScalarizationOverhead(const TargetTransformInfo &TTI, Type *ScalarTy, VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={})
This is similar to TargetTransformInfo::getScalarizationOverhead, but if ScalarTy is a FixedVectorTyp...
This file contains some templates that are useful if you are working with the STL at all.
#define OP(OPC)
Definition Instruction.h:46
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:114
#define DEBUG_WITH_TYPE(TYPE,...)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition Debug.h:72
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
This pass exposes codegen information to IR-level passes.
LocallyHashedType DenseMapInfo< LocallyHashedType >::Empty
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file contains the declarations of different VPlan-related auxiliary helpers.
This file provides utility VPlan to VPlan transformations.
This file declares the class VPlanVerifier, which contains utility functions to check the consistency...
This file contains the declarations of the Vectorization Plan base classes:
static const char PassName[]
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
A manager for alias analyses.
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:234
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1540
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition APInt.h:1512
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition ArrayRef.h:147
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
LLVM_ABI unsigned getVScaleRangeMin() const
Returns the minimum value for the vscale_range attribute.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition BasicBlock.h:528
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition BasicBlock.h:233
BinaryOps getOpcode() const
Definition InstrTypes.h:374
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Conditional or Unconditional Branch instruction.
bool isConditional() const
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i) const
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition InstrTypes.h:984
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:678
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:701
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:703
@ ICMP_NE
not equal
Definition InstrTypes.h:700
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:704
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:791
An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
A debug info location.
Definition DebugLoc.h:124
static DebugLoc getTemporary()
Definition DebugLoc.h:161
static DebugLoc getUnknown()
Definition DebugLoc.h:162
An analysis that produces DemandedBits for a function.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition DenseMap.h:194
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:167
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Definition DenseMap.h:237
iterator end()
Definition DenseMap.h:81
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:158
void insert_range(Range &&R)
Inserts range of 'std::pair<KeyT, ValueT>' values into the map.
Definition DenseMap.h:275
Implements a dense probed hash-table based set.
Definition DenseSet.h:279
Analysis pass which computes a DominatorTree.
Definition Dominators.h:284
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:165
constexpr bool isVector() const
One or more elements.
Definition TypeSize.h:324
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:312
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition TypeSize.h:309
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition TypeSize.h:315
constexpr bool isScalar() const
Exactly one element.
Definition TypeSize.h:320
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the epilogue loop strategy (i....
EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
A specialized derived class of inner loop vectorizer that performs vectorization of main loops in the...
void introduceCheckBlockInVPlan(BasicBlock *CheckIRBB)
Introduces a new VPIRBasicBlock for CheckIRBB to Plan between the vector preheader and its predecesso...
BasicBlock * emitIterationCountCheck(BasicBlock *VectorPH, BasicBlock *Bypass, bool ForEpilogue)
Emits an iteration count bypass check once for the main loop (when ForEpilogue is false) and once for...
EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
Value * createIterationCountCheck(BasicBlock *VectorPH, ElementCount VF, unsigned UF) const
void printDebugTracesAtStart() override
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
BasicBlock * createVectorizedLoopSkeleton() final
Implements the interface for creating a vectorized skeleton using the main loop strategy (i....
Convenience struct for specifying and reasoning about fast-math flags.
Definition FMF.h:22
Class to represent function types.
param_iterator param_begin() const
param_iterator param_end() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:762
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:727
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
void applyUpdates(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:114
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition IRBuilder.h:345
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2780
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_NoInduction
Not an induction variable.
@ IK_FpInduction
Floating point induction variable.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const SmallVectorImpl< Instruction * > & getCastInsts() const
Returns a reference to the type cast instructions in the induction update chain, that are redundant w...
Value * getStartValue() const
InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan, ElementCount VecWidth, ElementCount MinProfitableTripCount, unsigned UnrollFactor)
EpilogueLoopVectorizationInfo & EPI
Holds and updates state information required to vectorize the main loop and its epilogue in two separ...
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
virtual void printDebugTracesAtStart()
Allow subclasses to override and print debug traces before/after vplan execution, when trace informat...
Value * TripCount
Trip count of the original loop.
const TargetTransformInfo * TTI
Target Transform Info.
LoopVectorizationCostModel * Cost
The profitablity analysis.
BlockFrequencyInfo * BFI
BFI and PSI are used to check for profile guided size optimizations.
Value * getTripCount() const
Returns the original loop trip count.
friend class LoopVectorizationPlanner
PredicatedScalarEvolution & PSE
A wrapper around ScalarEvolution used to add runtime SCEV checks.
LoopInfo * LI
Loop Info.
ProfileSummaryInfo * PSI
DominatorTree * DT
Dominator Tree.
void setTripCount(Value *TC)
Used to set the trip count after ILV's construction and after the preheader block has been executed.
void fixVectorizedLoop(VPTransformState &State)
Fix the vectorized code, taking care of header phi's, and more.
virtual BasicBlock * createVectorizedLoopSkeleton()
Creates a basic block for the scalar preheader.
virtual void printDebugTracesAtEnd()
AssumptionCache * AC
Assumption Cache.
InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC, ElementCount VecWidth, unsigned UnrollFactor, LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks, VPlan &Plan)
IRBuilder Builder
The builder that we use.
void fixNonInductionPHIs(VPTransformState &State)
Fix the non-induction PHIs in Plan.
VPBasicBlock * VectorPHVPBB
The vector preheader block of Plan, used as target for check blocks introduced during skeleton creati...
unsigned UF
The vectorization unroll factor to use.
GeneratedRTChecks & RTChecks
Structure to hold information about generated runtime checks, responsible for cleaning the checks,...
virtual ~InnerLoopVectorizer()=default
ElementCount VF
The vectorization SIMD factor to use.
Loop * OrigLoop
The original loop.
BasicBlock * createScalarPreheader(StringRef Prefix)
Create and return a new IR basic block for the scalar preheader whose name is prefixed with Prefix.
InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.
static InstructionCost getInvalid(CostType Val=0)
static InstructionCost getMax()
CostType getValue() const
This function is intended to be used as sparingly as possible, since the class provides the full rang...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void moveBefore(InstListType::iterator InsertPos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool isBinaryOp() const
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:319
LLVM_ABI APInt getMask() const
For example, this is 0xFF for an 8 bit integer, 0xFFFF for i16, etc.
Definition Type.cpp:343
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
uint32_t getNumMembers() const
Drive the analysis of interleaved memory accesses in the loop.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
LLVM_ABI void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
An instruction for reading from memory.
Type * getPointerOperandType() const
This analysis provides dependence information for the memory accesses of a loop.
Drive the analysis of memory accesses in the loop.
const RuntimePointerChecking * getRuntimePointerChecking() const
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:569
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getHeader() const
iterator_range< block_iterator > blocks() const
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
ArrayRef< BlockT * > getBlocks() const
Get a list of the basic blocks which make up this loop.
Store the result of a depth first search within basic blocks contained by a single loop.
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
RPOIterator endRPO() const
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
void perform(const LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
SmallPtrSet< Type *, 16 > ElementTypesInLoop
All element types found in the loop.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked load operation for the given DataType and kind of ...
LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, LoopVectorizationLegality *Legal, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, InterleavedAccessInfo &IAI, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)
void collectElementTypesForWidening()
Collect all element types in the loop for which widening is needed.
bool canVectorizeReductions(ElementCount VF) const
Returns true if the target machine supports all of the reduction variables found for the given VF.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment, unsigned AddressSpace) const
Returns true if the target machine supports masked store operation for the given DataType and kind of...
bool isEpilogueVectorizationProfitable(const ElementCount VF, const unsigned IC) const
Returns true if epilogue vectorization is considered profitable, and false otherwise.
bool isPredicatedInst(Instruction *I) const
Returns true if I is an instruction that needs to be predicated at runtime.
void collectValuesToIgnore()
Collect values we want to ignore in the cost model.
void collectInLoopReductions()
Split reductions into those that happen in the loop, and those that happen outside.
std::pair< unsigned, unsigned > getSmallestAndWidestTypes()
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be uniform after vectorization.
void collectNonVectorizedAndSetWideningDecisions(ElementCount VF)
Collect values that will not be widened, including Uniforms, Scalars, and Instructions to Scalarize f...
PredicatedScalarEvolution & PSE
Predicated scalar evolution analysis.
const LoopVectorizeHints * Hints
Loop Vectorize Hint.
std::optional< unsigned > getMaxSafeElements() const
Return maximum safe number of elements to be processed per vector iteration, which do not prevent sto...
const TargetTransformInfo & TTI
Vector target information.
LoopVectorizationLegality * Legal
Vectorization legality.
std::optional< InstructionCost > getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy) const
Return the cost of instructions in an inloop reduction pattern, if I is part of that pattern.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF)
Returns the execution time cost of an instruction for a given vector width.
DemandedBits * DB
Demanded bits analysis.
bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const
Returns true if I is a memory instruction in an interleaved-group of memory accesses that can be vect...
const TargetLibraryInfo * TLI
Target Library Info.
bool memoryInstructionCanBeWidened(Instruction *I, ElementCount VF)
Returns true if I is a memory instruction with consecutive memory access that can be widened.
const InterleaveGroup< Instruction > * getInterleavedAccessGroup(Instruction *Instr) const
Get the interleaved access group that Instr belongs to.
InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const
Estimate cost of an intrinsic call instruction CI if it were vectorized with factor VF.
bool OptForSize
Whether this loop should be optimized for size based on function attribute or profile information.
bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind)
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const
Returns true if I is known to be scalar after vectorization.
bool isOptimizableIVTruncate(Instruction *I, ElementCount VF)
Return True if instruction I is an optimizable truncate whose operand is an induction variable.
FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC)
bool shouldConsiderRegPressureForVF(ElementCount VF)
Loop * TheLoop
The loop that we evaluate.
TTI::TargetCostKind CostKind
The kind of cost that we are calculating.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow=true) const
Returns the TailFoldingStyle that is best for the current loop.
InterleavedAccessInfo & InterleaveInfo
The interleave access information contains groups of interleaved accesses with the same stride and cl...
SmallPtrSet< const Value *, 16 > ValuesToIgnore
Values to ignore in the cost model.
void setVectorizedCallDecision(ElementCount VF)
A call may be vectorized in different ways depending on whether we have vectorized variants available...
void invalidateCostModelingDecisions()
Invalidates decisions already taken by the cost model.
bool isAccessInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleaved access group.
bool selectUserVectorizationFactor(ElementCount UserVF)
Setup cost-based decisions for user vectorization factor.
std::optional< unsigned > getVScaleForTuning() const
Return the value of vscale used for tuning the cost model.
OptimizationRemarkEmitter * ORE
Interface to emit optimization remarks.
LoopInfo * LI
Loop Info analysis.
bool requiresScalarEpilogue(bool IsVectorizing) const
Returns true if we're required to use a scalar epilogue for at least the final iteration of the origi...
SmallPtrSet< const Value *, 16 > VecValuesToIgnore
Values to ignore in the cost model when VF > 1.
bool isInLoopReduction(PHINode *Phi) const
Returns true if the Phi is part of an inloop reduction.
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const
void setWideningDecision(const InterleaveGroup< Instruction > *Grp, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for interleaving group Grp and vector ...
const MapVector< Instruction *, uint64_t > & getMinimalBitwidths() const
CallWideningDecision getCallWideningDecision(CallInst *CI, ElementCount VF) const
bool isLegalGatherOrScatter(Value *V, ElementCount VF)
Returns true if the target machine can represent V as a masked gather or scatter operation.
bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const
bool shouldConsiderInvariant(Value *Op)
Returns true if Op should be considered invariant and if it is trivially hoistable.
bool foldTailByMasking() const
Returns true if all loop blocks should be masked to fold tail loop.
bool foldTailWithEVL() const
Returns true if VP intrinsics with explicit vector length support should be generated in the tail fol...
bool usePredicatedReductionSelect() const
Returns true if the predicated reduction select should be used to set the incoming value for the redu...
bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const
Returns true if the instructions in this block requires predication for any reason,...
void setCallWideningDecision(CallInst *CI, ElementCount VF, InstWidening Kind, Function *Variant, Intrinsic::ID IID, std::optional< unsigned > MaskPos, InstructionCost Cost)
void setTailFoldingStyles(bool IsScalableVF, unsigned UserIC)
Selects and saves TailFoldingStyle for 2 options - if IV update may overflow or not.
AssumptionCache * AC
Assumption cache.
void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W, InstructionCost Cost)
Save vectorization decision W and Cost taken by the cost model for instruction I and vector width VF.
InstWidening
Decision that was taken during cost calculation for memory instruction.
bool isScalarWithPredication(Instruction *I, ElementCount VF) const
Returns true if I is an instruction which requires predication and for which our chosen predication s...
InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const
Estimate cost of a call instruction CI if it were vectorized with factor VF.
bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const
Returns true if we should use strict in-order reductions for the given RdxDesc.
std::pair< InstructionCost, InstructionCost > getDivRemSpeculationCost(Instruction *I, ElementCount VF) const
Return the costs for our two available strategies for lowering a div/rem operation which requires spe...
bool isDivRemScalarWithPredication(InstructionCost ScalarCost, InstructionCost SafeDivisorCost) const
Given costs for both strategies, return true if the scalar predication lowering should be used for di...
InstructionCost expectedCost(ElementCount VF)
Returns the expected execution cost.
void setCostBasedWideningDecision(ElementCount VF)
Memory access instruction may be vectorized in more than one way.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const
Return the cost model decision for the given instruction I and vector width VF.
FixedScalableVFPair MaxPermissibleVFWithoutMaxBW
The highest VF possible for this loop, without using MaxBandwidth.
bool isScalarEpilogueAllowed() const
Returns true if a scalar epilogue is not allowed due to optsize or a loop hint annotation.
InstructionCost getWideningCost(Instruction *I, ElementCount VF)
Return the vectorization cost for the given instruction I and vector width VF.
void collectInstsToScalarize(ElementCount VF)
Collects the instructions to scalarize for each predicated instruction in the loop.
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
const SmallPtrSetImpl< const Instruction * > & getPotentiallyFaultingLoads() const
Returns potentially faulting loads.
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
PHINode * getPrimaryInduction()
Returns the primary induction variable.
const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const
Returns all exiting blocks with a countable exit, i.e.
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
bool hasUncountableEarlyExit() const
Returns true if the loop has exactly one uncountable early exit, i.e.
bool hasHistograms() const
Returns a list of all known histogram operations in the loop.
const LoopAccessInfo * getLAI() const
Planner drives the vectorization process after having passed Legality checks.
VectorizationFactor selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC)
VPlan & getPlanFor(ElementCount VF) const
Return the VPlan for VF.
Definition VPlan.cpp:1602
VectorizationFactor planInVPlanNativePath(ElementCount UserVF)
Use the VPlan-native path to plan how to best vectorize, return the best VF and its cost.
void updateLoopMetadataAndProfileInfo(Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan, bool VectorizingEpilogue, MDNode *OrigLoopID, std::optional< unsigned > OrigAverageTripCount, unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF, bool DisableRuntimeUnroll)
Update loop metadata and profile info for both the scalar remainder loop and VectorLoop,...
Definition VPlan.cpp:1653
void buildVPlans(ElementCount MinVF, ElementCount MaxVF)
Build VPlans for power-of-2 VF's between MinVF and MaxVF inclusive, according to the information gath...
Definition VPlan.cpp:1586
VectorizationFactor computeBestVF()
Compute and return the most profitable vectorization factor.
DenseMap< const SCEV *, Value * > executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan, InnerLoopVectorizer &LB, DominatorTree *DT, bool VectorizingEpilogue)
Generate the IR code for the vectorized loop captured in VPlan BestPlan according to the best selecte...
unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF, InstructionCost LoopCost)
void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE)
Emit remarks for recipes with invalid costs in the available VPlans.
static bool getDecisionAndClampRange(const std::function< bool(ElementCount)> &Predicate, VFRange &Range)
Test a Predicate on a Range of VF's.
Definition VPlan.cpp:1567
void printPlans(raw_ostream &O)
Definition VPlan.cpp:1731
void plan(ElementCount UserVF, unsigned UserIC)
Build VPlans for the specified UserVF and UserIC if they are non-zero or all applicable candidate VFs...
void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount) const
Create a check to Plan to see if the vector loop should be executed based on its trip count.
bool hasPlanWithVF(ElementCount VF) const
Look through the existing plans and return true if we have one with vectorization factor VF.
This holds vectorization requirements that must be verified late in the process.
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
void emitRemarkWithHints() const
Dumps all the hint information.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
bool hasLoopInvariantOperands(const Instruction *I) const
Return true if all the operands of the specified instruction are loop invariant.
Definition LoopInfo.cpp:67
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
Definition LoopInfo.cpp:632
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition LoopInfo.cpp:61
Metadata node.
Definition Metadata.h:1077
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:36
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition MapVector.h:119
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition Module.cpp:230
Diagnostic information for optimization analysis remarks related to pointer aliasing.
Diagnostic information for optimization analysis remarks related to floating-point non-commutativity.
Diagnostic information for optimization analysis remarks.
The optimization diagnostic interface.
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for missed-optimization remarks.
Diagnostic information for applied optimization remarks.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
void setIncomingValueForBlock(const BasicBlock *BB, Value *V)
Set every incoming value(s) for block BB to V.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
unsigned getNumIncomingValues() const
Return the number of incoming edges.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
LLVM_ABI const SCEVPredicate & getPredicate() const
LLVM_ABI unsigned getSmallConstantMaxTripCount()
Returns the upper bound of the loop trip count as a normal unsigned value, or 0 if the trip count is ...
LLVM_ABI const SCEV * getBackedgeTakenCount()
Get the (predicated) backedge count for the analyzed loop.
LLVM_ABI const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
Analysis providing profile information.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
static bool isFMulAddIntrinsic(Instruction *I)
Returns true if the instruction is a call to the llvm.fmuladd intrinsic.
FastMathFlags getFastMathFlags() const
Instruction * getLoopExitInstr() const
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
const SmallPtrSet< Instruction *, 8 > & getCastInsts() const
Returns a reference to the instructions used for type-promoting the recurrence.
unsigned getMinWidthCastToRecurrenceTypeInBits() const
Returns the minimum width used by the recurrence in bits.
TrackingVH< Value > getRecurrenceStartValue() const
LLVM_ABI SmallVector< Instruction *, 4 > getReductionOpChain(PHINode *Phi, Loop *L) const
Attempts to find a chain of operations from Phi to LoopExitInst that can be treated as a set of reduc...
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
bool isOrdered() const
Expose an ordered FP reduction to the instance users.
static LLVM_ABI bool isFloatingPointRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is a floating point kind.
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Value * getSentinelValue() const
Returns the sentinel value for FindFirstIV & FindLastIV recurrences to replace the start value.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
std::optional< ArrayRef< PointerDiffInfo > > getDiffChecks() const
const SmallVectorImpl< RuntimePointerCheck > & getChecks() const
Returns the checks that generateChecks created.
This class uses information about analyze scalars to rewrite expressions in canonical form.
ScalarEvolution * getSE()
bool isInsertedInstruction(Instruction *I) const
Return true if the specified instruction was inserted by the code rewriter.
LLVM_ABI Value * expandCodeForPredicate(const SCEVPredicate *Pred, Instruction *Loc)
Generates a code sequence that evaluates this predicate.
void eraseDeadInstructions(Value *Root)
Remove inserted instructions that are dead, e.g.
virtual bool isAlwaysTrue() const =0
Returns true if the predicate is always true.
This class represents an analyzed expression in the program.
LLVM_ABI bool isZero() const
Return true if the expression is a constant zero.
LLVM_ABI Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
LLVM_ABI const SCEV * getURemExpr(const SCEV *LHS, const SCEV *RHS)
Represents an unsigned remainder expression based on unsigned division.
LLVM_ABI const SCEV * getBackedgeTakenCount(const Loop *L, ExitCountKind Kind=Exact)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
LLVM_ABI const SCEV * getConstant(ConstantInt *V)
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI const SCEV * getTripCountFromExitCount(const SCEV *ExitCount)
A version of getTripCountFromExitCount below which always picks an evaluation type which can not resu...
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
LLVM_ABI void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
LLVM_ABI bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
LLVM_ABI const SCEV * getElementCount(Type *Ty, ElementCount EC, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
LLVM_ABI void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)
Called when the client has changed the disposition of values in a loop or block.
const SCEV * getMinusOne(Type *Ty)
Return a SCEV for the constant -1 of a specific type.
LLVM_ABI void forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V)
Forget LCSSA phi node V of loop L to which a new predecessor was added, such that it may no longer be...
LLVM_ABI unsigned getSmallConstantTripCount(const Loop *L)
Returns the exact trip count of the loop if we can compute it, and the result is a small constant.
APInt getUnsignedRangeMax(const SCEV *S)
Determine the max of the unsigned range for a particular SCEV.
LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
LLVM_ABI bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition SetVector.h:59
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:102
void insert_range(Range &&R)
Definition SetVector.h:175
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition SetVector.h:261
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:150
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:338
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
LLVM_ABI std::optional< unsigned > getVScaleForTuning() const
LLVM_ABI InstructionCost getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, TTI::TargetCostKind CostKind, bool ForPoisonSrc=true, ArrayRef< Value * > VL={}) const
Estimate the overhead of scalarizing an instruction.
LLVM_ABI bool supportsEfficientVectorElementLoadStore() const
If target has efficient vector element load/store instructions, it can return true here so that inser...
LLVM_ABI bool prefersVectorizedAddressing() const
Return true if target doesn't mind addresses in vectors.
LLVM_ABI TypeSize getRegisterBitWidth(RegisterKind K) const
LLVM_ABI bool preferFixedOverScalableIfEqualCost(bool IsEpilogue) const
LLVM_ABI InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, OperandValueInfo OpdInfo={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const
LLVM_ABI InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, bool UseMaskForCond=false, bool UseMaskForGaps=false) const
LLVM_ABI InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef< int > Mask={}, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args={}, const Instruction *CxtI=nullptr) const
static LLVM_ABI PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I)
Get the kind of extension that an instruction represents.
static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)
Collect properties of V used in cost analysis, e.g. OP_PowerOf2.
LLVM_ABI bool isElementTypeLegalForScalableVector(Type *Ty) const
LLVM_ABI ElementCount getMinimumVF(unsigned ElemWidth, bool IsScalable) const
TargetCostKind
The kind of cost model.
@ TCK_RecipThroughput
Reciprocal throughput.
@ TCK_CodeSize
Instruction code size.
@ TCK_SizeAndLatency
The weighted sum of size and latency.
@ TCK_Latency
The latency of instruction.
LLVM_ABI InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const
LLVM_ABI InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr, TTI::TargetCostKind CostKind) const
LLVM_ABI InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, PartialReductionExtendKind OpAExtend, PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp, TTI::TargetCostKind CostKind) const
LLVM_ABI InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, const Instruction *I=nullptr) const
LLVM_ABI bool supportsScalableVectors() const
@ TCC_Free
Expected to fold away in lowering.
LLVM_ABI InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
LLVM_ABI InstructionCost getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index) const
LLVM_ABI InstructionCost getOperandsScalarizationOverhead(ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind) const
Estimate the overhead of scalarizing operands with the given types.
@ SK_Splice
Concatenates elements from the first input vector with elements of the second input vector.
@ SK_Broadcast
Broadcast element 0 to all other elements.
@ SK_Reverse
Reverse the order of the vector.
LLVM_ABI InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency, const Instruction *I=nullptr) const
CastContextHint
Represents a hint about the context in which a cast is used.
@ Reversed
The cast is used with a reversed load/store.
@ Masked
The cast is used with a masked load/store.
@ None
The cast is not used with a load/store of any kind.
@ Normal
The cast is used with a normal load/store.
@ Interleave
The cast is used with an interleaved load/store.
@ GatherScatter
The cast is used with a gather/scatter.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
Definition TypeSwitch.h:87
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
Definition TypeSwitch.h:96
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
LLVM_ABI unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:273
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Definition Type.cpp:281
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:352
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:198
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:128
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition Type.cpp:231
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
Definition Type.cpp:294
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:184
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:240
bool isVoidTy() const
Return true if this is 'void'.
Definition Type.h:139
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:292
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition User.cpp:21
Value * getOperand(unsigned i) const
Definition User.h:232
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
Definition VectorUtils.h:74
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:3781
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition VPlan.h:3856
RecipeListTy::iterator iterator
Instruction iterators...
Definition VPlan.h:3808
iterator end()
Definition VPlan.h:3818
iterator begin()
Recipe iterator methods.
Definition VPlan.h:3816
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition VPlan.h:3869
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition VPlan.cpp:246
VPRegionBlock * getEnclosingLoopRegion()
Definition VPlan.cpp:619
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition VPlan.h:3847
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition VPlan.h:81
VPRegionBlock * getParent()
Definition VPlan.h:173
const VPBasicBlock * getExitingBasicBlock() const
Definition VPlan.cpp:190
void setName(const Twine &newName)
Definition VPlan.h:166
size_t getNumSuccessors() const
Definition VPlan.h:219
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
Definition VPlan.h:322
size_t getNumPredecessors() const
Definition VPlan.h:220
VPlan * getPlan()
Definition VPlan.cpp:165
VPBlockBase * getSinglePredecessor() const
Definition VPlan.h:215
const VPBasicBlock * getEntryBasicBlock() const
Definition VPlan.cpp:170
VPBlockBase * getSingleSuccessor() const
Definition VPlan.h:209
const VPBlocksTy & getSuccessors() const
Definition VPlan.h:198
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
Definition VPlanUtils.h:232
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
Definition VPlanUtils.h:253
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
Definition VPlanUtils.h:191
static void reassociateBlocks(VPBlockBase *Old, VPBlockBase *New)
Reassociate all the blocks connected to Old so that they now point to New.
Definition VPlanUtils.h:218
VPlan-based builder utility analogous to IRBuilder.
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPPhi * createScalarPhi(ArrayRef< VPValue * > IncomingValues, DebugLoc DL, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPInstruction * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition VPlanValue.h:424
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition VPlanValue.h:397
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPValue * getStepValue() const
Definition VPlan.h:3658
VPValue * getStartValue() const
Definition VPlan.h:3657
A pure virtual base class for all recipes modeling header phis, including phis for first order recurr...
Definition VPlan.h:1977
virtual VPValue * getBackedgeValue()
Returns the incoming value from the loop backedge.
Definition VPlan.h:2025
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition VPlan.h:2014
A special type of VPBasicBlock that wraps an existing IR basic block.
Definition VPlan.h:3934
Helper to manage IR metadata for recipes.
Definition VPlan.h:942
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:983
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
Definition VPlan.h:1016
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
Definition VPlan.h:1063
@ FirstOrderRecurrenceSplice
Definition VPlan.h:989
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
Definition VPlan.h:1054
unsigned getOpcode() const
Definition VPlan.h:1119
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition VPlan.h:2576
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
A recipe for forming partial reductions.
Definition VPlan.h:2753
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
Definition VPlan.h:1290
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition VPlan.h:394
VPBasicBlock * getParent()
Definition VPlan.h:415
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition VPlan.h:482
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPRecipeBase * tryToCreateWidenRecipe(VPSingleDefRecipe *R, VFRange &Range)
Create and return a widened recipe for R if one can be created within the given VF Range.
VPValue * getBlockInMask(VPBasicBlock *VPBB) const
Returns the entry mask for block VPBB or null if the mask is all-true.
std::optional< unsigned > getScalingForReduction(const Instruction *ExitInst)
void collectScaledReductions(VFRange &Range)
Find all possible partial reductions in the loop and track all of those that are valid so recipes can...
VPReplicateRecipe * handleReplication(Instruction *I, ArrayRef< VPValue * > Operands, VFRange &Range)
Build a VPReplicationRecipe for I using Operands.
VPRecipeBase * tryToCreatePartialReduction(Instruction *Reduction, ArrayRef< VPValue * > Operands, unsigned ScaleFactor)
Create and return a partial reduction recipe for a reduction instruction along with binary operation ...
A recipe for handling reduction phis.
Definition VPlan.h:2331
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
Definition VPlan.h:2391
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
Definition VPlan.h:2385
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:3969
const VPBlockBase * getEntry() const
Definition VPlan.h:4005
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition VPlan.h:2856
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Definition VPlan.h:521
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition VPlan.h:586
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition VPlanValue.h:199
void setOperand(unsigned I, VPValue *New)
Definition VPlanValue.h:243
VPValue * getOperand(unsigned N) const
Definition VPlanValue.h:238
void addOperand(VPValue *Operand)
Definition VPlanValue.h:232
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition VPlan.cpp:135
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition VPlanValue.h:176
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
Definition VPlanValue.h:85
void replaceAllUsesWith(VPValue *New)
Definition VPlan.cpp:1403
user_iterator user_begin()
Definition VPlanValue.h:130
unsigned getNumUsers() const
Definition VPlanValue.h:113
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
Definition VPlan.cpp:1407
user_range users()
Definition VPlanValue.h:134
A recipe to compute a pointer to the last element of each part of a widened memory access for widened...
Definition VPlan.h:1841
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition VPlan.h:1482
A recipe for handling GEP instructions.
Definition VPlan.h:1769
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
Definition VPlan.h:2042
VPValue * getStepValue()
Returns the step value of the induction.
Definition VPlan.h:2070
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition VPlan.h:2087
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition VPlan.h:2117
A common base class for widening memory operations.
Definition VPlan.h:3150
A recipe for widened phis.
Definition VPlan.h:2253
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Definition VPlan.h:1439
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4072
bool hasVF(ElementCount VF) const
Definition VPlan.h:4281
VPBasicBlock * getEntry()
Definition VPlan.h:4171
VPValue & getVectorTripCount()
The vector trip count.
Definition VPlan.h:4261
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
Definition VPlan.h:4267
VPValue & getVF()
Returns the VF of the vector loop region.
Definition VPlan.h:4264
VPValue * getTripCount() const
The trip count of the original loop.
Definition VPlan.h:4233
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
Definition VPlan.h:4288
bool hasUF(unsigned UF) const
Definition VPlan.h:4299
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition VPlan.h:4223
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition VPlan.cpp:1037
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
Definition VPlan.h:4444
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
Definition VPlan.cpp:1019
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
Definition VPlan.h:4247
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
Definition VPlan.h:4196
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
Definition VPlan.h:4323
bool hasScalarVFOnly() const
Definition VPlan.h:4292
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
Definition VPlan.h:4214
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition VPlan.cpp:943
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition VPlan.h:4377
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition VPlan.h:4219
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition VPlan.h:4176
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
Definition VPlan.cpp:1179
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:256
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:166
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
Definition Value.cpp:390
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:439
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:546
iterator_range< user_iterator > users()
Definition Value.h:426
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.cpp:1101
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:202
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:175
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
Definition TypeSize.h:269
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:230
constexpr bool isNonZero() const
Definition TypeSize.h:156
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
Definition TypeSize.h:277
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:216
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:169
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
Definition TypeSize.h:256
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:172
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
constexpr bool isZero() const
Definition TypeSize.h:154
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:223
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:252
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:237
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
IteratorT end() const
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
A raw_ostream that writes to an std::string.
Changed
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Entry
Definition COFF.h:862
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition CallingConv.h:76
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
Definition DwarfDebug.h:189
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
OneOps_match< OpTy, Instruction::Freeze > m_Freeze(const OpTy &Op)
Matches FreezeInst.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
SpecificCmpClass_match< LHS, RHS, ICmpInst > m_SpecificICmp(CmpPredicate MatchPred, const LHS &L, const RHS &R)
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
MatchFunctor< Val, Pattern > match_fn(const Pattern &P)
A match functor that can be used as a UnaryPredicate in functional algorithms like all_of.
class_match< const SCEVVScale > m_SCEVVScale()
bind_cst_ty m_scev_APInt(const APInt *&C)
Match an SCEV constant and bind it to an APInt.
specificloop_ty m_SpecificLoop(const Loop *L)
cst_pred_ty< is_specific_signed_cst > m_scev_SpecificSInt(int64_t V)
Match an SCEV constant with a plain signed integer (sign-extended value will be matched)
SCEVAffineAddRec_match< Op0_t, Op1_t, class_match< const Loop > > m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1)
SCEVBinaryExpr_match< SCEVMulExpr, Op0_t, Op1_t > m_scev_Mul(const Op0_t &Op0, const Op1_t &Op1)
bool match(const SCEV *S, const Pattern &P)
class_match< const SCEV > m_SCEV()
match_combine_or< AllRecipe_match< Instruction::ZExt, Op0_t >, AllRecipe_match< Instruction::SExt, Op0_t > > m_ZExtOrSExt(const Op0_t &Op0)
VPInstruction_match< VPInstruction::ExtractLastElement, Op0_t > m_ExtractLastElement(const Op0_t &Op0)
class_match< VPValue > m_VPValue()
Match an arbitrary VPValue and ignore it.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< InstrNode * > Instr
Definition RDFGraph.h:389
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
Definition VPlanUtils.h:44
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr)
Get or create a VPValue that corresponds to the expansion of Expr.
VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)
Returns the header block of the first, top-level loop, or null if none exist.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
unsigned getVFScaleFactor(VPRecipeBase *R)
Get the VF scaling factor applied to the recipe's output, if the recipe has one.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, AssumptionCache *AC, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
Simplify each loop in a loop nest recursively.
LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:318
@ Offset
Definition DWP.cpp:477
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition STLExtras.h:831
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)
Add code that checks at runtime if the accessed arrays in PointerChecks overlap.
auto cast_if_present(const Y &Val)
cast_if_present<X> - Functionally identical to cast, except that a null value is accepted.
Definition Casting.h:689
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
cl::opt< bool > VerifyEachVPlan
LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)
Return either:
static void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop, VectorizationFactor VF, unsigned IC)
Report successful vectorization of the loop.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1705
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1657
LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate=false)
Verify invariants for general VPlans.
LLVM_ABI Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
InstructionCost Cost
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2452
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:649
LLVM_ABI bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)
Put a loop nest into LCSSA form.
Definition LCSSA.cpp:449
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2116
LLVM_ABI bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:634
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:293
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
Definition VPlanCFG.h:216
LLVM_ABI bool VerifySCEV
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:682
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
Definition VPlanCFG.h:243
SmallVector< VPRegisterUsage, 8 > calculateRegisterUsageForPlan(VPlan &Plan, ArrayRef< ElementCount > VFs, const TargetTransformInfo &TTI, const SmallPtrSetImpl< const Value * > &ValuesToIgnore)
Estimate the register usage for Plan and vectorization factors in VFs by calculating the highest numb...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:348
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:759
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1712
void collectEphemeralRecipesForVPlan(VPlan &Plan, DenseSet< VPRecipeBase * > &EphRecipes)
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
bool containsIrreducibleCFG(RPOTraversalT &RPOTraversal, const LoopInfoT &LI)
Return true if the control flow in RPOTraversal is irreducible.
Definition CFG.h:149
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:288
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1624
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1719
LLVM_ABI cl::opt< bool > EnableLoopVectorization
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
Definition Local.cpp:421
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
T * find_singleton(R &&Range, Predicate P, bool AllowRepeats=false)
Return the single value in Range that satisfies P(<member of Range> *, AllowRepeats)->T * returning n...
Definition STLExtras.h:1767
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
cl::opt< unsigned > ForceTargetInstructionCost
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition Format.h:126
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
Definition MathExtras.h:405
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
TargetTransformInfo TTI
static void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr, DebugLoc DL={})
Reports an informative message: print Msg for debugging purposes as well as an optimization remark.
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
@ FMulAdd
Sum of float products with llvm.fmuladd(a * b + sum).
@ Sub
Subtraction of integers.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:144
LLVM_ABI void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
DWARFExpression::Operation Op
ScalarEpilogueLowering
@ CM_ScalarEpilogueNotAllowedLowTripLoop
@ CM_ScalarEpilogueNotNeededUsePredicate
@ CM_ScalarEpilogueNotAllowedOptSize
@ CM_ScalarEpilogueAllowed
@ CM_ScalarEpilogueNotAllowedUsePredicate
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:565
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1738
auto predecessors(const MachineBasicBlock *BB)
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
Definition iterator.h:363
cl::opt< bool > EnableVPlanNativePath
Definition VPlan.cpp:56
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
ArrayRef< Type * > getContainedTypes(Type *const &Ty)
Returns the types contained in Ty.
LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)
bool pred_empty(const BasicBlock *BB)
Definition CFG.h:119
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ None
Don't use tail folding.
@ DataWithEVL
Use predicated EVL instructions for tail-folding.
@ DataAndControlFlow
Use predicate to control both data and control flow.
@ DataWithoutLaneMask
Same as Data, but avoids using the get.active.lane.mask intrinsic to calculate the mask and instead i...
@ Data
Use predicate only to mask operations on data in the loop.
unsigned getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind)
A helper function that returns how much we should divide the cost of a predicated block by.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI bool hasBranchWeightMD(const Instruction &I)
Checks if an instructions has Branch Weight Metadata.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
Definition Hashing.h:592
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:299
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
std::unique_ptr< VPlan > VPlanPtr
Definition VPlan.h:77
constexpr detail::IsaCheckPredicate< Types... > IsaPred
Function object wrapper for the llvm::isa type check.
Definition Casting.h:836
LLVM_ABI MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:466
LLVM_ABI cl::opt< bool > EnableLoopInterleaving
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:872
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
static LLVM_ABI void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
An information struct used to provide DenseMap with the various necessary components for a given valu...
Encapsulate information regarding vectorization of a loop and its epilogue.
EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF, ElementCount EVF, unsigned EUF, VPlan &EpiloguePlan)
A class that represents two vectorization factors (initialized with 0 by default).
static FixedScalableVFPair getNone()
This holds details about a histogram operation – a load -> update -> store sequence where each lane i...
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
TargetLibraryInfo * TLI
LLVM_ABI LoopVectorizeResult runImpl(Function &F)
LLVM_ABI bool processLoop(Loop *L)
ProfileSummaryInfo * PSI
LoopAccessInfoManager * LAIs
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
LLVM_ABI LoopVectorizePass(LoopVectorizeOptions Opts={})
BlockFrequencyInfo * BFI
ScalarEvolution * SE
AssumptionCache * AC
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
OptimizationRemarkEmitter * ORE
TargetTransformInfo * TTI
Storage for information about made changes.
A chain of instructions that form a partial reduction.
Instruction * Reduction
The top-level binary operation that forms the reduction to a scalar after the loop body.
Instruction * ExtendA
The extension of each of the inner binary operation's operands.
A CRTP mix-in to automatically provide informational APIs needed for passes.
Definition PassManager.h:70
A marker analysis to determine if extra passes should be run after loop vectorization.
static LLVM_ABI AnalysisKey Key
Holds the VFShape for a specific scalar to vector function mapping.
std::optional< unsigned > getParamIndexForOptionalMask() const
Instruction Set Architecture.
Encapsulates information needed to describe a parameter.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
ElementCount End
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool isLegacyUniformAfterVectorization(Instruction *I, ElementCount VF) const
Return true if I is considered uniform-after-vectorization in the legacy cost model for VF.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
Definition VPlan.h:2296
A struct that represents some properties of the register usage of a loop.
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
A recipe for widening select instructions.
Definition VPlan.h:1723
static void materializeBroadcasts(VPlan &Plan)
Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as v...
static void materializeBackedgeTakenCount(VPlan &Plan, VPBasicBlock *VectorPH)
Materialize the backedge-taken count to be computed explicitly using VPInstructions.
static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan > buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE)
Create a base VPlan0, serving as the common starting point for all later candidates.
static void optimizeInductionExitUsers(VPlan &Plan, DenseMap< VPValue *, VPValue * > &EndValues, ScalarEvolution &SE)
If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them p...
static LLVM_ABI_FOR_TEST void handleEarlyExits(VPlan &Plan, bool HasUncountableExit)
Update Plan to account for all early exits.
static void canonicalizeEVLLoops(VPlan &Plan)
Transform EVL loops to use variable-length stepping after region dissolution.
static void dropPoisonGeneratingRecipes(VPlan &Plan, const std::function< bool(BasicBlock *)> &BlockNeedsPredication)
Drop poison flags from recipes that may generate a poison value that is used after vectorization,...
static void createInterleaveGroups(VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed)
static bool runPass(bool(*Transform)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.
static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)
Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.
static void materializeBuildVectors(VPlan &Plan)
Add explicit Build[Struct]Vector recipes that combine multiple scalar values into single vectors.
static void unrollByUF(VPlan &Plan, unsigned UF)
Explicitly unroll Plan by UF.
static DenseMap< const SCEV *, Value * > expandSCEVs(VPlan &Plan, ScalarEvolution &SE)
Expand VPExpandSCEVRecipes in Plan's entry block.
static void convertToConcreteRecipes(VPlan &Plan)
Lower abstract recipes to concrete ones, that can be codegen'd.
static void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, bool CheckNeededWithTailFolding, Loop *OrigLoop, const uint32_t *MinItersBypassWeights, DebugLoc DL, ScalarEvolution &SE)
static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
This function converts initial recipes to the abstract recipes and clamps Range based on cost model f...
static void materializeConstantVectorTripCount(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
static DenseMap< VPBasicBlock *, VPValue * > introduceMasksAndLinearize(VPlan &Plan, bool FoldTail)
Predicate and linearize the control-flow in the only loop region of Plan.
static void addExplicitVectorLength(VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV...
static void replaceSymbolicStrides(VPlan &Plan, PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &StridesMap)
Replace symbolic strides from StridesMap in Plan with constants when possible.
static bool handleMaxMinNumReductions(VPlan &Plan)
Check if Plan contains any FMaxNum or FMinNum reductions.
static void removeBranchOnConst(VPlan &Plan)
Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their ...
static LLVM_ABI_FOR_TEST void createLoopRegions(VPlan &Plan)
Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG...
static void removeDeadRecipes(VPlan &Plan)
Remove dead recipes from Plan.
static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock, bool AddBranchWeights)
Wrap runtime check block CheckBlock in a VPIRBB and Cond in a VPValue and connect the block to Plan,...
static void materializeVectorTripCount(VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue)
Materialize vector trip count computations to a set of VPInstructions.
static void simplifyRecipes(VPlan &Plan)
Perform instcombine-like simplifications on recipes in Plan.
static LLVM_ABI_FOR_TEST bool tryToConvertVPInstructionsToVPRecipes(VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, const TargetLibraryInfo &TLI)
Replaces the VPInstructions in Plan with corresponding widen recipes.
static void replicateByVF(VPlan &Plan, ElementCount VF)
Replace each replicating VPReplicateRecipe and VPInstruction outside of any replicate region in Plan ...
static void clearReductionWrapFlags(VPlan &Plan)
Clear NSW/NUW flags from reduction instructions if necessary.
static void cse(VPlan &Plan)
Perform common-subexpression-elimination on Plan.
static void addActiveLaneMask(VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)
Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe,...
static void optimize(VPlan &Plan)
Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations,...
static void dissolveLoopRegions(VPlan &Plan)
Replace loop regions with explicit CFG.
static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF, unsigned VectorRegWidth)
Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups re...
static void truncateToMinimalBitwidths(VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
Insert truncates and extends for any truncated recipe.
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder)
Try to have all users of fixed-order recurrences appear after the recipe defining their previous valu...
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
Optimize Plan based on BestVF and BestUF.
static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)
Materialize VF and VFxUF to be computed explicitly using VPInstructions.
static void addMinimumVectorEpilogueIterationCheck(VPlan &Plan, Value *TripCount, Value *VectorTripCount, bool RequiresScalarEpilogue, ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep, unsigned EpilogueLoopStep, ScalarEvolution &SE)
Add a check to Plan to see if the epilogue vector loop should be executed.
static LLVM_ABI_FOR_TEST void addMiddleCheck(VPlan &Plan, bool RequiresScalarEpilogueCheck, bool TailFolded)
If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle bloc...
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
InstructionCost Cost
Cost of the loop with that width.
ElementCount MinProfitableTripCount
The minimum trip count required to make vectorization profitable, e.g.
ElementCount Width
Vector width with best cost.
InstructionCost ScalarCost
Cost of the scalar loop.
static VectorizationFactor Disabled()
Width 1 means no vectorization, cost 0 means uncomputed cost.
static LLVM_ABI bool HoistRuntimeChecks