Bug Summary

File:lib/Transforms/Vectorize/LoopVectorize.cpp
Warning:line 6433, column 35
Potential leak of memory pointed to by 'BlockMask'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LoopVectorize.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn358860/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-9~svn358860/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn358860/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn358860/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn358860=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-04-22-050718-5320-1 -x c++ /build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp -faddrsig

/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp

1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/Proposal/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlanHCFGBuilder.h"
60#include "VPlanHCFGTransforms.h"
61#include "VPlanPredicator.h"
62#include "llvm/ADT/APInt.h"
63#include "llvm/ADT/ArrayRef.h"
64#include "llvm/ADT/DenseMap.h"
65#include "llvm/ADT/DenseMapInfo.h"
66#include "llvm/ADT/Hashing.h"
67#include "llvm/ADT/MapVector.h"
68#include "llvm/ADT/None.h"
69#include "llvm/ADT/Optional.h"
70#include "llvm/ADT/STLExtras.h"
71#include "llvm/ADT/SetVector.h"
72#include "llvm/ADT/SmallPtrSet.h"
73#include "llvm/ADT/SmallVector.h"
74#include "llvm/ADT/Statistic.h"
75#include "llvm/ADT/StringRef.h"
76#include "llvm/ADT/Twine.h"
77#include "llvm/ADT/iterator_range.h"
78#include "llvm/Analysis/AssumptionCache.h"
79#include "llvm/Analysis/BasicAliasAnalysis.h"
80#include "llvm/Analysis/BlockFrequencyInfo.h"
81#include "llvm/Analysis/CFG.h"
82#include "llvm/Analysis/CodeMetrics.h"
83#include "llvm/Analysis/DemandedBits.h"
84#include "llvm/Analysis/GlobalsModRef.h"
85#include "llvm/Analysis/LoopAccessAnalysis.h"
86#include "llvm/Analysis/LoopAnalysisManager.h"
87#include "llvm/Analysis/LoopInfo.h"
88#include "llvm/Analysis/LoopIterator.h"
89#include "llvm/Analysis/MemorySSA.h"
90#include "llvm/Analysis/OptimizationRemarkEmitter.h"
91#include "llvm/Analysis/ProfileSummaryInfo.h"
92#include "llvm/Analysis/ScalarEvolution.h"
93#include "llvm/Analysis/ScalarEvolutionExpander.h"
94#include "llvm/Analysis/ScalarEvolutionExpressions.h"
95#include "llvm/Analysis/TargetLibraryInfo.h"
96#include "llvm/Analysis/TargetTransformInfo.h"
97#include "llvm/Analysis/VectorUtils.h"
98#include "llvm/IR/Attributes.h"
99#include "llvm/IR/BasicBlock.h"
100#include "llvm/IR/CFG.h"
101#include "llvm/IR/Constant.h"
102#include "llvm/IR/Constants.h"
103#include "llvm/IR/DataLayout.h"
104#include "llvm/IR/DebugInfoMetadata.h"
105#include "llvm/IR/DebugLoc.h"
106#include "llvm/IR/DerivedTypes.h"
107#include "llvm/IR/DiagnosticInfo.h"
108#include "llvm/IR/Dominators.h"
109#include "llvm/IR/Function.h"
110#include "llvm/IR/IRBuilder.h"
111#include "llvm/IR/InstrTypes.h"
112#include "llvm/IR/Instruction.h"
113#include "llvm/IR/Instructions.h"
114#include "llvm/IR/IntrinsicInst.h"
115#include "llvm/IR/Intrinsics.h"
116#include "llvm/IR/LLVMContext.h"
117#include "llvm/IR/Metadata.h"
118#include "llvm/IR/Module.h"
119#include "llvm/IR/Operator.h"
120#include "llvm/IR/Type.h"
121#include "llvm/IR/Use.h"
122#include "llvm/IR/User.h"
123#include "llvm/IR/Value.h"
124#include "llvm/IR/ValueHandle.h"
125#include "llvm/IR/Verifier.h"
126#include "llvm/Pass.h"
127#include "llvm/Support/Casting.h"
128#include "llvm/Support/CommandLine.h"
129#include "llvm/Support/Compiler.h"
130#include "llvm/Support/Debug.h"
131#include "llvm/Support/ErrorHandling.h"
132#include "llvm/Support/MathExtras.h"
133#include "llvm/Support/raw_ostream.h"
134#include "llvm/Transforms/Utils/BasicBlockUtils.h"
135#include "llvm/Transforms/Utils/LoopSimplify.h"
136#include "llvm/Transforms/Utils/LoopUtils.h"
137#include "llvm/Transforms/Utils/LoopVersioning.h"
138#include "llvm/Transforms/Utils/SizeOpts.h"
139#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
140#include <algorithm>
141#include <cassert>
142#include <cstdint>
143#include <cstdlib>
144#include <functional>
145#include <iterator>
146#include <limits>
147#include <memory>
148#include <string>
149#include <tuple>
150#include <utility>
151#include <vector>
152
153using namespace llvm;
154
155#define LV_NAME"loop-vectorize" "loop-vectorize"
156#define DEBUG_TYPE"loop-vectorize" LV_NAME"loop-vectorize"
157
158/// @{
159/// Metadata attribute names
160static const char *const LLVMLoopVectorizeFollowupAll =
161 "llvm.loop.vectorize.followup_all";
162static const char *const LLVMLoopVectorizeFollowupVectorized =
163 "llvm.loop.vectorize.followup_vectorized";
164static const char *const LLVMLoopVectorizeFollowupEpilogue =
165 "llvm.loop.vectorize.followup_epilogue";
166/// @}
167
168STATISTIC(LoopsVectorized, "Number of loops vectorized")static llvm::Statistic LoopsVectorized = {"loop-vectorize", "LoopsVectorized"
, "Number of loops vectorized", {0}, {false}}
;
169STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization")static llvm::Statistic LoopsAnalyzed = {"loop-vectorize", "LoopsAnalyzed"
, "Number of loops analyzed for vectorization", {0}, {false}}
;
170
171/// Loops with a known constant trip count below this number are vectorized only
172/// if no scalar iteration overheads are incurred.
173static cl::opt<unsigned> TinyTripCountVectorThreshold(
174 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
175 cl::desc("Loops with a constant trip count that is smaller than this "
176 "value are vectorized only if no scalar iteration overheads "
177 "are incurred."));
178
179static cl::opt<bool> MaximizeBandwidth(
180 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
181 cl::desc("Maximize bandwidth when selecting vectorization factor which "
182 "will be determined by the smallest type in loop."));
183
184static cl::opt<bool> EnableInterleavedMemAccesses(
185 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
186 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
187
188/// An interleave-group may need masking if it resides in a block that needs
189/// predication, or in order to mask away gaps.
190static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
191 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
192 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
193
194/// We don't interleave loops with a known constant trip count below this
195/// number.
196static const unsigned TinyTripCountInterleaveThreshold = 128;
197
198static cl::opt<unsigned> ForceTargetNumScalarRegs(
199 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
200 cl::desc("A flag that overrides the target's number of scalar registers."));
201
202static cl::opt<unsigned> ForceTargetNumVectorRegs(
203 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
204 cl::desc("A flag that overrides the target's number of vector registers."));
205
206static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
207 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
208 cl::desc("A flag that overrides the target's max interleave factor for "
209 "scalar loops."));
210
211static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
212 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
213 cl::desc("A flag that overrides the target's max interleave factor for "
214 "vectorized loops."));
215
216static cl::opt<unsigned> ForceTargetInstructionCost(
217 "force-target-instruction-cost", cl::init(0), cl::Hidden,
218 cl::desc("A flag that overrides the target's expected cost for "
219 "an instruction to a single constant value. Mostly "
220 "useful for getting consistent testing."));
221
222static cl::opt<unsigned> SmallLoopCost(
223 "small-loop-cost", cl::init(20), cl::Hidden,
224 cl::desc(
225 "The cost of a loop that is considered 'small' by the interleaver."));
226
227static cl::opt<bool> LoopVectorizeWithBlockFrequency(
228 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
229 cl::desc("Enable the use of the block frequency analysis to access PGO "
230 "heuristics minimizing code growth in cold regions and being more "
231 "aggressive in hot regions."));
232
233// Runtime interleave loops for load/store throughput.
234static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
235 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
236 cl::desc(
237 "Enable runtime interleaving until load/store ports are saturated"));
238
239/// The number of stores in a loop that are allowed to need predication.
240static cl::opt<unsigned> NumberOfStoresToPredicate(
241 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
242 cl::desc("Max number of stores to be predicated behind an if."));
243
244static cl::opt<bool> EnableIndVarRegisterHeur(
245 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
246 cl::desc("Count the induction variable only once when interleaving"));
247
248static cl::opt<bool> EnableCondStoresVectorization(
249 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
250 cl::desc("Enable if predication of stores during vectorization."));
251
252static cl::opt<unsigned> MaxNestedScalarReductionIC(
253 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
254 cl::desc("The maximum interleave count to use when interleaving a scalar "
255 "reduction in a nested loop."));
256
257cl::opt<bool> EnableVPlanNativePath(
258 "enable-vplan-native-path", cl::init(false), cl::Hidden,
259 cl::desc("Enable VPlan-native vectorization path with "
260 "support for outer loop vectorization."));
261
262// FIXME: Remove this switch once we have divergence analysis. Currently we
263// assume divergent non-backedge branches when this switch is true.
264cl::opt<bool> EnableVPlanPredication(
265 "enable-vplan-predication", cl::init(false), cl::Hidden,
266 cl::desc("Enable VPlan-native vectorization path predicator with "
267 "support for outer loop vectorization."));
268
269// This flag enables the stress testing of the VPlan H-CFG construction in the
270// VPlan-native vectorization path. It must be used in conjuction with
271// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
272// verification of the H-CFGs built.
273static cl::opt<bool> VPlanBuildStressTest(
274 "vplan-build-stress-test", cl::init(false), cl::Hidden,
275 cl::desc(
276 "Build VPlan for every supported loop nest in the function and bail "
277 "out right after the build (stress test the VPlan H-CFG construction "
278 "in the VPlan-native vectorization path)."));
279
280cl::opt<bool> llvm::EnableLoopInterleaving(
281 "interleave-loops", cl::init(true), cl::Hidden,
282 cl::desc("Enable loop interleaving in Loop vectorization passes"));
283cl::opt<bool> llvm::EnableLoopVectorization(
284 "vectorize-loops", cl::init(false), cl::Hidden,
285 cl::desc("Run the Loop vectorization passes"));
286
287/// A helper function for converting Scalar types to vector types.
288/// If the incoming type is void, we return void. If the VF is 1, we return
289/// the scalar type.
290static Type *ToVectorTy(Type *Scalar, unsigned VF) {
291 if (Scalar->isVoidTy() || VF == 1)
292 return Scalar;
293 return VectorType::get(Scalar, VF);
294}
295
296/// A helper function that returns the type of loaded or stored value.
297static Type *getMemInstValueType(Value *I) {
298 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction") ? static_cast<void>
(0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 299, __PRETTY_FUNCTION__))
299 "Expected Load or Store instruction")(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction") ? static_cast<void>
(0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 299, __PRETTY_FUNCTION__))
;
300 if (auto *LI = dyn_cast<LoadInst>(I))
301 return LI->getType();
302 return cast<StoreInst>(I)->getValueOperand()->getType();
303}
304
305/// A helper function that returns true if the given type is irregular. The
306/// type is irregular if its allocated size doesn't equal the store size of an
307/// element of the corresponding vector type at the given vectorization factor.
308static bool hasIrregularType(Type *Ty, const DataLayout &DL, unsigned VF) {
309 // Determine if an array of VF elements of type Ty is "bitcast compatible"
310 // with a <VF x Ty> vector.
311 if (VF > 1) {
312 auto *VectorTy = VectorType::get(Ty, VF);
313 return VF * DL.getTypeAllocSize(Ty) != DL.getTypeStoreSize(VectorTy);
314 }
315
316 // If the vectorization factor is one, we just check if an array of type Ty
317 // requires padding between elements.
318 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
319}
320
321/// A helper function that returns the reciprocal of the block probability of
322/// predicated blocks. If we return X, we are assuming the predicated block
323/// will execute once for every X iterations of the loop header.
324///
325/// TODO: We should use actual block probability here, if available. Currently,
326/// we always assume predicated blocks have a 50% chance of executing.
327static unsigned getReciprocalPredBlockProb() { return 2; }
328
329/// A helper function that adds a 'fast' flag to floating-point operations.
330static Value *addFastMathFlag(Value *V) {
331 if (isa<FPMathOperator>(V))
332 cast<Instruction>(V)->setFastMathFlags(FastMathFlags::getFast());
333 return V;
334}
335
336static Value *addFastMathFlag(Value *V, FastMathFlags FMF) {
337 if (isa<FPMathOperator>(V))
338 cast<Instruction>(V)->setFastMathFlags(FMF);
339 return V;
340}
341
342/// A helper function that returns an integer or floating-point constant with
343/// value C.
344static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
345 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
346 : ConstantFP::get(Ty, C);
347}
348
349namespace llvm {
350
351/// InnerLoopVectorizer vectorizes loops which contain only one basic
352/// block to a specified vectorization factor (VF).
353/// This class performs the widening of scalars into vectors, or multiple
354/// scalars. This class also implements the following features:
355/// * It inserts an epilogue loop for handling loops that don't have iteration
356/// counts that are known to be a multiple of the vectorization factor.
357/// * It handles the code generation for reduction variables.
358/// * Scalarization (implementation using scalars) of un-vectorizable
359/// instructions.
360/// InnerLoopVectorizer does not perform any vectorization-legality
361/// checks, and relies on the caller to check for the different legality
362/// aspects. The InnerLoopVectorizer relies on the
363/// LoopVectorizationLegality class to provide information about the induction
364/// and reduction variables that were found to a given vectorization factor.
365class InnerLoopVectorizer {
366public:
367 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
368 LoopInfo *LI, DominatorTree *DT,
369 const TargetLibraryInfo *TLI,
370 const TargetTransformInfo *TTI, AssumptionCache *AC,
371 OptimizationRemarkEmitter *ORE, unsigned VecWidth,
372 unsigned UnrollFactor, LoopVectorizationLegality *LVL,
373 LoopVectorizationCostModel *CM)
374 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
375 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
376 Builder(PSE.getSE()->getContext()),
377 VectorLoopValueMap(UnrollFactor, VecWidth), Legal(LVL), Cost(CM) {}
378 virtual ~InnerLoopVectorizer() = default;
379
380 /// Create a new empty loop. Unlink the old loop and connect the new one.
381 /// Return the pre-header block of the new loop.
382 BasicBlock *createVectorizedLoopSkeleton();
383
384 /// Widen a single instruction within the innermost loop.
385 void widenInstruction(Instruction &I);
386
387 /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
388 void fixVectorizedLoop();
389
390 // Return true if any runtime check is added.
391 bool areSafetyChecksAdded() { return AddedSafetyChecks; }
392
393 /// A type for vectorized values in the new loop. Each value from the
394 /// original loop, when vectorized, is represented by UF vector values in the
395 /// new unrolled loop, where UF is the unroll factor.
396 using VectorParts = SmallVector<Value *, 2>;
397
398 /// Vectorize a single PHINode in a block. This method handles the induction
399 /// variable canonicalization. It supports both VF = 1 for unrolled loops and
400 /// arbitrary length vectors.
401 void widenPHIInstruction(Instruction *PN, unsigned UF, unsigned VF);
402
403 /// A helper function to scalarize a single Instruction in the innermost loop.
404 /// Generates a sequence of scalar instances for each lane between \p MinLane
405 /// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
406 /// inclusive..
407 void scalarizeInstruction(Instruction *Instr, const VPIteration &Instance,
408 bool IfPredicateInstr);
409
410 /// Widen an integer or floating-point induction variable \p IV. If \p Trunc
411 /// is provided, the integer induction variable will first be truncated to
412 /// the corresponding type.
413 void widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc = nullptr);
414
415 /// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a
416 /// vector or scalar value on-demand if one is not yet available. When
417 /// vectorizing a loop, we visit the definition of an instruction before its
418 /// uses. When visiting the definition, we either vectorize or scalarize the
419 /// instruction, creating an entry for it in the corresponding map. (In some
420 /// cases, such as induction variables, we will create both vector and scalar
421 /// entries.) Then, as we encounter uses of the definition, we derive values
422 /// for each scalar or vector use unless such a value is already available.
423 /// For example, if we scalarize a definition and one of its uses is vector,
424 /// we build the required vector on-demand with an insertelement sequence
425 /// when visiting the use. Otherwise, if the use is scalar, we can use the
426 /// existing scalar definition.
427 ///
428 /// Return a value in the new loop corresponding to \p V from the original
429 /// loop at unroll index \p Part. If the value has already been vectorized,
430 /// the corresponding vector entry in VectorLoopValueMap is returned. If,
431 /// however, the value has a scalar entry in VectorLoopValueMap, we construct
432 /// a new vector value on-demand by inserting the scalar values into a vector
433 /// with an insertelement sequence. If the value has been neither vectorized
434 /// nor scalarized, it must be loop invariant, so we simply broadcast the
435 /// value into a vector.
436 Value *getOrCreateVectorValue(Value *V, unsigned Part);
437
438 /// Return a value in the new loop corresponding to \p V from the original
439 /// loop at unroll and vector indices \p Instance. If the value has been
440 /// vectorized but not scalarized, the necessary extractelement instruction
441 /// will be generated.
442 Value *getOrCreateScalarValue(Value *V, const VPIteration &Instance);
443
444 /// Construct the vector value of a scalarized value \p V one lane at a time.
445 void packScalarIntoVectorValue(Value *V, const VPIteration &Instance);
446
447 /// Try to vectorize the interleaved access group that \p Instr belongs to,
448 /// optionally masking the vector operations if \p BlockInMask is non-null.
449 void vectorizeInterleaveGroup(Instruction *Instr,
450 VectorParts *BlockInMask = nullptr);
451
452 /// Vectorize Load and Store instructions, optionally masking the vector
453 /// operations if \p BlockInMask is non-null.
454 void vectorizeMemoryInstruction(Instruction *Instr,
455 VectorParts *BlockInMask = nullptr);
456
457 /// Set the debug location in the builder using the debug location in
458 /// the instruction.
459 void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr);
460
461 /// Fix the non-induction PHIs in the OrigPHIsToFix vector.
462 void fixNonInductionPHIs(void);
463
464protected:
465 friend class LoopVectorizationPlanner;
466
467 /// A small list of PHINodes.
468 using PhiVector = SmallVector<PHINode *, 4>;
469
470 /// A type for scalarized values in the new loop. Each value from the
471 /// original loop, when scalarized, is represented by UF x VF scalar values
472 /// in the new unrolled loop, where UF is the unroll factor and VF is the
473 /// vectorization factor.
474 using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
475
476 /// Set up the values of the IVs correctly when exiting the vector loop.
477 void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
478 Value *CountRoundDown, Value *EndValue,
479 BasicBlock *MiddleBlock);
480
481 /// Create a new induction variable inside L.
482 PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
483 Value *Step, Instruction *DL);
484
485 /// Handle all cross-iteration phis in the header.
486 void fixCrossIterationPHIs();
487
488 /// Fix a first-order recurrence. This is the second phase of vectorizing
489 /// this phi node.
490 void fixFirstOrderRecurrence(PHINode *Phi);
491
492 /// Fix a reduction cross-iteration phi. This is the second phase of
493 /// vectorizing this phi node.
494 void fixReduction(PHINode *Phi);
495
496 /// The Loop exit block may have single value PHI nodes with some
497 /// incoming value. While vectorizing we only handled real values
498 /// that were defined inside the loop and we should have one value for
499 /// each predecessor of its parent basic block. See PR14725.
500 void fixLCSSAPHIs();
501
502 /// Iteratively sink the scalarized operands of a predicated instruction into
503 /// the block that was created for it.
504 void sinkScalarOperands(Instruction *PredInst);
505
506 /// Shrinks vector element sizes to the smallest bitwidth they can be legally
507 /// represented as.
508 void truncateToMinimalBitwidths();
509
510 /// Insert the new loop to the loop hierarchy and pass manager
511 /// and update the analysis passes.
512 void updateAnalysis();
513
514 /// Create a broadcast instruction. This method generates a broadcast
515 /// instruction (shuffle) for loop invariant values and for the induction
516 /// value. If this is the induction variable then we extend it to N, N+1, ...
517 /// this is needed because each iteration in the loop corresponds to a SIMD
518 /// element.
519 virtual Value *getBroadcastInstrs(Value *V);
520
521 /// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...)
522 /// to each vector element of Val. The sequence starts at StartIndex.
523 /// \p Opcode is relevant for FP induction variable.
524 virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step,
525 Instruction::BinaryOps Opcode =
526 Instruction::BinaryOpsEnd);
527
528 /// Compute scalar induction steps. \p ScalarIV is the scalar induction
529 /// variable on which to base the steps, \p Step is the size of the step, and
530 /// \p EntryVal is the value from the original loop that maps to the steps.
531 /// Note that \p EntryVal doesn't have to be an induction variable - it
532 /// can also be a truncate instruction.
533 void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal,
534 const InductionDescriptor &ID);
535
536 /// Create a vector induction phi node based on an existing scalar one. \p
537 /// EntryVal is the value from the original loop that maps to the vector phi
538 /// node, and \p Step is the loop-invariant step. If \p EntryVal is a
539 /// truncate instruction, instead of widening the original IV, we widen a
540 /// version of the IV truncated to \p EntryVal's type.
541 void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
542 Value *Step, Instruction *EntryVal);
543
544 /// Returns true if an instruction \p I should be scalarized instead of
545 /// vectorized for the chosen vectorization factor.
546 bool shouldScalarizeInstruction(Instruction *I) const;
547
548 /// Returns true if we should generate a scalar version of \p IV.
549 bool needsScalarInduction(Instruction *IV) const;
550
551 /// If there is a cast involved in the induction variable \p ID, which should
552 /// be ignored in the vectorized loop body, this function records the
553 /// VectorLoopValue of the respective Phi also as the VectorLoopValue of the
554 /// cast. We had already proved that the casted Phi is equal to the uncasted
555 /// Phi in the vectorized loop (under a runtime guard), and therefore
556 /// there is no need to vectorize the cast - the same value can be used in the
557 /// vector loop for both the Phi and the cast.
558 /// If \p VectorLoopValue is a scalarized value, \p Lane is also specified,
559 /// Otherwise, \p VectorLoopValue is a widened/vectorized value.
560 ///
561 /// \p EntryVal is the value from the original loop that maps to the vector
562 /// phi node and is used to distinguish what is the IV currently being
563 /// processed - original one (if \p EntryVal is a phi corresponding to the
564 /// original IV) or the "newly-created" one based on the proof mentioned above
565 /// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the
566 /// latter case \p EntryVal is a TruncInst and we must not record anything for
567 /// that IV, but it's error-prone to expect callers of this routine to care
568 /// about that, hence this explicit parameter.
569 void recordVectorLoopValueForInductionCast(const InductionDescriptor &ID,
570 const Instruction *EntryVal,
571 Value *VectorLoopValue,
572 unsigned Part,
573 unsigned Lane = UINT_MAX(2147483647 *2U +1U));
574
575 /// Generate a shuffle sequence that will reverse the vector Vec.
576 virtual Value *reverseVector(Value *Vec);
577
578 /// Returns (and creates if needed) the original loop trip count.
579 Value *getOrCreateTripCount(Loop *NewLoop);
580
581 /// Returns (and creates if needed) the trip count of the widened loop.
582 Value *getOrCreateVectorTripCount(Loop *NewLoop);
583
584 /// Returns a bitcasted value to the requested vector type.
585 /// Also handles bitcasts of vector<float> <-> vector<pointer> types.
586 Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
587 const DataLayout &DL);
588
589 /// Emit a bypass check to see if the vector trip count is zero, including if
590 /// it overflows.
591 void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);
592
593 /// Emit a bypass check to see if all of the SCEV assumptions we've
594 /// had to make are correct.
595 void emitSCEVChecks(Loop *L, BasicBlock *Bypass);
596
597 /// Emit bypass checks to check any memory assumptions we may have made.
598 void emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);
599
600 /// Compute the transformed value of Index at offset StartValue using step
601 /// StepValue.
602 /// For integer induction, returns StartValue + Index * StepValue.
603 /// For pointer induction, returns StartValue[Index * StepValue].
604 /// FIXME: The newly created binary instructions should contain nsw/nuw
605 /// flags, which can be found from the original scalar operations.
606 Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
607 const DataLayout &DL,
608 const InductionDescriptor &ID) const;
609
610 /// Add additional metadata to \p To that was not present on \p Orig.
611 ///
612 /// Currently this is used to add the noalias annotations based on the
613 /// inserted memchecks. Use this for instructions that are *cloned* into the
614 /// vector loop.
615 void addNewMetadata(Instruction *To, const Instruction *Orig);
616
617 /// Add metadata from one instruction to another.
618 ///
619 /// This includes both the original MDs from \p From and additional ones (\see
620 /// addNewMetadata). Use this for *newly created* instructions in the vector
621 /// loop.
622 void addMetadata(Instruction *To, Instruction *From);
623
624 /// Similar to the previous function but it adds the metadata to a
625 /// vector of instructions.
626 void addMetadata(ArrayRef<Value *> To, Instruction *From);
627
628 /// The original loop.
629 Loop *OrigLoop;
630
631 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
632 /// dynamic knowledge to simplify SCEV expressions and converts them to a
633 /// more usable form.
634 PredicatedScalarEvolution &PSE;
635
636 /// Loop Info.
637 LoopInfo *LI;
638
639 /// Dominator Tree.
640 DominatorTree *DT;
641
642 /// Alias Analysis.
643 AliasAnalysis *AA;
644
645 /// Target Library Info.
646 const TargetLibraryInfo *TLI;
647
648 /// Target Transform Info.
649 const TargetTransformInfo *TTI;
650
651 /// Assumption Cache.
652 AssumptionCache *AC;
653
654 /// Interface to emit optimization remarks.
655 OptimizationRemarkEmitter *ORE;
656
657 /// LoopVersioning. It's only set up (non-null) if memchecks were
658 /// used.
659 ///
660 /// This is currently only used to add no-alias metadata based on the
661 /// memchecks. The actually versioning is performed manually.
662 std::unique_ptr<LoopVersioning> LVer;
663
664 /// The vectorization SIMD factor to use. Each vector will have this many
665 /// vector elements.
666 unsigned VF;
667
668 /// The vectorization unroll factor to use. Each scalar is vectorized to this
669 /// many different vector instructions.
670 unsigned UF;
671
672 /// The builder that we use
673 IRBuilder<> Builder;
674
675 // --- Vectorization state ---
676
677 /// The vector-loop preheader.
678 BasicBlock *LoopVectorPreHeader;
679
680 /// The scalar-loop preheader.
681 BasicBlock *LoopScalarPreHeader;
682
683 /// Middle Block between the vector and the scalar.
684 BasicBlock *LoopMiddleBlock;
685
686 /// The ExitBlock of the scalar loop.
687 BasicBlock *LoopExitBlock;
688
689 /// The vector loop body.
690 BasicBlock *LoopVectorBody;
691
692 /// The scalar loop body.
693 BasicBlock *LoopScalarBody;
694
695 /// A list of all bypass blocks. The first block is the entry of the loop.
696 SmallVector<BasicBlock *, 4> LoopBypassBlocks;
697
698 /// The new Induction variable which was added to the new block.
699 PHINode *Induction = nullptr;
700
701 /// The induction variable of the old basic block.
702 PHINode *OldInduction = nullptr;
703
704 /// Maps values from the original loop to their corresponding values in the
705 /// vectorized loop. A key value can map to either vector values, scalar
706 /// values or both kinds of values, depending on whether the key was
707 /// vectorized and scalarized.
708 VectorizerValueMap VectorLoopValueMap;
709
710 /// Store instructions that were predicated.
711 SmallVector<Instruction *, 4> PredicatedInstructions;
712
713 /// Trip count of the original loop.
714 Value *TripCount = nullptr;
715
716 /// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
717 Value *VectorTripCount = nullptr;
718
719 /// The legality analysis.
720 LoopVectorizationLegality *Legal;
721
722 /// The profitablity analysis.
723 LoopVectorizationCostModel *Cost;
724
725 // Record whether runtime checks are added.
726 bool AddedSafetyChecks = false;
727
728 // Holds the end values for each induction variable. We save the end values
729 // so we can later fix-up the external users of the induction variables.
730 DenseMap<PHINode *, Value *> IVEndValues;
731
732 // Vector of original scalar PHIs whose corresponding widened PHIs need to be
733 // fixed up at the end of vector code generation.
734 SmallVector<PHINode *, 8> OrigPHIsToFix;
735};
736
737class InnerLoopUnroller : public InnerLoopVectorizer {
738public:
739 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
740 LoopInfo *LI, DominatorTree *DT,
741 const TargetLibraryInfo *TLI,
742 const TargetTransformInfo *TTI, AssumptionCache *AC,
743 OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
744 LoopVectorizationLegality *LVL,
745 LoopVectorizationCostModel *CM)
746 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE, 1,
747 UnrollFactor, LVL, CM) {}
748
749private:
750 Value *getBroadcastInstrs(Value *V) override;
751 Value *getStepVector(Value *Val, int StartIdx, Value *Step,
752 Instruction::BinaryOps Opcode =
753 Instruction::BinaryOpsEnd) override;
754 Value *reverseVector(Value *Vec) override;
755};
756
757} // end namespace llvm
758
759/// Look for a meaningful debug location on the instruction or it's
760/// operands.
761static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
762 if (!I)
763 return I;
764
765 DebugLoc Empty;
766 if (I->getDebugLoc() != Empty)
767 return I;
768
769 for (User::op_iterator OI = I->op_begin(), OE = I->op_end(); OI != OE; ++OI) {
770 if (Instruction *OpInst = dyn_cast<Instruction>(*OI))
771 if (OpInst->getDebugLoc() != Empty)
772 return OpInst;
773 }
774
775 return I;
776}
777
778void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) {
779 if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) {
780 const DILocation *DIL = Inst->getDebugLoc();
781 if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
782 !isa<DbgInfoIntrinsic>(Inst)) {
783 auto NewDIL = DIL->cloneByMultiplyingDuplicationFactor(UF * VF);
784 if (NewDIL)
785 B.SetCurrentDebugLocation(NewDIL.getValue());
786 else
787 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
788 << "Failed to create new discriminator: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
789 << DIL->getFilename() << " Line: " << DIL->getLine())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
;
790 }
791 else
792 B.SetCurrentDebugLocation(DIL);
793 } else
794 B.SetCurrentDebugLocation(DebugLoc());
795}
796
797#ifndef NDEBUG
798/// \return string containing a file name and a line # for the given loop.
799static std::string getDebugLocString(const Loop *L) {
800 std::string Result;
801 if (L) {
802 raw_string_ostream OS(Result);
803 if (const DebugLoc LoopDbgLoc = L->getStartLoc())
804 LoopDbgLoc.print(OS);
805 else
806 // Just print the module name.
807 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
808 OS.flush();
809 }
810 return Result;
811}
812#endif
813
814void InnerLoopVectorizer::addNewMetadata(Instruction *To,
815 const Instruction *Orig) {
816 // If the loop was versioned with memchecks, add the corresponding no-alias
817 // metadata.
818 if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
819 LVer->annotateInstWithNoAlias(To, Orig);
820}
821
822void InnerLoopVectorizer::addMetadata(Instruction *To,
823 Instruction *From) {
824 propagateMetadata(To, From);
825 addNewMetadata(To, From);
826}
827
828void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,
829 Instruction *From) {
830 for (Value *V : To) {
831 if (Instruction *I = dyn_cast<Instruction>(V))
832 addMetadata(I, From);
833 }
834}
835
836namespace llvm {
837
838/// LoopVectorizationCostModel - estimates the expected speedups due to
839/// vectorization.
840/// In many cases vectorization is not profitable. This can happen because of
841/// a number of reasons. In this class we mainly attempt to predict the
842/// expected speedup/slowdowns due to the supported instruction set. We use the
843/// TargetTransformInfo to query the different backends for the cost of
844/// different operations.
845class LoopVectorizationCostModel {
846public:
847 LoopVectorizationCostModel(Loop *L, PredicatedScalarEvolution &PSE,
848 LoopInfo *LI, LoopVectorizationLegality *Legal,
849 const TargetTransformInfo &TTI,
850 const TargetLibraryInfo *TLI, DemandedBits *DB,
851 AssumptionCache *AC,
852 OptimizationRemarkEmitter *ORE, const Function *F,
853 const LoopVectorizeHints *Hints,
854 InterleavedAccessInfo &IAI)
855 : TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB),
856 AC(AC), ORE(ORE), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {}
857
858 /// \return An upper bound for the vectorization factor, or None if
859 /// vectorization and interleaving should be avoided up front.
860 Optional<unsigned> computeMaxVF(bool OptForSize);
861
862 /// \return The most profitable vectorization factor and the cost of that VF.
863 /// This method checks every power of two up to MaxVF. If UserVF is not ZERO
864 /// then this vectorization factor will be selected if vectorization is
865 /// possible.
866 VectorizationFactor selectVectorizationFactor(unsigned MaxVF);
867
868 /// Setup cost-based decisions for user vectorization factor.
869 void selectUserVectorizationFactor(unsigned UserVF) {
870 collectUniformsAndScalars(UserVF);
871 collectInstsToScalarize(UserVF);
872 }
873
874 /// \return The size (in bits) of the smallest and widest types in the code
875 /// that needs to be vectorized. We ignore values that remain scalar such as
876 /// 64 bit loop indices.
877 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
878
879 /// \return The desired interleave count.
880 /// If interleave count has been specified by metadata it will be returned.
881 /// Otherwise, the interleave count is computed and returned. VF and LoopCost
882 /// are the selected vectorization factor and the cost of the selected VF.
883 unsigned selectInterleaveCount(bool OptForSize, unsigned VF,
884 unsigned LoopCost);
885
886 /// Memory access instruction may be vectorized in more than one way.
887 /// Form of instruction after vectorization depends on cost.
888 /// This function takes cost-based decisions for Load/Store instructions
889 /// and collects them in a map. This decisions map is used for building
890 /// the lists of loop-uniform and loop-scalar instructions.
891 /// The calculated cost is saved with widening decision in order to
892 /// avoid redundant calculations.
893 void setCostBasedWideningDecision(unsigned VF);
894
895 /// A struct that represents some properties of the register usage
896 /// of a loop.
897 struct RegisterUsage {
898 /// Holds the number of loop invariant values that are used in the loop.
899 unsigned LoopInvariantRegs;
900
901 /// Holds the maximum number of concurrent live intervals in the loop.
902 unsigned MaxLocalUsers;
903 };
904
905 /// \return Returns information about the register usages of the loop for the
906 /// given vectorization factors.
907 SmallVector<RegisterUsage, 8> calculateRegisterUsage(ArrayRef<unsigned> VFs);
908
909 /// Collect values we want to ignore in the cost model.
910 void collectValuesToIgnore();
911
912 /// \returns The smallest bitwidth each instruction can be represented with.
913 /// The vector equivalents of these instructions should be truncated to this
914 /// type.
915 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
916 return MinBWs;
917 }
918
919 /// \returns True if it is more profitable to scalarize instruction \p I for
920 /// vectorization factor \p VF.
921 bool isProfitableToScalarize(Instruction *I, unsigned VF) const {
922 assert(VF > 1 && "Profitable to scalarize relevant only for VF > 1.")((VF > 1 && "Profitable to scalarize relevant only for VF > 1."
) ? static_cast<void> (0) : __assert_fail ("VF > 1 && \"Profitable to scalarize relevant only for VF > 1.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 922, __PRETTY_FUNCTION__))
;
923
924 // Cost model is not run in the VPlan-native path - return conservative
925 // result until this changes.
926 if (EnableVPlanNativePath)
927 return false;
928
929 auto Scalars = InstsToScalarize.find(VF);
930 assert(Scalars != InstsToScalarize.end() &&((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 931, __PRETTY_FUNCTION__))
931 "VF not yet analyzed for scalarization profitability")((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 931, __PRETTY_FUNCTION__))
;
932 return Scalars->second.find(I) != Scalars->second.end();
933 }
934
935 /// Returns true if \p I is known to be uniform after vectorization.
936 bool isUniformAfterVectorization(Instruction *I, unsigned VF) const {
937 if (VF == 1)
938 return true;
939
940 // Cost model is not run in the VPlan-native path - return conservative
941 // result until this changes.
942 if (EnableVPlanNativePath)
943 return false;
944
945 auto UniformsPerVF = Uniforms.find(VF);
946 assert(UniformsPerVF != Uniforms.end() &&((UniformsPerVF != Uniforms.end() && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 947, __PRETTY_FUNCTION__))
947 "VF not yet analyzed for uniformity")((UniformsPerVF != Uniforms.end() && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 947, __PRETTY_FUNCTION__))
;
948 return UniformsPerVF->second.find(I) != UniformsPerVF->second.end();
949 }
950
951 /// Returns true if \p I is known to be scalar after vectorization.
952 bool isScalarAfterVectorization(Instruction *I, unsigned VF) const {
953 if (VF == 1)
954 return true;
955
956 // Cost model is not run in the VPlan-native path - return conservative
957 // result until this changes.
958 if (EnableVPlanNativePath)
959 return false;
960
961 auto ScalarsPerVF = Scalars.find(VF);
962 assert(ScalarsPerVF != Scalars.end() &&((ScalarsPerVF != Scalars.end() && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 963, __PRETTY_FUNCTION__))
963 "Scalar values are not calculated for VF")((ScalarsPerVF != Scalars.end() && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 963, __PRETTY_FUNCTION__))
;
964 return ScalarsPerVF->second.find(I) != ScalarsPerVF->second.end();
965 }
966
967 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
968 /// for vectorization factor \p VF.
969 bool canTruncateToMinimalBitwidth(Instruction *I, unsigned VF) const {
970 return VF > 1 && MinBWs.find(I) != MinBWs.end() &&
971 !isProfitableToScalarize(I, VF) &&
972 !isScalarAfterVectorization(I, VF);
973 }
974
975 /// Decision that was taken during cost calculation for memory instruction.
976 enum InstWidening {
977 CM_Unknown,
978 CM_Widen, // For consecutive accesses with stride +1.
979 CM_Widen_Reverse, // For consecutive accesses with stride -1.
980 CM_Interleave,
981 CM_GatherScatter,
982 CM_Scalarize
983 };
984
985 /// Save vectorization decision \p W and \p Cost taken by the cost model for
986 /// instruction \p I and vector width \p VF.
987 void setWideningDecision(Instruction *I, unsigned VF, InstWidening W,
988 unsigned Cost) {
989 assert(VF >= 2 && "Expected VF >=2")((VF >= 2 && "Expected VF >=2") ? static_cast<
void> (0) : __assert_fail ("VF >= 2 && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 989, __PRETTY_FUNCTION__))
;
990 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
991 }
992
993 /// Save vectorization decision \p W and \p Cost taken by the cost model for
994 /// interleaving group \p Grp and vector width \p VF.
995 void setWideningDecision(const InterleaveGroup<Instruction> *Grp, unsigned VF,
996 InstWidening W, unsigned Cost) {
997 assert(VF >= 2 && "Expected VF >=2")((VF >= 2 && "Expected VF >=2") ? static_cast<
void> (0) : __assert_fail ("VF >= 2 && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 997, __PRETTY_FUNCTION__))
;
998 /// Broadcast this decicion to all instructions inside the group.
999 /// But the cost will be assigned to one instruction only.
1000 for (unsigned i = 0; i < Grp->getFactor(); ++i) {
1001 if (auto *I = Grp->getMember(i)) {
1002 if (Grp->getInsertPos() == I)
1003 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1004 else
1005 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
1006 }
1007 }
1008 }
1009
1010 /// Return the cost model decision for the given instruction \p I and vector
1011 /// width \p VF. Return CM_Unknown if this instruction did not pass
1012 /// through the cost modeling.
1013 InstWidening getWideningDecision(Instruction *I, unsigned VF) {
1014 assert(VF >= 2 && "Expected VF >=2")((VF >= 2 && "Expected VF >=2") ? static_cast<
void> (0) : __assert_fail ("VF >= 2 && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1014, __PRETTY_FUNCTION__))
;
1015
1016 // Cost model is not run in the VPlan-native path - return conservative
1017 // result until this changes.
1018 if (EnableVPlanNativePath)
1019 return CM_GatherScatter;
1020
1021 std::pair<Instruction *, unsigned> InstOnVF = std::make_pair(I, VF);
1022 auto Itr = WideningDecisions.find(InstOnVF);
1023 if (Itr == WideningDecisions.end())
1024 return CM_Unknown;
1025 return Itr->second.first;
1026 }
1027
1028 /// Return the vectorization cost for the given instruction \p I and vector
1029 /// width \p VF.
1030 unsigned getWideningCost(Instruction *I, unsigned VF) {
1031 assert(VF >= 2 && "Expected VF >=2")((VF >= 2 && "Expected VF >=2") ? static_cast<
void> (0) : __assert_fail ("VF >= 2 && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1031, __PRETTY_FUNCTION__))
;
1032 std::pair<Instruction *, unsigned> InstOnVF = std::make_pair(I, VF);
1033 assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() &&((WideningDecisions.find(InstOnVF) != WideningDecisions.end()
&& "The cost is not calculated") ? static_cast<void
> (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1034, __PRETTY_FUNCTION__))
1034 "The cost is not calculated")((WideningDecisions.find(InstOnVF) != WideningDecisions.end()
&& "The cost is not calculated") ? static_cast<void
> (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1034, __PRETTY_FUNCTION__))
;
1035 return WideningDecisions[InstOnVF].second;
1036 }
1037
1038 /// Return True if instruction \p I is an optimizable truncate whose operand
1039 /// is an induction variable. Such a truncate will be removed by adding a new
1040 /// induction variable with the destination type.
1041 bool isOptimizableIVTruncate(Instruction *I, unsigned VF) {
1042 // If the instruction is not a truncate, return false.
1043 auto *Trunc = dyn_cast<TruncInst>(I);
1044 if (!Trunc)
1045 return false;
1046
1047 // Get the source and destination types of the truncate.
1048 Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);
1049 Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);
1050
1051 // If the truncate is free for the given types, return false. Replacing a
1052 // free truncate with an induction variable would add an induction variable
1053 // update instruction to each iteration of the loop. We exclude from this
1054 // check the primary induction variable since it will need an update
1055 // instruction regardless.
1056 Value *Op = Trunc->getOperand(0);
1057 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1058 return false;
1059
1060 // If the truncated value is not an induction variable, return false.
1061 return Legal->isInductionPhi(Op);
1062 }
1063
1064 /// Collects the instructions to scalarize for each predicated instruction in
1065 /// the loop.
1066 void collectInstsToScalarize(unsigned VF);
1067
1068 /// Collect Uniform and Scalar values for the given \p VF.
1069 /// The sets depend on CM decision for Load/Store instructions
1070 /// that may be vectorized as interleave, gather-scatter or scalarized.
1071 void collectUniformsAndScalars(unsigned VF) {
1072 // Do the analysis once.
1073 if (VF == 1 || Uniforms.find(VF) != Uniforms.end())
1074 return;
1075 setCostBasedWideningDecision(VF);
1076 collectLoopUniforms(VF);
1077 collectLoopScalars(VF);
1078 }
1079
1080 /// Returns true if the target machine supports masked store operation
1081 /// for the given \p DataType and kind of access to \p Ptr.
1082 bool isLegalMaskedStore(Type *DataType, Value *Ptr) {
1083 return Legal->isConsecutivePtr(Ptr) && TTI.isLegalMaskedStore(DataType);
1084 }
1085
1086 /// Returns true if the target machine supports masked load operation
1087 /// for the given \p DataType and kind of access to \p Ptr.
1088 bool isLegalMaskedLoad(Type *DataType, Value *Ptr) {
1089 return Legal->isConsecutivePtr(Ptr) && TTI.isLegalMaskedLoad(DataType);
1090 }
1091
1092 /// Returns true if the target machine supports masked scatter operation
1093 /// for the given \p DataType.
1094 bool isLegalMaskedScatter(Type *DataType) {
1095 return TTI.isLegalMaskedScatter(DataType);
1096 }
1097
1098 /// Returns true if the target machine supports masked gather operation
1099 /// for the given \p DataType.
1100 bool isLegalMaskedGather(Type *DataType) {
1101 return TTI.isLegalMaskedGather(DataType);
1102 }
1103
1104 /// Returns true if the target machine can represent \p V as a masked gather
1105 /// or scatter operation.
1106 bool isLegalGatherOrScatter(Value *V) {
1107 bool LI = isa<LoadInst>(V);
1108 bool SI = isa<StoreInst>(V);
1109 if (!LI && !SI)
1110 return false;
1111 auto *Ty = getMemInstValueType(V);
1112 return (LI && isLegalMaskedGather(Ty)) || (SI && isLegalMaskedScatter(Ty));
1113 }
1114
1115 /// Returns true if \p I is an instruction that will be scalarized with
1116 /// predication. Such instructions include conditional stores and
1117 /// instructions that may divide by zero.
1118 /// If a non-zero VF has been calculated, we check if I will be scalarized
1119 /// predication for that VF.
1120 bool isScalarWithPredication(Instruction *I, unsigned VF = 1);
1121
1122 // Returns true if \p I is an instruction that will be predicated either
1123 // through scalar predication or masked load/store or masked gather/scatter.
1124 // Superset of instructions that return true for isScalarWithPredication.
1125 bool isPredicatedInst(Instruction *I) {
1126 if (!blockNeedsPredication(I->getParent()))
1127 return false;
1128 // Loads and stores that need some form of masked operation are predicated
1129 // instructions.
1130 if (isa<LoadInst>(I) || isa<StoreInst>(I))
1131 return Legal->isMaskRequired(I);
1132 return isScalarWithPredication(I);
1133 }
1134
1135 /// Returns true if \p I is a memory instruction with consecutive memory
1136 /// access that can be widened.
1137 bool memoryInstructionCanBeWidened(Instruction *I, unsigned VF = 1);
1138
1139 /// Returns true if \p I is a memory instruction in an interleaved-group
1140 /// of memory accesses that can be vectorized with wide vector loads/stores
1141 /// and shuffles.
1142 bool interleavedAccessCanBeWidened(Instruction *I, unsigned VF = 1);
1143
1144 /// Check if \p Instr belongs to any interleaved access group.
1145 bool isAccessInterleaved(Instruction *Instr) {
1146 return InterleaveInfo.isInterleaved(Instr);
1147 }
1148
1149 /// Get the interleaved access group that \p Instr belongs to.
1150 const InterleaveGroup<Instruction> *
1151 getInterleavedAccessGroup(Instruction *Instr) {
1152 return InterleaveInfo.getInterleaveGroup(Instr);
1153 }
1154
1155 /// Returns true if an interleaved group requires a scalar iteration
1156 /// to handle accesses with gaps, and there is nothing preventing us from
1157 /// creating a scalar epilogue.
1158 bool requiresScalarEpilogue() const {
1159 return IsScalarEpilogueAllowed && InterleaveInfo.requiresScalarEpilogue();
1160 }
1161
1162 /// Returns true if a scalar epilogue is not allowed due to optsize.
1163 bool isScalarEpilogueAllowed() const { return IsScalarEpilogueAllowed; }
1164
1165 /// Returns true if all loop blocks should be masked to fold tail loop.
1166 bool foldTailByMasking() const { return FoldTailByMasking; }
1167
1168 bool blockNeedsPredication(BasicBlock *BB) {
1169 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1170 }
1171
1172private:
1173 unsigned NumPredStores = 0;
1174
1175 /// \return An upper bound for the vectorization factor, larger than zero.
1176 /// One is returned if vectorization should best be avoided due to cost.
1177 unsigned computeFeasibleMaxVF(bool OptForSize, unsigned ConstTripCount);
1178
1179 /// The vectorization cost is a combination of the cost itself and a boolean
1180 /// indicating whether any of the contributing operations will actually
1181 /// operate on
1182 /// vector values after type legalization in the backend. If this latter value
1183 /// is
1184 /// false, then all operations will be scalarized (i.e. no vectorization has
1185 /// actually taken place).
1186 using VectorizationCostTy = std::pair<unsigned, bool>;
1187
1188 /// Returns the expected execution cost. The unit of the cost does
1189 /// not matter because we use the 'cost' units to compare different
1190 /// vector widths. The cost that is returned is *not* normalized by
1191 /// the factor width.
1192 VectorizationCostTy expectedCost(unsigned VF);
1193
1194 /// Returns the execution time cost of an instruction for a given vector
1195 /// width. Vector width of one means scalar.
1196 VectorizationCostTy getInstructionCost(Instruction *I, unsigned VF);
1197
1198 /// The cost-computation logic from getInstructionCost which provides
1199 /// the vector type as an output parameter.
1200 unsigned getInstructionCost(Instruction *I, unsigned VF, Type *&VectorTy);
1201
1202 /// Calculate vectorization cost of memory instruction \p I.
1203 unsigned getMemoryInstructionCost(Instruction *I, unsigned VF);
1204
1205 /// The cost computation for scalarized memory instruction.
1206 unsigned getMemInstScalarizationCost(Instruction *I, unsigned VF);
1207
1208 /// The cost computation for interleaving group of memory instructions.
1209 unsigned getInterleaveGroupCost(Instruction *I, unsigned VF);
1210
1211 /// The cost computation for Gather/Scatter instruction.
1212 unsigned getGatherScatterCost(Instruction *I, unsigned VF);
1213
1214 /// The cost computation for widening instruction \p I with consecutive
1215 /// memory access.
1216 unsigned getConsecutiveMemOpCost(Instruction *I, unsigned VF);
1217
1218 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1219 /// Load: scalar load + broadcast.
1220 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1221 /// element)
1222 unsigned getUniformMemOpCost(Instruction *I, unsigned VF);
1223
1224 /// Returns whether the instruction is a load or store and will be a emitted
1225 /// as a vector operation.
1226 bool isConsecutiveLoadOrStore(Instruction *I);
1227
1228 /// Returns true if an artificially high cost for emulated masked memrefs
1229 /// should be used.
1230 bool useEmulatedMaskMemRefHack(Instruction *I);
1231
1232 /// Create an analysis remark that explains why vectorization failed
1233 ///
1234 /// \p RemarkName is the identifier for the remark. \return the remark object
1235 /// that can be streamed to.
1236 OptimizationRemarkAnalysis createMissedAnalysis(StringRef RemarkName) {
1237 return createLVMissedAnalysis(Hints->vectorizeAnalysisPassName(),
1238 RemarkName, TheLoop);
1239 }
1240
1241 /// Map of scalar integer values to the smallest bitwidth they can be legally
1242 /// represented as. The vector equivalents of these values should be truncated
1243 /// to this type.
1244 MapVector<Instruction *, uint64_t> MinBWs;
1245
1246 /// A type representing the costs for instructions if they were to be
1247 /// scalarized rather than vectorized. The entries are Instruction-Cost
1248 /// pairs.
1249 using ScalarCostsTy = DenseMap<Instruction *, unsigned>;
1250
1251 /// A set containing all BasicBlocks that are known to present after
1252 /// vectorization as a predicated block.
1253 SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;
1254
1255 /// Records whether it is allowed to have the original scalar loop execute at
1256 /// least once. This may be needed as a fallback loop in case runtime
1257 /// aliasing/dependence checks fail, or to handle the tail/remainder
1258 /// iterations when the trip count is unknown or doesn't divide by the VF,
1259 /// or as a peel-loop to handle gaps in interleave-groups.
1260 /// Under optsize and when the trip count is very small we don't allow any
1261 /// iterations to execute in the scalar loop.
1262 bool IsScalarEpilogueAllowed = true;
1263
1264 /// All blocks of loop are to be masked to fold tail of scalar iterations.
1265 bool FoldTailByMasking = false;
1266
1267 /// A map holding scalar costs for different vectorization factors. The
1268 /// presence of a cost for an instruction in the mapping indicates that the
1269 /// instruction will be scalarized when vectorizing with the associated
1270 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1271 DenseMap<unsigned, ScalarCostsTy> InstsToScalarize;
1272
1273 /// Holds the instructions known to be uniform after vectorization.
1274 /// The data is collected per VF.
1275 DenseMap<unsigned, SmallPtrSet<Instruction *, 4>> Uniforms;
1276
1277 /// Holds the instructions known to be scalar after vectorization.
1278 /// The data is collected per VF.
1279 DenseMap<unsigned, SmallPtrSet<Instruction *, 4>> Scalars;
1280
1281 /// Holds the instructions (address computations) that are forced to be
1282 /// scalarized.
1283 DenseMap<unsigned, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1284
1285 /// Returns the expected difference in cost from scalarizing the expression
1286 /// feeding a predicated instruction \p PredInst. The instructions to
1287 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1288 /// non-negative return value implies the expression will be scalarized.
1289 /// Currently, only single-use chains are considered for scalarization.
1290 int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,
1291 unsigned VF);
1292
1293 /// Collect the instructions that are uniform after vectorization. An
1294 /// instruction is uniform if we represent it with a single scalar value in
1295 /// the vectorized loop corresponding to each vector iteration. Examples of
1296 /// uniform instructions include pointer operands of consecutive or
1297 /// interleaved memory accesses. Note that although uniformity implies an
1298 /// instruction will be scalar, the reverse is not true. In general, a
1299 /// scalarized instruction will be represented by VF scalar values in the
1300 /// vectorized loop, each corresponding to an iteration of the original
1301 /// scalar loop.
1302 void collectLoopUniforms(unsigned VF);
1303
1304 /// Collect the instructions that are scalar after vectorization. An
1305 /// instruction is scalar if it is known to be uniform or will be scalarized
1306 /// during vectorization. Non-uniform scalarized instructions will be
1307 /// represented by VF values in the vectorized loop, each corresponding to an
1308 /// iteration of the original scalar loop.
1309 void collectLoopScalars(unsigned VF);
1310
1311 /// Keeps cost model vectorization decision and cost for instructions.
1312 /// Right now it is used for memory instructions only.
1313 using DecisionList = DenseMap<std::pair<Instruction *, unsigned>,
1314 std::pair<InstWidening, unsigned>>;
1315
1316 DecisionList WideningDecisions;
1317
1318public:
1319 /// The loop that we evaluate.
1320 Loop *TheLoop;
1321
1322 /// Predicated scalar evolution analysis.
1323 PredicatedScalarEvolution &PSE;
1324
1325 /// Loop Info analysis.
1326 LoopInfo *LI;
1327
1328 /// Vectorization legality.
1329 LoopVectorizationLegality *Legal;
1330
1331 /// Vector target information.
1332 const TargetTransformInfo &TTI;
1333
1334 /// Target Library Info.
1335 const TargetLibraryInfo *TLI;
1336
1337 /// Demanded bits analysis.
1338 DemandedBits *DB;
1339
1340 /// Assumption cache.
1341 AssumptionCache *AC;
1342
1343 /// Interface to emit optimization remarks.
1344 OptimizationRemarkEmitter *ORE;
1345
1346 const Function *TheFunction;
1347
1348 /// Loop Vectorize Hint.
1349 const LoopVectorizeHints *Hints;
1350
1351 /// The interleave access information contains groups of interleaved accesses
1352 /// with the same stride and close to each other.
1353 InterleavedAccessInfo &InterleaveInfo;
1354
1355 /// Values to ignore in the cost model.
1356 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1357
1358 /// Values to ignore in the cost model when VF > 1.
1359 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1360};
1361
1362} // end namespace llvm
1363
1364// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1365// vectorization. The loop needs to be annotated with #pragma omp simd
1366// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1367// vector length information is not provided, vectorization is not considered
1368// explicit. Interleave hints are not allowed either. These limitations will be
1369// relaxed in the future.
1370// Please, note that we are currently forced to abuse the pragma 'clang
1371// vectorize' semantics. This pragma provides *auto-vectorization hints*
1372// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1373// provides *explicit vectorization hints* (LV can bypass legal checks and
1374// assume that vectorization is legal). However, both hints are implemented
1375// using the same metadata (llvm.loop.vectorize, processed by
1376// LoopVectorizeHints). This will be fixed in the future when the native IR
1377// representation for pragma 'omp simd' is introduced.
1378static bool isExplicitVecOuterLoop(Loop *OuterLp,
1379 OptimizationRemarkEmitter *ORE) {
1380 assert(!OuterLp->empty() && "This is not an outer loop")((!OuterLp->empty() && "This is not an outer loop"
) ? static_cast<void> (0) : __assert_fail ("!OuterLp->empty() && \"This is not an outer loop\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1380, __PRETTY_FUNCTION__))
;
1381 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
1382
1383 // Only outer loops with an explicit vectorization hint are supported.
1384 // Unannotated outer loops are ignored.
1385 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1386 return false;
1387
1388 Function *Fn = OuterLp->getHeader()->getParent();
1389 if (!Hints.allowVectorization(Fn, OuterLp,
1390 true /*VectorizeOnlyWhenForced*/)) {
1391 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints prevent outer loop vectorization.\n"
; } } while (false)
;
1392 return false;
1393 }
1394
1395 if (Hints.getInterleave() > 1) {
1396 // TODO: Interleave support is future work.
1397 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n"; } } while (false)
1398 "outer loops.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n"; } } while (false)
;
1399 Hints.emitRemarkWithHints();
1400 return false;
1401 }
1402
1403 return true;
1404}
1405
1406static void collectSupportedLoops(Loop &L, LoopInfo *LI,
1407 OptimizationRemarkEmitter *ORE,
1408 SmallVectorImpl<Loop *> &V) {
1409 // Collect inner loops and outer loops without irreducible control flow. For
1410 // now, only collect outer loops that have explicit vectorization hints. If we
1411 // are stress testing the VPlan H-CFG construction, we collect the outermost
1412 // loop of every loop nest.
1413 if (L.empty() || VPlanBuildStressTest ||
1414 (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
1415 LoopBlocksRPO RPOT(&L);
1416 RPOT.perform(LI);
1417 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
1418 V.push_back(&L);
1419 // TODO: Collect inner loops inside marked outer loops in case
1420 // vectorization fails for the outer loop. Do not invoke
1421 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1422 // already known to be reducible. We can use an inherited attribute for
1423 // that.
1424 return;
1425 }
1426 }
1427 for (Loop *InnerL : L)
1428 collectSupportedLoops(*InnerL, LI, ORE, V);
1429}
1430
1431namespace {
1432
1433/// The LoopVectorize Pass.
1434struct LoopVectorize : public FunctionPass {
1435 /// Pass identification, replacement for typeid
1436 static char ID;
1437
1438 LoopVectorizePass Impl;
1439
1440 explicit LoopVectorize(bool InterleaveOnlyWhenForced = false,
1441 bool VectorizeOnlyWhenForced = false)
1442 : FunctionPass(ID) {
1443 Impl.InterleaveOnlyWhenForced = InterleaveOnlyWhenForced;
1444 Impl.VectorizeOnlyWhenForced = VectorizeOnlyWhenForced;
1445 initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
1446 }
1447
1448 bool runOnFunction(Function &F) override {
1449 if (skipFunction(F))
1450 return false;
1451
1452 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1453 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1454 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1455 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1456 auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
1457 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
1458 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
1459 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1460 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1461 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
1462 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
1463 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
1464 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1465
1466 std::function<const LoopAccessInfo &(Loop &)> GetLAA =
1467 [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };
1468
1469 return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,
1470 GetLAA, *ORE, PSI);
1471 }
1472
1473 void getAnalysisUsage(AnalysisUsage &AU) const override {
1474 AU.addRequired<AssumptionCacheTracker>();
1475 AU.addRequired<BlockFrequencyInfoWrapperPass>();
1476 AU.addRequired<DominatorTreeWrapperPass>();
1477 AU.addRequired<LoopInfoWrapperPass>();
1478 AU.addRequired<ScalarEvolutionWrapperPass>();
1479 AU.addRequired<TargetTransformInfoWrapperPass>();
1480 AU.addRequired<AAResultsWrapperPass>();
1481 AU.addRequired<LoopAccessLegacyAnalysis>();
1482 AU.addRequired<DemandedBitsWrapperPass>();
1483 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
1484
1485 // We currently do not preserve loopinfo/dominator analyses with outer loop
1486 // vectorization. Until this is addressed, mark these analyses as preserved
1487 // only for non-VPlan-native path.
1488 // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
1489 if (!EnableVPlanNativePath) {
1490 AU.addPreserved<LoopInfoWrapperPass>();
1491 AU.addPreserved<DominatorTreeWrapperPass>();
1492 }
1493
1494 AU.addPreserved<BasicAAWrapperPass>();
1495 AU.addPreserved<GlobalsAAWrapperPass>();
1496 AU.addRequired<ProfileSummaryInfoWrapperPass>();
1497 }
1498};
1499
1500} // end anonymous namespace
1501
1502//===----------------------------------------------------------------------===//
1503// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
1504// LoopVectorizationCostModel and LoopVectorizationPlanner.
1505//===----------------------------------------------------------------------===//
1506
1507Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
1508 // We need to place the broadcast of invariant variables outside the loop,
1509 // but only if it's proven safe to do so. Else, broadcast will be inside
1510 // vector loop body.
1511 Instruction *Instr = dyn_cast<Instruction>(V);
1512 bool SafeToHoist = OrigLoop->isLoopInvariant(V) &&
1513 (!Instr ||
1514 DT->dominates(Instr->getParent(), LoopVectorPreHeader));
1515 // Place the code for broadcasting invariant variables in the new preheader.
1516 IRBuilder<>::InsertPointGuard Guard(Builder);
1517 if (SafeToHoist)
1518 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
1519
1520 // Broadcast the scalar into all locations in the vector.
1521 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
1522
1523 return Shuf;
1524}
1525
1526void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
1527 const InductionDescriptor &II, Value *Step, Instruction *EntryVal) {
1528 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1529, __PRETTY_FUNCTION__))
1529 "Expected either an induction phi-node or a truncate of it!")(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1529, __PRETTY_FUNCTION__))
;
1530 Value *Start = II.getStartValue();
1531
1532 // Construct the initial value of the vector IV in the vector loop preheader
1533 auto CurrIP = Builder.saveIP();
1534 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
1535 if (isa<TruncInst>(EntryVal)) {
1536 assert(Start->getType()->isIntegerTy() &&((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1537, __PRETTY_FUNCTION__))
1537 "Truncation requires an integer type")((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1537, __PRETTY_FUNCTION__))
;
1538 auto *TruncType = cast<IntegerType>(EntryVal->getType());
1539 Step = Builder.CreateTrunc(Step, TruncType);
1540 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
1541 }
1542 Value *SplatStart = Builder.CreateVectorSplat(VF, Start);
1543 Value *SteppedStart =
1544 getStepVector(SplatStart, 0, Step, II.getInductionOpcode());
1545
1546 // We create vector phi nodes for both integer and floating-point induction
1547 // variables. Here, we determine the kind of arithmetic we will perform.
1548 Instruction::BinaryOps AddOp;
1549 Instruction::BinaryOps MulOp;
1550 if (Step->getType()->isIntegerTy()) {
1551 AddOp = Instruction::Add;
1552 MulOp = Instruction::Mul;
1553 } else {
1554 AddOp = II.getInductionOpcode();
1555 MulOp = Instruction::FMul;
1556 }
1557
1558 // Multiply the vectorization factor by the step using integer or
1559 // floating-point arithmetic as appropriate.
1560 Value *ConstVF = getSignedIntOrFpConstant(Step->getType(), VF);
1561 Value *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, Step, ConstVF));
1562
1563 // Create a vector splat to use in the induction update.
1564 //
1565 // FIXME: If the step is non-constant, we create the vector splat with
1566 // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
1567 // handle a constant vector splat.
1568 Value *SplatVF = isa<Constant>(Mul)
1569 ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
1570 : Builder.CreateVectorSplat(VF, Mul);
1571 Builder.restoreIP(CurrIP);
1572
1573 // We may need to add the step a number of times, depending on the unroll
1574 // factor. The last of those goes into the PHI.
1575 PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",
1576 &*LoopVectorBody->getFirstInsertionPt());
1577 VecInd->setDebugLoc(EntryVal->getDebugLoc());
1578 Instruction *LastInduction = VecInd;
1579 for (unsigned Part = 0; Part < UF; ++Part) {
1580 VectorLoopValueMap.setVectorValue(EntryVal, Part, LastInduction);
1581
1582 if (isa<TruncInst>(EntryVal))
1583 addMetadata(LastInduction, EntryVal);
1584 recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, Part);
1585
1586 LastInduction = cast<Instruction>(addFastMathFlag(
1587 Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add")));
1588 LastInduction->setDebugLoc(EntryVal->getDebugLoc());
1589 }
1590
1591 // Move the last step to the end of the latch block. This ensures consistent
1592 // placement of all induction updates.
1593 auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
1594 auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());
1595 auto *ICmp = cast<Instruction>(Br->getCondition());
1596 LastInduction->moveBefore(ICmp);
1597 LastInduction->setName("vec.ind.next");
1598
1599 VecInd->addIncoming(SteppedStart, LoopVectorPreHeader);
1600 VecInd->addIncoming(LastInduction, LoopVectorLatch);
1601}
1602
1603bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const {
1604 return Cost->isScalarAfterVectorization(I, VF) ||
1605 Cost->isProfitableToScalarize(I, VF);
1606}
1607
1608bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {
1609 if (shouldScalarizeInstruction(IV))
1610 return true;
1611 auto isScalarInst = [&](User *U) -> bool {
1612 auto *I = cast<Instruction>(U);
1613 return (OrigLoop->contains(I) && shouldScalarizeInstruction(I));
1614 };
1615 return llvm::any_of(IV->users(), isScalarInst);
1616}
1617
1618void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
1619 const InductionDescriptor &ID, const Instruction *EntryVal,
1620 Value *VectorLoopVal, unsigned Part, unsigned Lane) {
1621 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1622, __PRETTY_FUNCTION__))
1622 "Expected either an induction phi-node or a truncate of it!")(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1622, __PRETTY_FUNCTION__))
;
1623
1624 // This induction variable is not the phi from the original loop but the
1625 // newly-created IV based on the proof that casted Phi is equal to the
1626 // uncasted Phi in the vectorized loop (under a runtime guard possibly). It
1627 // re-uses the same InductionDescriptor that original IV uses but we don't
1628 // have to do any recording in this case - that is done when original IV is
1629 // processed.
1630 if (isa<TruncInst>(EntryVal))
1631 return;
1632
1633 const SmallVectorImpl<Instruction *> &Casts = ID.getCastInsts();
1634 if (Casts.empty())
1635 return;
1636 // Only the first Cast instruction in the Casts vector is of interest.
1637 // The rest of the Casts (if exist) have no uses outside the
1638 // induction update chain itself.
1639 Instruction *CastInst = *Casts.begin();
1640 if (Lane < UINT_MAX(2147483647 *2U +1U))
1641 VectorLoopValueMap.setScalarValue(CastInst, {Part, Lane}, VectorLoopVal);
1642 else
1643 VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal);
1644}
1645
1646void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
1647 assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1648, __PRETTY_FUNCTION__))
1648 "Primary induction variable must have an integer type")(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1648, __PRETTY_FUNCTION__))
;
1649
1650 auto II = Legal->getInductionVars()->find(IV);
1651 assert(II != Legal->getInductionVars()->end() && "IV is not an induction")((II != Legal->getInductionVars()->end() && "IV is not an induction"
) ? static_cast<void> (0) : __assert_fail ("II != Legal->getInductionVars()->end() && \"IV is not an induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1651, __PRETTY_FUNCTION__))
;
1652
1653 auto ID = II->second;
1654 assert(IV->getType() == ID.getStartValue()->getType() && "Types must match")((IV->getType() == ID.getStartValue()->getType() &&
"Types must match") ? static_cast<void> (0) : __assert_fail
("IV->getType() == ID.getStartValue()->getType() && \"Types must match\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1654, __PRETTY_FUNCTION__))
;
1655
1656 // The scalar value to broadcast. This will be derived from the canonical
1657 // induction variable.
1658 Value *ScalarIV = nullptr;
1659
1660 // The value from the original loop to which we are mapping the new induction
1661 // variable.
1662 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
1663
1664 // True if we have vectorized the induction variable.
1665 auto VectorizedIV = false;
1666
1667 // Determine if we want a scalar version of the induction variable. This is
1668 // true if the induction variable itself is not widened, or if it has at
1669 // least one user in the loop that is not widened.
1670 auto NeedsScalarIV = VF > 1 && needsScalarInduction(EntryVal);
1671
1672 // Generate code for the induction step. Note that induction steps are
1673 // required to be loop-invariant
1674 assert(PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&((PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&
"Induction step should be loop invariant") ? static_cast<
void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) && \"Induction step should be loop invariant\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1675, __PRETTY_FUNCTION__))
1675 "Induction step should be loop invariant")((PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) &&
"Induction step should be loop invariant") ? static_cast<
void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(ID.getStep(), OrigLoop) && \"Induction step should be loop invariant\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1675, __PRETTY_FUNCTION__))
;
1676 auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
1677 Value *Step = nullptr;
1678 if (PSE.getSE()->isSCEVable(IV->getType())) {
1679 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
1680 Step = Exp.expandCodeFor(ID.getStep(), ID.getStep()->getType(),
1681 LoopVectorPreHeader->getTerminator());
1682 } else {
1683 Step = cast<SCEVUnknown>(ID.getStep())->getValue();
1684 }
1685
1686 // Try to create a new independent vector induction variable. If we can't
1687 // create the phi node, we will splat the scalar induction variable in each
1688 // loop iteration.
1689 if (VF > 1 && !shouldScalarizeInstruction(EntryVal)) {
1690 createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
1691 VectorizedIV = true;
1692 }
1693
1694 // If we haven't yet vectorized the induction variable, or if we will create
1695 // a scalar one, we need to define the scalar induction variable and step
1696 // values. If we were given a truncation type, truncate the canonical
1697 // induction variable and step. Otherwise, derive these values from the
1698 // induction descriptor.
1699 if (!VectorizedIV || NeedsScalarIV) {
1700 ScalarIV = Induction;
1701 if (IV != OldInduction) {
1702 ScalarIV = IV->getType()->isIntegerTy()
1703 ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
1704 : Builder.CreateCast(Instruction::SIToFP, Induction,
1705 IV->getType());
1706 ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID);
1707 ScalarIV->setName("offset.idx");
1708 }
1709 if (Trunc) {
1710 auto *TruncType = cast<IntegerType>(Trunc->getType());
1711 assert(Step->getType()->isIntegerTy() &&((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1712, __PRETTY_FUNCTION__))
1712 "Truncation requires an integer step")((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1712, __PRETTY_FUNCTION__))
;
1713 ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);
1714 Step = Builder.CreateTrunc(Step, TruncType);
1715 }
1716 }
1717
1718 // If we haven't yet vectorized the induction variable, splat the scalar
1719 // induction variable, and build the necessary step vectors.
1720 // TODO: Don't do it unless the vectorized IV is really required.
1721 if (!VectorizedIV) {
1722 Value *Broadcasted = getBroadcastInstrs(ScalarIV);
1723 for (unsigned Part = 0; Part < UF; ++Part) {
1724 Value *EntryPart =
1725 getStepVector(Broadcasted, VF * Part, Step, ID.getInductionOpcode());
1726 VectorLoopValueMap.setVectorValue(EntryVal, Part, EntryPart);
1727 if (Trunc)
1728 addMetadata(EntryPart, Trunc);
1729 recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, Part);
1730 }
1731 }
1732
1733 // If an induction variable is only used for counting loop iterations or
1734 // calculating addresses, it doesn't need to be widened. Create scalar steps
1735 // that can be used by instructions we will later scalarize. Note that the
1736 // addition of the scalar steps will not increase the number of instructions
1737 // in the loop in the common case prior to InstCombine. We will be trading
1738 // one vector extract for each scalar step.
1739 if (NeedsScalarIV)
1740 buildScalarSteps(ScalarIV, Step, EntryVal, ID);
1741}
1742
1743Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
1744 Instruction::BinaryOps BinOp) {
1745 // Create and check the types.
1746 assert(Val->getType()->isVectorTy() && "Must be a vector")((Val->getType()->isVectorTy() && "Must be a vector"
) ? static_cast<void> (0) : __assert_fail ("Val->getType()->isVectorTy() && \"Must be a vector\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1746, __PRETTY_FUNCTION__))
;
1747 int VLen = Val->getType()->getVectorNumElements();
1748
1749 Type *STy = Val->getType()->getScalarType();
1750 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
"Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1751, __PRETTY_FUNCTION__))
1751 "Induction Step must be an integer or FP")(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
"Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1751, __PRETTY_FUNCTION__))
;
1752 assert(Step->getType() == STy && "Step has wrong type")((Step->getType() == STy && "Step has wrong type")
? static_cast<void> (0) : __assert_fail ("Step->getType() == STy && \"Step has wrong type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1752, __PRETTY_FUNCTION__))
;
1753
1754 SmallVector<Constant *, 8> Indices;
1755
1756 if (STy->isIntegerTy()) {
1757 // Create a vector of consecutive numbers from zero to VF.
1758 for (int i = 0; i < VLen; ++i)
1759 Indices.push_back(ConstantInt::get(STy, StartIdx + i));
1760
1761 // Add the consecutive indices to the vector value.
1762 Constant *Cv = ConstantVector::get(Indices);
1763 assert(Cv->getType() == Val->getType() && "Invalid consecutive vec")((Cv->getType() == Val->getType() && "Invalid consecutive vec"
) ? static_cast<void> (0) : __assert_fail ("Cv->getType() == Val->getType() && \"Invalid consecutive vec\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1763, __PRETTY_FUNCTION__))
;
1764 Step = Builder.CreateVectorSplat(VLen, Step);
1765 assert(Step->getType() == Val->getType() && "Invalid step vec")((Step->getType() == Val->getType() && "Invalid step vec"
) ? static_cast<void> (0) : __assert_fail ("Step->getType() == Val->getType() && \"Invalid step vec\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1765, __PRETTY_FUNCTION__))
;
1766 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
1767 // which can be found from the original scalar operations.
1768 Step = Builder.CreateMul(Cv, Step);
1769 return Builder.CreateAdd(Val, Step, "induction");
1770 }
1771
1772 // Floating point induction.
1773 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
"Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1774, __PRETTY_FUNCTION__))
1774 "Binary Opcode should be specified for FP induction")(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
"Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1774, __PRETTY_FUNCTION__))
;
1775 // Create a vector of consecutive numbers from zero to VF.
1776 for (int i = 0; i < VLen; ++i)
1777 Indices.push_back(ConstantFP::get(STy, (double)(StartIdx + i)));
1778
1779 // Add the consecutive indices to the vector value.
1780 Constant *Cv = ConstantVector::get(Indices);
1781
1782 Step = Builder.CreateVectorSplat(VLen, Step);
1783
1784 // Floating point operations had to be 'fast' to enable the induction.
1785 FastMathFlags Flags;
1786 Flags.setFast();
1787
1788 Value *MulOp = Builder.CreateFMul(Cv, Step);
1789 if (isa<Instruction>(MulOp))
1790 // Have to check, MulOp may be a constant
1791 cast<Instruction>(MulOp)->setFastMathFlags(Flags);
1792
1793 Value *BOp = Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
1794 if (isa<Instruction>(BOp))
1795 cast<Instruction>(BOp)->setFastMathFlags(Flags);
1796 return BOp;
1797}
1798
1799void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
1800 Instruction *EntryVal,
1801 const InductionDescriptor &ID) {
1802 // We shouldn't have to build scalar steps if we aren't vectorizing.
1803 assert(VF > 1 && "VF should be greater than one")((VF > 1 && "VF should be greater than one") ? static_cast
<void> (0) : __assert_fail ("VF > 1 && \"VF should be greater than one\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1803, __PRETTY_FUNCTION__))
;
1804
1805 // Get the value type and ensure it and the step have the same integer type.
1806 Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
1807 assert(ScalarIVTy == Step->getType() &&((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1808, __PRETTY_FUNCTION__))
1808 "Val and Step should have the same type")((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1808, __PRETTY_FUNCTION__))
;
1809
1810 // We build scalar steps for both integer and floating-point induction
1811 // variables. Here, we determine the kind of arithmetic we will perform.
1812 Instruction::BinaryOps AddOp;
1813 Instruction::BinaryOps MulOp;
1814 if (ScalarIVTy->isIntegerTy()) {
1815 AddOp = Instruction::Add;
1816 MulOp = Instruction::Mul;
1817 } else {
1818 AddOp = ID.getInductionOpcode();
1819 MulOp = Instruction::FMul;
1820 }
1821
1822 // Determine the number of scalars we need to generate for each unroll
1823 // iteration. If EntryVal is uniform, we only need to generate the first
1824 // lane. Otherwise, we generate all VF values.
1825 unsigned Lanes =
1826 Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF) ? 1
1827 : VF;
1828 // Compute the scalar steps and save the results in VectorLoopValueMap.
1829 for (unsigned Part = 0; Part < UF; ++Part) {
1830 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
1831 auto *StartIdx = getSignedIntOrFpConstant(ScalarIVTy, VF * Part + Lane);
1832 auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step));
1833 auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul));
1834 VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add);
1835 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, Part, Lane);
1836 }
1837 }
1838}
1839
1840Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) {
1841 assert(V != Induction && "The new induction variable should not be used.")((V != Induction && "The new induction variable should not be used."
) ? static_cast<void> (0) : __assert_fail ("V != Induction && \"The new induction variable should not be used.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1841, __PRETTY_FUNCTION__))
;
1842 assert(!V->getType()->isVectorTy() && "Can't widen a vector")((!V->getType()->isVectorTy() && "Can't widen a vector"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVectorTy() && \"Can't widen a vector\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1842, __PRETTY_FUNCTION__))
;
1843 assert(!V->getType()->isVoidTy() && "Type does not produce a value")((!V->getType()->isVoidTy() && "Type does not produce a value"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVoidTy() && \"Type does not produce a value\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1843, __PRETTY_FUNCTION__))
;
1844
1845 // If we have a stride that is replaced by one, do it here. Defer this for
1846 // the VPlan-native path until we start running Legal checks in that path.
1847 if (!EnableVPlanNativePath && Legal->hasStride(V))
1848 V = ConstantInt::get(V->getType(), 1);
1849
1850 // If we have a vector mapped to this value, return it.
1851 if (VectorLoopValueMap.hasVectorValue(V, Part))
1852 return VectorLoopValueMap.getVectorValue(V, Part);
1853
1854 // If the value has not been vectorized, check if it has been scalarized
1855 // instead. If it has been scalarized, and we actually need the value in
1856 // vector form, we will construct the vector values on demand.
1857 if (VectorLoopValueMap.hasAnyScalarValue(V)) {
1858 Value *ScalarValue = VectorLoopValueMap.getScalarValue(V, {Part, 0});
1859
1860 // If we've scalarized a value, that value should be an instruction.
1861 auto *I = cast<Instruction>(V);
1862
1863 // If we aren't vectorizing, we can just copy the scalar map values over to
1864 // the vector map.
1865 if (VF == 1) {
1866 VectorLoopValueMap.setVectorValue(V, Part, ScalarValue);
1867 return ScalarValue;
1868 }
1869
1870 // Get the last scalar instruction we generated for V and Part. If the value
1871 // is known to be uniform after vectorization, this corresponds to lane zero
1872 // of the Part unroll iteration. Otherwise, the last instruction is the one
1873 // we created for the last vector lane of the Part unroll iteration.
1874 unsigned LastLane = Cost->isUniformAfterVectorization(I, VF) ? 0 : VF - 1;
1875 auto *LastInst = cast<Instruction>(
1876 VectorLoopValueMap.getScalarValue(V, {Part, LastLane}));
1877
1878 // Set the insert point after the last scalarized instruction. This ensures
1879 // the insertelement sequence will directly follow the scalar definitions.
1880 auto OldIP = Builder.saveIP();
1881 auto NewIP = std::next(BasicBlock::iterator(LastInst));
1882 Builder.SetInsertPoint(&*NewIP);
1883
1884 // However, if we are vectorizing, we need to construct the vector values.
1885 // If the value is known to be uniform after vectorization, we can just
1886 // broadcast the scalar value corresponding to lane zero for each unroll
1887 // iteration. Otherwise, we construct the vector values using insertelement
1888 // instructions. Since the resulting vectors are stored in
1889 // VectorLoopValueMap, we will only generate the insertelements once.
1890 Value *VectorValue = nullptr;
1891 if (Cost->isUniformAfterVectorization(I, VF)) {
1892 VectorValue = getBroadcastInstrs(ScalarValue);
1893 VectorLoopValueMap.setVectorValue(V, Part, VectorValue);
1894 } else {
1895 // Initialize packing with insertelements to start from undef.
1896 Value *Undef = UndefValue::get(VectorType::get(V->getType(), VF));
1897 VectorLoopValueMap.setVectorValue(V, Part, Undef);
1898 for (unsigned Lane = 0; Lane < VF; ++Lane)
1899 packScalarIntoVectorValue(V, {Part, Lane});
1900 VectorValue = VectorLoopValueMap.getVectorValue(V, Part);
1901 }
1902 Builder.restoreIP(OldIP);
1903 return VectorValue;
1904 }
1905
1906 // If this scalar is unknown, assume that it is a constant or that it is
1907 // loop invariant. Broadcast V and save the value for future uses.
1908 Value *B = getBroadcastInstrs(V);
1909 VectorLoopValueMap.setVectorValue(V, Part, B);
1910 return B;
1911}
1912
1913Value *
1914InnerLoopVectorizer::getOrCreateScalarValue(Value *V,
1915 const VPIteration &Instance) {
1916 // If the value is not an instruction contained in the loop, it should
1917 // already be scalar.
1918 if (OrigLoop->isLoopInvariant(V))
1919 return V;
1920
1921 assert(Instance.Lane > 0((Instance.Lane > 0 ? !Cost->isUniformAfterVectorization
(cast<Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Instance.Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1923, __PRETTY_FUNCTION__))
1922 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF)((Instance.Lane > 0 ? !Cost->isUniformAfterVectorization
(cast<Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Instance.Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1923, __PRETTY_FUNCTION__))
1923 : true && "Uniform values only have lane zero")((Instance.Lane > 0 ? !Cost->isUniformAfterVectorization
(cast<Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Instance.Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1923, __PRETTY_FUNCTION__))
;
1924
1925 // If the value from the original loop has not been vectorized, it is
1926 // represented by UF x VF scalar values in the new loop. Return the requested
1927 // scalar value.
1928 if (VectorLoopValueMap.hasScalarValue(V, Instance))
1929 return VectorLoopValueMap.getScalarValue(V, Instance);
1930
1931 // If the value has not been scalarized, get its entry in VectorLoopValueMap
1932 // for the given unroll part. If this entry is not a vector type (i.e., the
1933 // vectorization factor is one), there is no need to generate an
1934 // extractelement instruction.
1935 auto *U = getOrCreateVectorValue(V, Instance.Part);
1936 if (!U->getType()->isVectorTy()) {
1937 assert(VF == 1 && "Value not scalarized has non-vector type")((VF == 1 && "Value not scalarized has non-vector type"
) ? static_cast<void> (0) : __assert_fail ("VF == 1 && \"Value not scalarized has non-vector type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1937, __PRETTY_FUNCTION__))
;
1938 return U;
1939 }
1940
1941 // Otherwise, the value from the original loop has been vectorized and is
1942 // represented by UF vector values. Extract and return the requested scalar
1943 // value from the appropriate vector lane.
1944 return Builder.CreateExtractElement(U, Builder.getInt32(Instance.Lane));
1945}
1946
1947void InnerLoopVectorizer::packScalarIntoVectorValue(
1948 Value *V, const VPIteration &Instance) {
1949 assert(V != Induction && "The new induction variable should not be used.")((V != Induction && "The new induction variable should not be used."
) ? static_cast<void> (0) : __assert_fail ("V != Induction && \"The new induction variable should not be used.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1949, __PRETTY_FUNCTION__))
;
1950 assert(!V->getType()->isVectorTy() && "Can't pack a vector")((!V->getType()->isVectorTy() && "Can't pack a vector"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVectorTy() && \"Can't pack a vector\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1950, __PRETTY_FUNCTION__))
;
1951 assert(!V->getType()->isVoidTy() && "Type does not produce a value")((!V->getType()->isVoidTy() && "Type does not produce a value"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVoidTy() && \"Type does not produce a value\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1951, __PRETTY_FUNCTION__))
;
1952
1953 Value *ScalarInst = VectorLoopValueMap.getScalarValue(V, Instance);
1954 Value *VectorValue = VectorLoopValueMap.getVectorValue(V, Instance.Part);
1955 VectorValue = Builder.CreateInsertElement(VectorValue, ScalarInst,
1956 Builder.getInt32(Instance.Lane));
1957 VectorLoopValueMap.resetVectorValue(V, Instance.Part, VectorValue);
1958}
1959
1960Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
1961 assert(Vec->getType()->isVectorTy() && "Invalid type")((Vec->getType()->isVectorTy() && "Invalid type"
) ? static_cast<void> (0) : __assert_fail ("Vec->getType()->isVectorTy() && \"Invalid type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1961, __PRETTY_FUNCTION__))
;
1962 SmallVector<Constant *, 8> ShuffleMask;
1963 for (unsigned i = 0; i < VF; ++i)
1964 ShuffleMask.push_back(Builder.getInt32(VF - i - 1));
1965
1966 return Builder.CreateShuffleVector(Vec, UndefValue::get(Vec->getType()),
1967 ConstantVector::get(ShuffleMask),
1968 "reverse");
1969}
1970
1971// Return whether we allow using masked interleave-groups (for dealing with
1972// strided loads/stores that reside in predicated blocks, or for dealing
1973// with gaps).
1974static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
1975 // If an override option has been passed in for interleaved accesses, use it.
1976 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
1977 return EnableMaskedInterleavedMemAccesses;
1978
1979 return TTI.enableMaskedInterleavedAccessVectorization();
1980}
1981
1982// Try to vectorize the interleave group that \p Instr belongs to.
1983//
1984// E.g. Translate following interleaved load group (factor = 3):
1985// for (i = 0; i < N; i+=3) {
1986// R = Pic[i]; // Member of index 0
1987// G = Pic[i+1]; // Member of index 1
1988// B = Pic[i+2]; // Member of index 2
1989// ... // do something to R, G, B
1990// }
1991// To:
1992// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
1993// %R.vec = shuffle %wide.vec, undef, <0, 3, 6, 9> ; R elements
1994// %G.vec = shuffle %wide.vec, undef, <1, 4, 7, 10> ; G elements
1995// %B.vec = shuffle %wide.vec, undef, <2, 5, 8, 11> ; B elements
1996//
1997// Or translate following interleaved store group (factor = 3):
1998// for (i = 0; i < N; i+=3) {
1999// ... do something to R, G, B
2000// Pic[i] = R; // Member of index 0
2001// Pic[i+1] = G; // Member of index 1
2002// Pic[i+2] = B; // Member of index 2
2003// }
2004// To:
2005// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
2006// %B_U.vec = shuffle %B.vec, undef, <0, 1, 2, 3, u, u, u, u>
2007// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
2008// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
2009// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
2010void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,
2011 VectorParts *BlockInMask) {
2012 const InterleaveGroup<Instruction> *Group =
2013 Cost->getInterleavedAccessGroup(Instr);
2014 assert(Group && "Fail to get an interleaved access group.")((Group && "Fail to get an interleaved access group."
) ? static_cast<void> (0) : __assert_fail ("Group && \"Fail to get an interleaved access group.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2014, __PRETTY_FUNCTION__))
;
2015
2016 // Skip if current instruction is not the insert position.
2017 if (Instr != Group->getInsertPos())
2018 return;
2019
2020 const DataLayout &DL = Instr->getModule()->getDataLayout();
2021 Value *Ptr = getLoadStorePointerOperand(Instr);
2022
2023 // Prepare for the vector type of the interleaved load/store.
2024 Type *ScalarTy = getMemInstValueType(Instr);
2025 unsigned InterleaveFactor = Group->getFactor();
2026 Type *VecTy = VectorType::get(ScalarTy, InterleaveFactor * VF);
2027 Type *PtrTy = VecTy->getPointerTo(getLoadStoreAddressSpace(Instr));
2028
2029 // Prepare for the new pointers.
2030 setDebugLocFromInst(Builder, Ptr);
2031 SmallVector<Value *, 2> NewPtrs;
2032 unsigned Index = Group->getIndex(Instr);
2033
2034 VectorParts Mask;
2035 bool IsMaskForCondRequired = BlockInMask;
2036 if (IsMaskForCondRequired) {
2037 Mask = *BlockInMask;
2038 // TODO: extend the masked interleaved-group support to reversed access.
2039 assert(!Group->isReverse() && "Reversed masked interleave-group "((!Group->isReverse() && "Reversed masked interleave-group "
"not supported.") ? static_cast<void> (0) : __assert_fail
("!Group->isReverse() && \"Reversed masked interleave-group \" \"not supported.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2040, __PRETTY_FUNCTION__))
2040 "not supported.")((!Group->isReverse() && "Reversed masked interleave-group "
"not supported.") ? static_cast<void> (0) : __assert_fail
("!Group->isReverse() && \"Reversed masked interleave-group \" \"not supported.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2040, __PRETTY_FUNCTION__))
;
2041 }
2042
2043 // If the group is reverse, adjust the index to refer to the last vector lane
2044 // instead of the first. We adjust the index from the first vector lane,
2045 // rather than directly getting the pointer for lane VF - 1, because the
2046 // pointer operand of the interleaved access is supposed to be uniform. For
2047 // uniform instructions, we're only required to generate a value for the
2048 // first vector lane in each unroll iteration.
2049 if (Group->isReverse())
2050 Index += (VF - 1) * Group->getFactor();
2051
2052 bool InBounds = false;
2053 if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts()))
2054 InBounds = gep->isInBounds();
2055
2056 for (unsigned Part = 0; Part < UF; Part++) {
2057 Value *NewPtr = getOrCreateScalarValue(Ptr, {Part, 0});
2058
2059 // Notice current instruction could be any index. Need to adjust the address
2060 // to the member of index 0.
2061 //
2062 // E.g. a = A[i+1]; // Member of index 1 (Current instruction)
2063 // b = A[i]; // Member of index 0
2064 // Current pointer is pointed to A[i+1], adjust it to A[i].
2065 //
2066 // E.g. A[i+1] = a; // Member of index 1
2067 // A[i] = b; // Member of index 0
2068 // A[i+2] = c; // Member of index 2 (Current instruction)
2069 // Current pointer is pointed to A[i+2], adjust it to A[i].
2070 NewPtr = Builder.CreateGEP(ScalarTy, NewPtr, Builder.getInt32(-Index));
2071 if (InBounds)
2072 cast<GetElementPtrInst>(NewPtr)->setIsInBounds(true);
2073
2074 // Cast to the vector pointer type.
2075 NewPtrs.push_back(Builder.CreateBitCast(NewPtr, PtrTy));
2076 }
2077
2078 setDebugLocFromInst(Builder, Instr);
2079 Value *UndefVec = UndefValue::get(VecTy);
2080
2081 Value *MaskForGaps = nullptr;
2082 if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) {
2083 MaskForGaps = createBitMaskForGaps(Builder, VF, *Group);
2084 assert(MaskForGaps && "Mask for Gaps is required but it is null")((MaskForGaps && "Mask for Gaps is required but it is null"
) ? static_cast<void> (0) : __assert_fail ("MaskForGaps && \"Mask for Gaps is required but it is null\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2084, __PRETTY_FUNCTION__))
;
2085 }
2086
2087 // Vectorize the interleaved load group.
2088 if (isa<LoadInst>(Instr)) {
2089 // For each unroll part, create a wide load for the group.
2090 SmallVector<Value *, 2> NewLoads;
2091 for (unsigned Part = 0; Part < UF; Part++) {
2092 Instruction *NewLoad;
2093 if (IsMaskForCondRequired || MaskForGaps) {
2094 assert(useMaskedInterleavedAccesses(*TTI) &&((useMaskedInterleavedAccesses(*TTI) && "masked interleaved groups are not allowed."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2095, __PRETTY_FUNCTION__))
2095 "masked interleaved groups are not allowed.")((useMaskedInterleavedAccesses(*TTI) && "masked interleaved groups are not allowed."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2095, __PRETTY_FUNCTION__))
;
2096 Value *GroupMask = MaskForGaps;
2097 if (IsMaskForCondRequired) {
2098 auto *Undefs = UndefValue::get(Mask[Part]->getType());
2099 auto *RepMask = createReplicatedMask(Builder, InterleaveFactor, VF);
2100 Value *ShuffledMask = Builder.CreateShuffleVector(
2101 Mask[Part], Undefs, RepMask, "interleaved.mask");
2102 GroupMask = MaskForGaps
2103 ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
2104 MaskForGaps)
2105 : ShuffledMask;
2106 }
2107 NewLoad =
2108 Builder.CreateMaskedLoad(NewPtrs[Part], Group->getAlignment(),
2109 GroupMask, UndefVec, "wide.masked.vec");
2110 }
2111 else
2112 NewLoad = Builder.CreateAlignedLoad(VecTy, NewPtrs[Part],
2113 Group->getAlignment(), "wide.vec");
2114 Group->addMetadata(NewLoad);
2115 NewLoads.push_back(NewLoad);
2116 }
2117
2118 // For each member in the group, shuffle out the appropriate data from the
2119 // wide loads.
2120 for (unsigned I = 0; I < InterleaveFactor; ++I) {
2121 Instruction *Member = Group->getMember(I);
2122
2123 // Skip the gaps in the group.
2124 if (!Member)
2125 continue;
2126
2127 Constant *StrideMask = createStrideMask(Builder, I, InterleaveFactor, VF);
2128 for (unsigned Part = 0; Part < UF; Part++) {
2129 Value *StridedVec = Builder.CreateShuffleVector(
2130 NewLoads[Part], UndefVec, StrideMask, "strided.vec");
2131
2132 // If this member has different type, cast the result type.
2133 if (Member->getType() != ScalarTy) {
2134 VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
2135 StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
2136 }
2137
2138 if (Group->isReverse())
2139 StridedVec = reverseVector(StridedVec);
2140
2141 VectorLoopValueMap.setVectorValue(Member, Part, StridedVec);
2142 }
2143 }
2144 return;
2145 }
2146
2147 // The sub vector type for current instruction.
2148 VectorType *SubVT = VectorType::get(ScalarTy, VF);
2149
2150 // Vectorize the interleaved store group.
2151 for (unsigned Part = 0; Part < UF; Part++) {
2152 // Collect the stored vector from each member.
2153 SmallVector<Value *, 4> StoredVecs;
2154 for (unsigned i = 0; i < InterleaveFactor; i++) {
2155 // Interleaved store group doesn't allow a gap, so each index has a member
2156 Instruction *Member = Group->getMember(i);
2157 assert(Member && "Fail to get a member from an interleaved store group")((Member && "Fail to get a member from an interleaved store group"
) ? static_cast<void> (0) : __assert_fail ("Member && \"Fail to get a member from an interleaved store group\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2157, __PRETTY_FUNCTION__))
;
2158
2159 Value *StoredVec = getOrCreateVectorValue(
2160 cast<StoreInst>(Member)->getValueOperand(), Part);
2161 if (Group->isReverse())
2162 StoredVec = reverseVector(StoredVec);
2163
2164 // If this member has different type, cast it to a unified type.
2165
2166 if (StoredVec->getType() != SubVT)
2167 StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);
2168
2169 StoredVecs.push_back(StoredVec);
2170 }
2171
2172 // Concatenate all vectors into a wide vector.
2173 Value *WideVec = concatenateVectors(Builder, StoredVecs);
2174
2175 // Interleave the elements in the wide vector.
2176 Constant *IMask = createInterleaveMask(Builder, VF, InterleaveFactor);
2177 Value *IVec = Builder.CreateShuffleVector(WideVec, UndefVec, IMask,
2178 "interleaved.vec");
2179
2180 Instruction *NewStoreInstr;
2181 if (IsMaskForCondRequired) {
2182 auto *Undefs = UndefValue::get(Mask[Part]->getType());
2183 auto *RepMask = createReplicatedMask(Builder, InterleaveFactor, VF);
2184 Value *ShuffledMask = Builder.CreateShuffleVector(
2185 Mask[Part], Undefs, RepMask, "interleaved.mask");
2186 NewStoreInstr = Builder.CreateMaskedStore(
2187 IVec, NewPtrs[Part], Group->getAlignment(), ShuffledMask);
2188 }
2189 else
2190 NewStoreInstr = Builder.CreateAlignedStore(IVec, NewPtrs[Part],
2191 Group->getAlignment());
2192
2193 Group->addMetadata(NewStoreInstr);
2194 }
2195}
2196
2197void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
2198 VectorParts *BlockInMask) {
2199 // Attempt to issue a wide load.
2200 LoadInst *LI = dyn_cast<LoadInst>(Instr);
2201 StoreInst *SI = dyn_cast<StoreInst>(Instr);
2202
2203 assert((LI || SI) && "Invalid Load/Store instruction")(((LI || SI) && "Invalid Load/Store instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid Load/Store instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2203, __PRETTY_FUNCTION__))
;
2204
2205 LoopVectorizationCostModel::InstWidening Decision =
2206 Cost->getWideningDecision(Instr, VF);
2207 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&((Decision != LoopVectorizationCostModel::CM_Unknown &&
"CM decision should be taken at this point") ? static_cast<
void> (0) : __assert_fail ("Decision != LoopVectorizationCostModel::CM_Unknown && \"CM decision should be taken at this point\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2208, __PRETTY_FUNCTION__))
2208 "CM decision should be taken at this point")((Decision != LoopVectorizationCostModel::CM_Unknown &&
"CM decision should be taken at this point") ? static_cast<
void> (0) : __assert_fail ("Decision != LoopVectorizationCostModel::CM_Unknown && \"CM decision should be taken at this point\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2208, __PRETTY_FUNCTION__))
;
2209 if (Decision == LoopVectorizationCostModel::CM_Interleave)
2210 return vectorizeInterleaveGroup(Instr);
2211
2212 Type *ScalarDataTy = getMemInstValueType(Instr);
2213 Type *DataTy = VectorType::get(ScalarDataTy, VF);
2214 Value *Ptr = getLoadStorePointerOperand(Instr);
2215 unsigned Alignment = getLoadStoreAlignment(Instr);
2216 // An alignment of 0 means target abi alignment. We need to use the scalar's
2217 // target abi alignment in such a case.
2218 const DataLayout &DL = Instr->getModule()->getDataLayout();
2219 if (!Alignment)
2220 Alignment = DL.getABITypeAlignment(ScalarDataTy);
2221 unsigned AddressSpace = getLoadStoreAddressSpace(Instr);
2222
2223 // Determine if the pointer operand of the access is either consecutive or
2224 // reverse consecutive.
2225 bool Reverse = (Decision == LoopVectorizationCostModel::CM_Widen_Reverse);
2226 bool ConsecutiveStride =
2227 Reverse || (Decision == LoopVectorizationCostModel::CM_Widen);
2228 bool CreateGatherScatter =
2229 (Decision == LoopVectorizationCostModel::CM_GatherScatter);
2230
2231 // Either Ptr feeds a vector load/store, or a vector GEP should feed a vector
2232 // gather/scatter. Otherwise Decision should have been to Scalarize.
2233 assert((ConsecutiveStride || CreateGatherScatter) &&(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2234, __PRETTY_FUNCTION__))
2234 "The instruction should be scalarized")(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2234, __PRETTY_FUNCTION__))
;
2235
2236 // Handle consecutive loads/stores.
2237 if (ConsecutiveStride)
2238 Ptr = getOrCreateScalarValue(Ptr, {0, 0});
2239
2240 VectorParts Mask;
2241 bool isMaskRequired = BlockInMask;
2242 if (isMaskRequired)
2243 Mask = *BlockInMask;
2244
2245 bool InBounds = false;
2246 if (auto *gep = dyn_cast<GetElementPtrInst>(
2247 getLoadStorePointerOperand(Instr)->stripPointerCasts()))
2248 InBounds = gep->isInBounds();
2249
2250 const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
2251 // Calculate the pointer for the specific unroll-part.
2252 GetElementPtrInst *PartPtr = nullptr;
2253
2254 if (Reverse) {
2255 // If the address is consecutive but reversed, then the
2256 // wide store needs to start at the last vector element.
2257 PartPtr = cast<GetElementPtrInst>(
2258 Builder.CreateGEP(ScalarDataTy, Ptr, Builder.getInt32(-Part * VF)));
2259 PartPtr->setIsInBounds(InBounds);
2260 PartPtr = cast<GetElementPtrInst>(
2261 Builder.CreateGEP(ScalarDataTy, PartPtr, Builder.getInt32(1 - VF)));
2262 PartPtr->setIsInBounds(InBounds);
2263 if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
2264 Mask[Part] = reverseVector(Mask[Part]);
2265 } else {
2266 PartPtr = cast<GetElementPtrInst>(
2267 Builder.CreateGEP(ScalarDataTy, Ptr, Builder.getInt32(Part * VF)));
2268 PartPtr->setIsInBounds(InBounds);
2269 }
2270
2271 return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));
2272 };
2273
2274 // Handle Stores:
2275 if (SI) {
2276 setDebugLocFromInst(Builder, SI);
2277
2278 for (unsigned Part = 0; Part < UF; ++Part) {
2279 Instruction *NewSI = nullptr;
2280 Value *StoredVal = getOrCreateVectorValue(SI->getValueOperand(), Part);
2281 if (CreateGatherScatter) {
2282 Value *MaskPart = isMaskRequired ? Mask[Part] : nullptr;
2283 Value *VectorGep = getOrCreateVectorValue(Ptr, Part);
2284 NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
2285 MaskPart);
2286 } else {
2287 if (Reverse) {
2288 // If we store to reverse consecutive memory locations, then we need
2289 // to reverse the order of elements in the stored value.
2290 StoredVal = reverseVector(StoredVal);
2291 // We don't want to update the value in the map as it might be used in
2292 // another expression. So don't call resetVectorValue(StoredVal).
2293 }
2294 auto *VecPtr = CreateVecPtr(Part, Ptr);
2295 if (isMaskRequired)
2296 NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
2297 Mask[Part]);
2298 else
2299 NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
2300 }
2301 addMetadata(NewSI, SI);
2302 }
2303 return;
2304 }
2305
2306 // Handle loads.
2307 assert(LI && "Must have a load instruction")((LI && "Must have a load instruction") ? static_cast
<void> (0) : __assert_fail ("LI && \"Must have a load instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2307, __PRETTY_FUNCTION__))
;
2308 setDebugLocFromInst(Builder, LI);
2309 for (unsigned Part = 0; Part < UF; ++Part) {
2310 Value *NewLI;
2311 if (CreateGatherScatter) {
2312 Value *MaskPart = isMaskRequired ? Mask[Part] : nullptr;
2313 Value *VectorGep = getOrCreateVectorValue(Ptr, Part);
2314 NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,
2315 nullptr, "wide.masked.gather");
2316 addMetadata(NewLI, LI);
2317 } else {
2318 auto *VecPtr = CreateVecPtr(Part, Ptr);
2319 if (isMaskRequired)
2320 NewLI = Builder.CreateMaskedLoad(VecPtr, Alignment, Mask[Part],
2321 UndefValue::get(DataTy),
2322 "wide.masked.load");
2323 else
2324 NewLI =
2325 Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
2326
2327 // Add metadata to the load, but setVectorValue to the reverse shuffle.
2328 addMetadata(NewLI, LI);
2329 if (Reverse)
2330 NewLI = reverseVector(NewLI);
2331 }
2332 VectorLoopValueMap.setVectorValue(Instr, Part, NewLI);
2333 }
2334}
2335
2336void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,
2337 const VPIteration &Instance,
2338 bool IfPredicateInstr) {
2339 assert(!Instr->getType()->isAggregateType() && "Can't handle vectors")((!Instr->getType()->isAggregateType() && "Can't handle vectors"
) ? static_cast<void> (0) : __assert_fail ("!Instr->getType()->isAggregateType() && \"Can't handle vectors\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2339, __PRETTY_FUNCTION__))
;
2340
2341 setDebugLocFromInst(Builder, Instr);
2342
2343 // Does this instruction return a value ?
2344 bool IsVoidRetTy = Instr->getType()->isVoidTy();
2345
2346 Instruction *Cloned = Instr->clone();
2347 if (!IsVoidRetTy)
2348 Cloned->setName(Instr->getName() + ".cloned");
2349
2350 // Replace the operands of the cloned instructions with their scalar
2351 // equivalents in the new loop.
2352 for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
2353 auto *NewOp = getOrCreateScalarValue(Instr->getOperand(op), Instance);
2354 Cloned->setOperand(op, NewOp);
2355 }
2356 addNewMetadata(Cloned, Instr);
2357
2358 // Place the cloned scalar in the new loop.
2359 Builder.Insert(Cloned);
2360
2361 // Add the cloned scalar to the scalar map entry.
2362 VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned);
2363
2364 // If we just cloned a new assumption, add it the assumption cache.
2365 if (auto *II = dyn_cast<IntrinsicInst>(Cloned))
2366 if (II->getIntrinsicID() == Intrinsic::assume)
2367 AC->registerAssumption(II);
2368
2369 // End if-block.
2370 if (IfPredicateInstr)
2371 PredicatedInstructions.push_back(Cloned);
2372}
2373
2374PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,
2375 Value *End, Value *Step,
2376 Instruction *DL) {
2377 BasicBlock *Header = L->getHeader();
2378 BasicBlock *Latch = L->getLoopLatch();
2379 // As we're just creating this loop, it's possible no latch exists
2380 // yet. If so, use the header as this will be a single block loop.
2381 if (!Latch)
2382 Latch = Header;
2383
2384 IRBuilder<> Builder(&*Header->getFirstInsertionPt());
2385 Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);
2386 setDebugLocFromInst(Builder, OldInst);
2387 auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index");
2388
2389 Builder.SetInsertPoint(Latch->getTerminator());
2390 setDebugLocFromInst(Builder, OldInst);
2391
2392 // Create i+1 and fill the PHINode.
2393 Value *Next = Builder.CreateAdd(Induction, Step, "index.next");
2394 Induction->addIncoming(Start, L->getLoopPreheader());
2395 Induction->addIncoming(Next, Latch);
2396 // Create the compare.
2397 Value *ICmp = Builder.CreateICmpEQ(Next, End);
2398 Builder.CreateCondBr(ICmp, L->getExitBlock(), Header);
2399
2400 // Now we have two terminators. Remove the old one from the block.
2401 Latch->getTerminator()->eraseFromParent();
2402
2403 return Induction;
2404}
2405
2406Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
2407 if (TripCount)
2408 return TripCount;
2409
2410 assert(L && "Create Trip Count for null loop.")((L && "Create Trip Count for null loop.") ? static_cast
<void> (0) : __assert_fail ("L && \"Create Trip Count for null loop.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2410, __PRETTY_FUNCTION__))
;
2411 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
2412 // Find the loop boundaries.
2413 ScalarEvolution *SE = PSE.getSE();
2414 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
2415 assert(BackedgeTakenCount != SE->getCouldNotCompute() &&((BackedgeTakenCount != SE->getCouldNotCompute() &&
"Invalid loop count") ? static_cast<void> (0) : __assert_fail
("BackedgeTakenCount != SE->getCouldNotCompute() && \"Invalid loop count\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2416, __PRETTY_FUNCTION__))
2416 "Invalid loop count")((BackedgeTakenCount != SE->getCouldNotCompute() &&
"Invalid loop count") ? static_cast<void> (0) : __assert_fail
("BackedgeTakenCount != SE->getCouldNotCompute() && \"Invalid loop count\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2416, __PRETTY_FUNCTION__))
;
2417
2418 Type *IdxTy = Legal->getWidestInductionType();
2419 assert(IdxTy && "No type for induction")((IdxTy && "No type for induction") ? static_cast<
void> (0) : __assert_fail ("IdxTy && \"No type for induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2419, __PRETTY_FUNCTION__))
;
2420
2421 // The exit count might have the type of i64 while the phi is i32. This can
2422 // happen if we have an induction variable that is sign extended before the
2423 // compare. The only way that we get a backedge taken count is that the
2424 // induction variable was signed and as such will not overflow. In such a case
2425 // truncation is legal.
2426 if (BackedgeTakenCount->getType()->getPrimitiveSizeInBits() >
2427 IdxTy->getPrimitiveSizeInBits())
2428 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
2429 BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
2430
2431 // Get the total trip count from the count by adding 1.
2432 const SCEV *ExitCount = SE->getAddExpr(
2433 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
2434
2435 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
2436
2437 // Expand the trip count and place the new instructions in the preheader.
2438 // Notice that the pre-header does not change, only the loop body.
2439 SCEVExpander Exp(*SE, DL, "induction");
2440
2441 // Count holds the overall loop count (N).
2442 TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
2443 L->getLoopPreheader()->getTerminator());
2444
2445 if (TripCount->getType()->isPointerTy())
2446 TripCount =
2447 CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
2448 L->getLoopPreheader()->getTerminator());
2449
2450 return TripCount;
2451}
2452
2453Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
2454 if (VectorTripCount)
2455 return VectorTripCount;
2456
2457 Value *TC = getOrCreateTripCount(L);
2458 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
2459
2460 Type *Ty = TC->getType();
2461 Constant *Step = ConstantInt::get(Ty, VF * UF);
2462
2463 // If the tail is to be folded by masking, round the number of iterations N
2464 // up to a multiple of Step instead of rounding down. This is done by first
2465 // adding Step-1 and then rounding down. Note that it's ok if this addition
2466 // overflows: the vector induction variable will eventually wrap to zero given
2467 // that it starts at zero and its Step is a power of two; the loop will then
2468 // exit, with the last early-exit vector comparison also producing all-true.
2469 if (Cost->foldTailByMasking()) {
2470 assert(isPowerOf2_32(VF * UF) &&((isPowerOf2_32(VF * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2471, __PRETTY_FUNCTION__))
2471 "VF*UF must be a power of 2 when folding tail by masking")((isPowerOf2_32(VF * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2471, __PRETTY_FUNCTION__))
;
2472 TC = Builder.CreateAdd(TC, ConstantInt::get(Ty, VF * UF - 1), "n.rnd.up");
2473 }
2474
2475 // Now we need to generate the expression for the part of the loop that the
2476 // vectorized body will execute. This is equal to N - (N % Step) if scalar
2477 // iterations are not required for correctness, or N - Step, otherwise. Step
2478 // is equal to the vectorization factor (number of SIMD elements) times the
2479 // unroll factor (number of SIMD instructions).
2480 Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
2481
2482 // If there is a non-reversed interleaved group that may speculatively access
2483 // memory out-of-bounds, we need to ensure that there will be at least one
2484 // iteration of the scalar epilogue loop. Thus, if the step evenly divides
2485 // the trip count, we set the remainder to be equal to the step. If the step
2486 // does not evenly divide the trip count, no adjustment is necessary since
2487 // there will already be scalar iterations. Note that the minimum iterations
2488 // check ensures that N >= Step.
2489 if (VF > 1 && Cost->requiresScalarEpilogue()) {
2490 auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
2491 R = Builder.CreateSelect(IsZero, Step, R);
2492 }
2493
2494 VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
2495
2496 return VectorTripCount;
2497}
2498
2499Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
2500 const DataLayout &DL) {
2501 // Verify that V is a vector type with same number of elements as DstVTy.
2502 unsigned VF = DstVTy->getNumElements();
2503 VectorType *SrcVecTy = cast<VectorType>(V->getType());
2504 assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match")(((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match"
) ? static_cast<void> (0) : __assert_fail ("(VF == SrcVecTy->getNumElements()) && \"Vector dimensions do not match\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2504, __PRETTY_FUNCTION__))
;
2505 Type *SrcElemTy = SrcVecTy->getElementType();
2506 Type *DstElemTy = DstVTy->getElementType();
2507 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&(((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy
)) && "Vector elements must have same size") ? static_cast
<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2508, __PRETTY_FUNCTION__))
2508 "Vector elements must have same size")(((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy
)) && "Vector elements must have same size") ? static_cast
<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2508, __PRETTY_FUNCTION__))
;
2509
2510 // Do a direct cast if element types are castable.
2511 if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
2512 return Builder.CreateBitOrPointerCast(V, DstVTy);
2513 }
2514 // V cannot be directly casted to desired vector type.
2515 // May happen when V is a floating point vector but DstVTy is a vector of
2516 // pointers or vice-versa. Handle this using a two-step bitcast using an
2517 // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
2518 assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&(((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()
) && "Only one type should be a pointer type") ? static_cast
<void> (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2519, __PRETTY_FUNCTION__))
2519 "Only one type should be a pointer type")(((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()
) && "Only one type should be a pointer type") ? static_cast
<void> (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2519, __PRETTY_FUNCTION__))
;
2520 assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&(((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy
()) && "Only one type should be a floating point type"
) ? static_cast<void> (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2521, __PRETTY_FUNCTION__))
2521 "Only one type should be a floating point type")(((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy
()) && "Only one type should be a floating point type"
) ? static_cast<void> (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2521, __PRETTY_FUNCTION__))
;
2522 Type *IntTy =
2523 IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
2524 VectorType *VecIntTy = VectorType::get(IntTy, VF);
2525 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
2526 return Builder.CreateBitOrPointerCast(CastVal, DstVTy);
2527}
2528
2529void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
2530 BasicBlock *Bypass) {
2531 Value *Count = getOrCreateTripCount(L);
2532 BasicBlock *BB = L->getLoopPreheader();
2533 IRBuilder<> Builder(BB->getTerminator());
2534
2535 // Generate code to check if the loop's trip count is less than VF * UF, or
2536 // equal to it in case a scalar epilogue is required; this implies that the
2537 // vector trip count is zero. This check also covers the case where adding one
2538 // to the backedge-taken count overflowed leading to an incorrect trip count
2539 // of zero. In this case we will also jump to the scalar loop.
2540 auto P = Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE
2541 : ICmpInst::ICMP_ULT;
2542
2543 // If tail is to be folded, vector loop takes care of all iterations.
2544 Value *CheckMinIters = Builder.getFalse();
2545 if (!Cost->foldTailByMasking())
2546 CheckMinIters = Builder.CreateICmp(
2547 P, Count, ConstantInt::get(Count->getType(), VF * UF),
2548 "min.iters.check");
2549
2550 BasicBlock *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");
2551 // Update dominator tree immediately if the generated block is a
2552 // LoopBypassBlock because SCEV expansions to generate loop bypass
2553 // checks may query it before the current function is finished.
2554 DT->addNewBlock(NewBB, BB);
2555 if (L->getParentLoop())
2556 L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
2557 ReplaceInstWithInst(BB->getTerminator(),
2558 BranchInst::Create(Bypass, NewBB, CheckMinIters));
2559 LoopBypassBlocks.push_back(BB);
2560}
2561
2562void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {
2563 BasicBlock *BB = L->getLoopPreheader();
2564
2565 // Generate the code to check that the SCEV assumptions that we made.
2566 // We want the new basic block to start at the first instruction in a
2567 // sequence of instructions that form a check.
2568 SCEVExpander Exp(*PSE.getSE(), Bypass->getModule()->getDataLayout(),
2569 "scev.check");
2570 Value *SCEVCheck =
2571 Exp.expandCodeForPredicate(&PSE.getUnionPredicate(), BB->getTerminator());
2572
2573 if (auto *C = dyn_cast<ConstantInt>(SCEVCheck))
2574 if (C->isZero())
2575 return;
2576
2577 assert(!Cost->foldTailByMasking() &&((!Cost->foldTailByMasking() && "Cannot SCEV check stride or overflow when folding tail"
) ? static_cast<void> (0) : __assert_fail ("!Cost->foldTailByMasking() && \"Cannot SCEV check stride or overflow when folding tail\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2578, __PRETTY_FUNCTION__))
2578 "Cannot SCEV check stride or overflow when folding tail")((!Cost->foldTailByMasking() && "Cannot SCEV check stride or overflow when folding tail"
) ? static_cast<void> (0) : __assert_fail ("!Cost->foldTailByMasking() && \"Cannot SCEV check stride or overflow when folding tail\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2578, __PRETTY_FUNCTION__))
;
2579 // Create a new block containing the stride check.
2580 BB->setName("vector.scevcheck");
2581 auto *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");
2582 // Update dominator tree immediately if the generated block is a
2583 // LoopBypassBlock because SCEV expansions to generate loop bypass
2584 // checks may query it before the current function is finished.
2585 DT->addNewBlock(NewBB, BB);
2586 if (L->getParentLoop())
2587 L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
2588 ReplaceInstWithInst(BB->getTerminator(),
2589 BranchInst::Create(Bypass, NewBB, SCEVCheck));
2590 LoopBypassBlocks.push_back(BB);
2591 AddedSafetyChecks = true;
2592}
2593
2594void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) {
2595 // VPlan-native path does not do any analysis for runtime checks currently.
2596 if (EnableVPlanNativePath)
2597 return;
2598
2599 BasicBlock *BB = L->getLoopPreheader();
2600
2601 // Generate the code that checks in runtime if arrays overlap. We put the
2602 // checks into a separate block to make the more common case of few elements
2603 // faster.
2604 Instruction *FirstCheckInst;
2605 Instruction *MemRuntimeCheck;
2606 std::tie(FirstCheckInst, MemRuntimeCheck) =
2607 Legal->getLAI()->addRuntimeChecks(BB->getTerminator());
2608 if (!MemRuntimeCheck)
2609 return;
2610
2611 assert(!Cost->foldTailByMasking() && "Cannot check memory when folding tail")((!Cost->foldTailByMasking() && "Cannot check memory when folding tail"
) ? static_cast<void> (0) : __assert_fail ("!Cost->foldTailByMasking() && \"Cannot check memory when folding tail\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2611, __PRETTY_FUNCTION__))
;
2612 // Create a new block containing the memory check.
2613 BB->setName("vector.memcheck");
2614 auto *NewBB = BB->splitBasicBlock(BB->getTerminator(), "vector.ph");
2615 // Update dominator tree immediately if the generated block is a
2616 // LoopBypassBlock because SCEV expansions to generate loop bypass
2617 // checks may query it before the current function is finished.
2618 DT->addNewBlock(NewBB, BB);
2619 if (L->getParentLoop())
2620 L->getParentLoop()->addBasicBlockToLoop(NewBB, *LI);
2621 ReplaceInstWithInst(BB->getTerminator(),
2622 BranchInst::Create(Bypass, NewBB, MemRuntimeCheck));
2623 LoopBypassBlocks.push_back(BB);
2624 AddedSafetyChecks = true;
2625
2626 // We currently don't use LoopVersioning for the actual loop cloning but we
2627 // still use it to add the noalias metadata.
2628 LVer = llvm::make_unique<LoopVersioning>(*Legal->getLAI(), OrigLoop, LI, DT,
2629 PSE.getSE());
2630 LVer->prepareNoAliasMetadata();
2631}
2632
2633Value *InnerLoopVectorizer::emitTransformedIndex(
2634 IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL,
2635 const InductionDescriptor &ID) const {
2636
2637 SCEVExpander Exp(*SE, DL, "induction");
2638 auto Step = ID.getStep();
2639 auto StartValue = ID.getStartValue();
2640 assert(Index->getType() == Step->getType() &&((Index->getType() == Step->getType() && "Index type does not match StepValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == Step->getType() && \"Index type does not match StepValue type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2641, __PRETTY_FUNCTION__))
2641 "Index type does not match StepValue type")((Index->getType() == Step->getType() && "Index type does not match StepValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == Step->getType() && \"Index type does not match StepValue type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2641, __PRETTY_FUNCTION__))
;
2642
2643 // Note: the IR at this point is broken. We cannot use SE to create any new
2644 // SCEV and then expand it, hoping that SCEV's simplification will give us
2645 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
2646 // lead to various SCEV crashes. So all we can do is to use builder and rely
2647 // on InstCombine for future simplifications. Here we handle some trivial
2648 // cases only.
2649 auto CreateAdd = [&B](Value *X, Value *Y) {
2650 assert(X->getType() == Y->getType() && "Types don't match!")((X->getType() == Y->getType() && "Types don't match!"
) ? static_cast<void> (0) : __assert_fail ("X->getType() == Y->getType() && \"Types don't match!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2650, __PRETTY_FUNCTION__))
;
2651 if (auto *CX = dyn_cast<ConstantInt>(X))
2652 if (CX->isZero())
2653 return Y;
2654 if (auto *CY = dyn_cast<ConstantInt>(Y))
2655 if (CY->isZero())
2656 return X;
2657 return B.CreateAdd(X, Y);
2658 };
2659
2660 auto CreateMul = [&B](Value *X, Value *Y) {
2661 assert(X->getType() == Y->getType() && "Types don't match!")((X->getType() == Y->getType() && "Types don't match!"
) ? static_cast<void> (0) : __assert_fail ("X->getType() == Y->getType() && \"Types don't match!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2661, __PRETTY_FUNCTION__))
;
2662 if (auto *CX = dyn_cast<ConstantInt>(X))
2663 if (CX->isOne())
2664 return Y;
2665 if (auto *CY = dyn_cast<ConstantInt>(Y))
2666 if (CY->isOne())
2667 return X;
2668 return B.CreateMul(X, Y);
2669 };
2670
2671 switch (ID.getKind()) {
2672 case InductionDescriptor::IK_IntInduction: {
2673 assert(Index->getType() == StartValue->getType() &&((Index->getType() == StartValue->getType() && "Index type does not match StartValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2674, __PRETTY_FUNCTION__))
2674 "Index type does not match StartValue type")((Index->getType() == StartValue->getType() && "Index type does not match StartValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2674, __PRETTY_FUNCTION__))
;
2675 if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne())
2676 return B.CreateSub(StartValue, Index);
2677 auto *Offset = CreateMul(
2678 Index, Exp.expandCodeFor(Step, Index->getType(), &*B.GetInsertPoint()));
2679 return CreateAdd(StartValue, Offset);
2680 }
2681 case InductionDescriptor::IK_PtrInduction: {
2682 assert(isa<SCEVConstant>(Step) &&((isa<SCEVConstant>(Step) && "Expected constant step for pointer induction"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(Step) && \"Expected constant step for pointer induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2683, __PRETTY_FUNCTION__))
2683 "Expected constant step for pointer induction")((isa<SCEVConstant>(Step) && "Expected constant step for pointer induction"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(Step) && \"Expected constant step for pointer induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2683, __PRETTY_FUNCTION__))
;
2684 return B.CreateGEP(
2685 StartValue->getType()->getPointerElementType(), StartValue,
2686 CreateMul(Index, Exp.expandCodeFor(Step, Index->getType(),
2687 &*B.GetInsertPoint())));
2688 }
2689 case InductionDescriptor::IK_FpInduction: {
2690 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value")((Step->getType()->isFloatingPointTy() && "Expected FP Step value"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isFloatingPointTy() && \"Expected FP Step value\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2690, __PRETTY_FUNCTION__))
;
2691 auto InductionBinOp = ID.getInductionBinOp();
2692 assert(InductionBinOp &&((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2695, __PRETTY_FUNCTION__))
2693 (InductionBinOp->getOpcode() == Instruction::FAdd ||((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2695, __PRETTY_FUNCTION__))
2694 InductionBinOp->getOpcode() == Instruction::FSub) &&((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2695, __PRETTY_FUNCTION__))
2695 "Original bin op should be defined for FP induction")((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2695, __PRETTY_FUNCTION__))
;
2696
2697 Value *StepValue = cast<SCEVUnknown>(Step)->getValue();
2698
2699 // Floating point operations had to be 'fast' to enable the induction.
2700 FastMathFlags Flags;
2701 Flags.setFast();
2702
2703 Value *MulExp = B.CreateFMul(StepValue, Index);
2704 if (isa<Instruction>(MulExp))
2705 // We have to check, the MulExp may be a constant.
2706 cast<Instruction>(MulExp)->setFastMathFlags(Flags);
2707
2708 Value *BOp = B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
2709 "induction");
2710 if (isa<Instruction>(BOp))
2711 cast<Instruction>(BOp)->setFastMathFlags(Flags);
2712
2713 return BOp;
2714 }
2715 case InductionDescriptor::IK_NoInduction:
2716 return nullptr;
2717 }
2718 llvm_unreachable("invalid enum")::llvm::llvm_unreachable_internal("invalid enum", "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2718)
;
2719}
2720
2721BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
2722 /*
2723 In this function we generate a new loop. The new loop will contain
2724 the vectorized instructions while the old loop will continue to run the
2725 scalar remainder.
2726
2727 [ ] <-- loop iteration number check.
2728 / |
2729 / v
2730 | [ ] <-- vector loop bypass (may consist of multiple blocks).
2731 | / |
2732 | / v
2733 || [ ] <-- vector pre header.
2734 |/ |
2735 | v
2736 | [ ] \
2737 | [ ]_| <-- vector loop.
2738 | |
2739 | v
2740 | -[ ] <--- middle-block.
2741 | / |
2742 | / v
2743 -|- >[ ] <--- new preheader.
2744 | |
2745 | v
2746 | [ ] \
2747 | [ ]_| <-- old scalar loop to handle remainder.
2748 \ |
2749 \ v
2750 >[ ] <-- exit block.
2751 ...
2752 */
2753
2754 BasicBlock *OldBasicBlock = OrigLoop->getHeader();
2755 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
2756 BasicBlock *ExitBlock = OrigLoop->getExitBlock();
2757 MDNode *OrigLoopID = OrigLoop->getLoopID();
2758 assert(VectorPH && "Invalid loop structure")((VectorPH && "Invalid loop structure") ? static_cast
<void> (0) : __assert_fail ("VectorPH && \"Invalid loop structure\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2758, __PRETTY_FUNCTION__))
;
2759 assert(ExitBlock && "Must have an exit block")((ExitBlock && "Must have an exit block") ? static_cast
<void> (0) : __assert_fail ("ExitBlock && \"Must have an exit block\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2759, __PRETTY_FUNCTION__))
;
2760
2761 // Some loops have a single integer induction variable, while other loops
2762 // don't. One example is c++ iterators that often have multiple pointer
2763 // induction variables. In the code below we also support a case where we
2764 // don't have a single induction variable.
2765 //
2766 // We try to obtain an induction variable from the original loop as hard
2767 // as possible. However if we don't find one that:
2768 // - is an integer
2769 // - counts from zero, stepping by one
2770 // - is the size of the widest induction variable type
2771 // then we create a new one.
2772 OldInduction = Legal->getPrimaryInduction();
2773 Type *IdxTy = Legal->getWidestInductionType();
2774
2775 // Split the single block loop into the two loop structure described above.
2776 BasicBlock *VecBody =
2777 VectorPH->splitBasicBlock(VectorPH->getTerminator(), "vector.body");
2778 BasicBlock *MiddleBlock =
2779 VecBody->splitBasicBlock(VecBody->getTerminator(), "middle.block");
2780 BasicBlock *ScalarPH =
2781 MiddleBlock->splitBasicBlock(MiddleBlock->getTerminator(), "scalar.ph");
2782
2783 // Create and register the new vector loop.
2784 Loop *Lp = LI->AllocateLoop();
2785 Loop *ParentLoop = OrigLoop->getParentLoop();
2786
2787 // Insert the new loop into the loop nest and register the new basic blocks
2788 // before calling any utilities such as SCEV that require valid LoopInfo.
2789 if (ParentLoop) {
2790 ParentLoop->addChildLoop(Lp);
2791 ParentLoop->addBasicBlockToLoop(ScalarPH, *LI);
2792 ParentLoop->addBasicBlockToLoop(MiddleBlock, *LI);
2793 } else {
2794 LI->addTopLevelLoop(Lp);
2795 }
2796 Lp->addBasicBlockToLoop(VecBody, *LI);
2797
2798 // Find the loop boundaries.
2799 Value *Count = getOrCreateTripCount(Lp);
2800
2801 Value *StartIdx = ConstantInt::get(IdxTy, 0);
2802
2803 // Now, compare the new count to zero. If it is zero skip the vector loop and
2804 // jump to the scalar loop. This check also covers the case where the
2805 // backedge-taken count is uint##_max: adding one to it will overflow leading
2806 // to an incorrect trip count of zero. In this (rare) case we will also jump
2807 // to the scalar loop.
2808 emitMinimumIterationCountCheck(Lp, ScalarPH);
2809
2810 // Generate the code to check any assumptions that we've made for SCEV
2811 // expressions.
2812 emitSCEVChecks(Lp, ScalarPH);
2813
2814 // Generate the code that checks in runtime if arrays overlap. We put the
2815 // checks into a separate block to make the more common case of few elements
2816 // faster.
2817 emitMemRuntimeChecks(Lp, ScalarPH);
2818
2819 // Generate the induction variable.
2820 // The loop step is equal to the vectorization factor (num of SIMD elements)
2821 // times the unroll factor (num of SIMD instructions).
2822 Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
2823 Constant *Step = ConstantInt::get(IdxTy, VF * UF);
2824 Induction =
2825 createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
2826 getDebugLocFromInstOrOperands(OldInduction));
2827
2828 // We are going to resume the execution of the scalar loop.
2829 // Go over all of the induction variables that we found and fix the
2830 // PHIs that are left in the scalar version of the loop.
2831 // The starting values of PHI nodes depend on the counter of the last
2832 // iteration in the vectorized loop.
2833 // If we come from a bypass edge then we need to start from the original
2834 // start value.
2835
2836 // This variable saves the new starting index for the scalar loop. It is used
2837 // to test if there are any tail iterations left once the vector loop has
2838 // completed.
2839 LoopVectorizationLegality::InductionList *List = Legal->getInductionVars();
2840 for (auto &InductionEntry : *List) {
2841 PHINode *OrigPhi = InductionEntry.first;
2842 InductionDescriptor II = InductionEntry.second;
2843
2844 // Create phi nodes to merge from the backedge-taken check block.
2845 PHINode *BCResumeVal = PHINode::Create(
2846 OrigPhi->getType(), 3, "bc.resume.val", ScalarPH->getTerminator());
2847 // Copy original phi DL over to the new one.
2848 BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
2849 Value *&EndValue = IVEndValues[OrigPhi];
2850 if (OrigPhi == OldInduction) {
2851 // We know what the end value is.
2852 EndValue = CountRoundDown;
2853 } else {
2854 IRBuilder<> B(Lp->getLoopPreheader()->getTerminator());
2855 Type *StepType = II.getStep()->getType();
2856 Instruction::CastOps CastOp =
2857 CastInst::getCastOpcode(CountRoundDown, true, StepType, true);
2858 Value *CRD = B.CreateCast(CastOp, CountRoundDown, StepType, "cast.crd");
2859 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
2860 EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
2861 EndValue->setName("ind.end");
2862 }
2863
2864 // The new PHI merges the original incoming value, in case of a bypass,
2865 // or the value at the end of the vectorized loop.
2866 BCResumeVal->addIncoming(EndValue, MiddleBlock);
2867
2868 // Fix the scalar body counter (PHI node).
2869 unsigned BlockIdx = OrigPhi->getBasicBlockIndex(ScalarPH);
2870
2871 // The old induction's phi node in the scalar body needs the truncated
2872 // value.
2873 for (BasicBlock *BB : LoopBypassBlocks)
2874 BCResumeVal->addIncoming(II.getStartValue(), BB);
2875 OrigPhi->setIncomingValue(BlockIdx, BCResumeVal);
2876 }
2877
2878 // We need the OrigLoop (scalar loop part) latch terminator to help
2879 // produce correct debug info for the middle block BB instructions.
2880 // The legality check stage guarantees that the loop will have a single
2881 // latch.
2882 assert(isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator()) &&((isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator
()) && "Scalar loop latch terminator isn't a branch")
? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator()) && \"Scalar loop latch terminator isn't a branch\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2883, __PRETTY_FUNCTION__))
2883 "Scalar loop latch terminator isn't a branch")((isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator
()) && "Scalar loop latch terminator isn't a branch")
? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator()) && \"Scalar loop latch terminator isn't a branch\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2883, __PRETTY_FUNCTION__))
;
2884 BranchInst *ScalarLatchBr =
2885 cast<BranchInst>(OrigLoop->getLoopLatch()->getTerminator());
2886
2887 // Add a check in the middle block to see if we have completed
2888 // all of the iterations in the first vector loop.
2889 // If (N - N%VF) == N, then we *don't* need to run the remainder.
2890 // If tail is to be folded, we know we don't need to run the remainder.
2891 Value *CmpN = Builder.getTrue();
2892 if (!Cost->foldTailByMasking()) {
2893 CmpN =
2894 CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, Count,
2895 CountRoundDown, "cmp.n", MiddleBlock->getTerminator());
2896
2897 // Provide correct stepping behaviour by using the same DebugLoc as the
2898 // scalar loop latch branch cmp if it exists.
2899 if (CmpInst *ScalarLatchCmp =
2900 dyn_cast_or_null<CmpInst>(ScalarLatchBr->getCondition()))
2901 cast<Instruction>(CmpN)->setDebugLoc(ScalarLatchCmp->getDebugLoc());
2902 }
2903
2904 BranchInst *BrInst = BranchInst::Create(ExitBlock, ScalarPH, CmpN);
2905 BrInst->setDebugLoc(ScalarLatchBr->getDebugLoc());
2906 ReplaceInstWithInst(MiddleBlock->getTerminator(), BrInst);
2907
2908 // Get ready to start creating new instructions into the vectorized body.
2909 Builder.SetInsertPoint(&*VecBody->getFirstInsertionPt());
2910
2911 // Save the state.
2912 LoopVectorPreHeader = Lp->getLoopPreheader();
2913 LoopScalarPreHeader = ScalarPH;
2914 LoopMiddleBlock = MiddleBlock;
2915 LoopExitBlock = ExitBlock;
2916 LoopVectorBody = VecBody;
2917 LoopScalarBody = OldBasicBlock;
2918
2919 Optional<MDNode *> VectorizedLoopID =
2920 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
2921 LLVMLoopVectorizeFollowupVectorized});
2922 if (VectorizedLoopID.hasValue()) {
2923 Lp->setLoopID(VectorizedLoopID.getValue());
2924
2925 // Do not setAlreadyVectorized if loop attributes have been defined
2926 // explicitly.
2927 return LoopVectorPreHeader;
2928 }
2929
2930 // Keep all loop hints from the original loop on the vector loop (we'll
2931 // replace the vectorizer-specific hints below).
2932 if (MDNode *LID = OrigLoop->getLoopID())
2933 Lp->setLoopID(LID);
2934
2935 LoopVectorizeHints Hints(Lp, true, *ORE);
2936 Hints.setAlreadyVectorized();
2937
2938 return LoopVectorPreHeader;
2939}
2940
2941// Fix up external users of the induction variable. At this point, we are
2942// in LCSSA form, with all external PHIs that use the IV having one input value,
2943// coming from the remainder loop. We need those PHIs to also have a correct
2944// value for the IV when arriving directly from the middle block.
2945void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
2946 const InductionDescriptor &II,
2947 Value *CountRoundDown, Value *EndValue,
2948 BasicBlock *MiddleBlock) {
2949 // There are two kinds of external IV usages - those that use the value
2950 // computed in the last iteration (the PHI) and those that use the penultimate
2951 // value (the value that feeds into the phi from the loop latch).
2952 // We allow both, but they, obviously, have different values.
2953
2954 assert(OrigLoop->getExitBlock() && "Expected a single exit block")((OrigLoop->getExitBlock() && "Expected a single exit block"
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->getExitBlock() && \"Expected a single exit block\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2954, __PRETTY_FUNCTION__))
;
2955
2956 DenseMap<Value *, Value *> MissingVals;
2957
2958 // An external user of the last iteration's value should see the value that
2959 // the remainder loop uses to initialize its own IV.
2960 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());
2961 for (User *U : PostInc->users()) {
2962 Instruction *UI = cast<Instruction>(U);
2963 if (!OrigLoop->contains(UI)) {
2964 assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2964, __PRETTY_FUNCTION__))
;
2965 MissingVals[UI] = EndValue;
2966 }
2967 }
2968
2969 // An external user of the penultimate value need to see EndValue - Step.
2970 // The simplest way to get this is to recompute it from the constituent SCEVs,
2971 // that is Start + (Step * (CRD - 1)).
2972 for (User *U : OrigPhi->users()) {
2973 auto *UI = cast<Instruction>(U);
2974 if (!OrigLoop->contains(UI)) {
2975 const DataLayout &DL =
2976 OrigLoop->getHeader()->getModule()->getDataLayout();
2977 assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2977, __PRETTY_FUNCTION__))
;
2978
2979 IRBuilder<> B(MiddleBlock->getTerminator());
2980 Value *CountMinusOne = B.CreateSub(
2981 CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));
2982 Value *CMO =
2983 !II.getStep()->getType()->isIntegerTy()
2984 ? B.CreateCast(Instruction::SIToFP, CountMinusOne,
2985 II.getStep()->getType())
2986 : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
2987 CMO->setName("cast.cmo");
2988 Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II);
2989 Escape->setName("ind.escape");
2990 MissingVals[UI] = Escape;
2991 }
2992 }
2993
2994 for (auto &I : MissingVals) {
2995 PHINode *PHI = cast<PHINode>(I.first);
2996 // One corner case we have to handle is two IVs "chasing" each-other,
2997 // that is %IV2 = phi [...], [ %IV1, %latch ]
2998 // In this case, if IV1 has an external use, we need to avoid adding both
2999 // "last value of IV1" and "penultimate value of IV2". So, verify that we
3000 // don't already have an incoming value for the middle block.
3001 if (PHI->getBasicBlockIndex(MiddleBlock) == -1)
3002 PHI->addIncoming(I.second, MiddleBlock);
3003 }
3004}
3005
3006namespace {
3007
3008struct CSEDenseMapInfo {
3009 static bool canHandle(const Instruction *I) {
3010 return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
3011 isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
3012 }
3013
3014 static inline Instruction *getEmptyKey() {
3015 return DenseMapInfo<Instruction *>::getEmptyKey();
3016 }
3017
3018 static inline Instruction *getTombstoneKey() {
3019 return DenseMapInfo<Instruction *>::getTombstoneKey();
3020 }
3021
3022 static unsigned getHashValue(const Instruction *I) {
3023 assert(canHandle(I) && "Unknown instruction!")((canHandle(I) && "Unknown instruction!") ? static_cast
<void> (0) : __assert_fail ("canHandle(I) && \"Unknown instruction!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3023, __PRETTY_FUNCTION__))
;
3024 return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
3025 I->value_op_end()));
3026 }
3027
3028 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
3029 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
3030 LHS == getTombstoneKey() || RHS == getTombstoneKey())
3031 return LHS == RHS;
3032 return LHS->isIdenticalTo(RHS);
3033 }
3034};
3035
3036} // end anonymous namespace
3037
3038///Perform cse of induction variable instructions.
3039static void cse(BasicBlock *BB) {
3040 // Perform simple cse.
3041 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
3042 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
3043 Instruction *In = &*I++;
3044
3045 if (!CSEDenseMapInfo::canHandle(In))
3046 continue;
3047
3048 // Check if we can replace this instruction with any of the
3049 // visited instructions.
3050 if (Instruction *V = CSEMap.lookup(In)) {
3051 In->replaceAllUsesWith(V);
3052 In->eraseFromParent();
3053 continue;
3054 }
3055
3056 CSEMap[In] = In;
3057 }
3058}
3059
3060/// Estimate the overhead of scalarizing an instruction. This is a
3061/// convenience wrapper for the type-based getScalarizationOverhead API.
3062static unsigned getScalarizationOverhead(Instruction *I, unsigned VF,
3063 const TargetTransformInfo &TTI) {
3064 if (VF == 1)
3065 return 0;
3066
3067 unsigned Cost = 0;
3068 Type *RetTy = ToVectorTy(I->getType(), VF);
3069 if (!RetTy->isVoidTy() &&
3070 (!isa<LoadInst>(I) ||
3071 !TTI.supportsEfficientVectorElementLoadStore()))
3072 Cost += TTI.getScalarizationOverhead(RetTy, true, false);
3073
3074 // Some targets keep addresses scalar.
3075 if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
3076 return Cost;
3077
3078 if (CallInst *CI = dyn_cast<CallInst>(I)) {
3079 SmallVector<const Value *, 4> Operands(CI->arg_operands());
3080 Cost += TTI.getOperandsScalarizationOverhead(Operands, VF);
3081 }
3082 else if (!isa<StoreInst>(I) ||
3083 !TTI.supportsEfficientVectorElementLoadStore()) {
3084 SmallVector<const Value *, 4> Operands(I->operand_values());
3085 Cost += TTI.getOperandsScalarizationOverhead(Operands, VF);
3086 }
3087
3088 return Cost;
3089}
3090
3091// Estimate cost of a call instruction CI if it were vectorized with factor VF.
3092// Return the cost of the instruction, including scalarization overhead if it's
3093// needed. The flag NeedToScalarize shows if the call needs to be scalarized -
3094// i.e. either vector version isn't available, or is too expensive.
3095static unsigned getVectorCallCost(CallInst *CI, unsigned VF,
3096 const TargetTransformInfo &TTI,
3097 const TargetLibraryInfo *TLI,
3098 bool &NeedToScalarize) {
3099 Function *F = CI->getCalledFunction();
3100 StringRef FnName = CI->getCalledFunction()->getName();
3101 Type *ScalarRetTy = CI->getType();
3102 SmallVector<Type *, 4> Tys, ScalarTys;
3103 for (auto &ArgOp : CI->arg_operands())
3104 ScalarTys.push_back(ArgOp->getType());
3105
3106 // Estimate cost of scalarized vector call. The source operands are assumed
3107 // to be vectors, so we need to extract individual elements from there,
3108 // execute VF scalar calls, and then gather the result into the vector return
3109 // value.
3110 unsigned ScalarCallCost = TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys);
3111 if (VF == 1)
3112 return ScalarCallCost;
3113
3114 // Compute corresponding vector type for return value and arguments.
3115 Type *RetTy = ToVectorTy(ScalarRetTy, VF);
3116 for (Type *ScalarTy : ScalarTys)
3117 Tys.push_back(ToVectorTy(ScalarTy, VF));
3118
3119 // Compute costs of unpacking argument values for the scalar calls and
3120 // packing the return values to a vector.
3121 unsigned ScalarizationCost = getScalarizationOverhead(CI, VF, TTI);
3122
3123 unsigned Cost = ScalarCallCost * VF + ScalarizationCost;
3124
3125 // If we can't emit a vector call for this function, then the currently found
3126 // cost is the cost we need to return.
3127 NeedToScalarize = true;
3128 if (!TLI || !TLI->isFunctionVectorizable(FnName, VF) || CI->isNoBuiltin())
3129 return Cost;
3130
3131 // If the corresponding vector cost is cheaper, return its cost.
3132 unsigned VectorCallCost = TTI.getCallInstrCost(nullptr, RetTy, Tys);
3133 if (VectorCallCost < Cost) {
3134 NeedToScalarize = false;
3135 return VectorCallCost;
3136 }
3137 return Cost;
3138}
3139
3140// Estimate cost of an intrinsic call instruction CI if it were vectorized with
3141// factor VF. Return the cost of the instruction, including scalarization
3142// overhead if it's needed.
3143static unsigned getVectorIntrinsicCost(CallInst *CI, unsigned VF,
3144 const TargetTransformInfo &TTI,
3145 const TargetLibraryInfo *TLI) {
3146 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3147 assert(ID && "Expected intrinsic call!")((ID && "Expected intrinsic call!") ? static_cast<
void> (0) : __assert_fail ("ID && \"Expected intrinsic call!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3147, __PRETTY_FUNCTION__))
;
3148
3149 FastMathFlags FMF;
3150 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
3151 FMF = FPMO->getFastMathFlags();
3152
3153 SmallVector<Value *, 4> Operands(CI->arg_operands());
3154 return TTI.getIntrinsicInstrCost(ID, CI->getType(), Operands, FMF, VF);
3155}
3156
3157static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
3158 auto *I1 = cast<IntegerType>(T1->getVectorElementType());
3159 auto *I2 = cast<IntegerType>(T2->getVectorElementType());
3160 return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
3161}
3162static Type *largestIntegerVectorType(Type *T1, Type *T2) {
3163 auto *I1 = cast<IntegerType>(T1->getVectorElementType());
3164 auto *I2 = cast<IntegerType>(T2->getVectorElementType());
3165 return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
3166}
3167
3168void InnerLoopVectorizer::truncateToMinimalBitwidths() {
3169 // For every instruction `I` in MinBWs, truncate the operands, create a
3170 // truncated version of `I` and reextend its result. InstCombine runs
3171 // later and will remove any ext/trunc pairs.
3172 SmallPtrSet<Value *, 4> Erased;
3173 for (const auto &KV : Cost->getMinimalBitwidths()) {
3174 // If the value wasn't vectorized, we must maintain the original scalar
3175 // type. The absence of the value from VectorLoopValueMap indicates that it
3176 // wasn't vectorized.
3177 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))
3178 continue;
3179 for (unsigned Part = 0; Part < UF; ++Part) {
3180 Value *I = getOrCreateVectorValue(KV.first, Part);
3181 if (Erased.find(I) != Erased.end() || I->use_empty() ||
3182 !isa<Instruction>(I))
3183 continue;
3184 Type *OriginalTy = I->getType();
3185 Type *ScalarTruncatedTy =
3186 IntegerType::get(OriginalTy->getContext(), KV.second);
3187 Type *TruncatedTy = VectorType::get(ScalarTruncatedTy,
3188 OriginalTy->getVectorNumElements());
3189 if (TruncatedTy == OriginalTy)
3190 continue;
3191
3192 IRBuilder<> B(cast<Instruction>(I));
3193 auto ShrinkOperand = [&](Value *V) -> Value * {
3194 if (auto *ZI = dyn_cast<ZExtInst>(V))
3195 if (ZI->getSrcTy() == TruncatedTy)
3196 return ZI->getOperand(0);
3197 return B.CreateZExtOrTrunc(V, TruncatedTy);
3198 };
3199
3200 // The actual instruction modification depends on the instruction type,
3201 // unfortunately.
3202 Value *NewI = nullptr;
3203 if (auto *BO = dyn_cast<BinaryOperator>(I)) {
3204 NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),
3205 ShrinkOperand(BO->getOperand(1)));
3206
3207 // Any wrapping introduced by shrinking this operation shouldn't be
3208 // considered undefined behavior. So, we can't unconditionally copy
3209 // arithmetic wrapping flags to NewI.
3210 cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);
3211 } else if (auto *CI = dyn_cast<ICmpInst>(I)) {
3212 NewI =
3213 B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),
3214 ShrinkOperand(CI->getOperand(1)));
3215 } else if (auto *SI = dyn_cast<SelectInst>(I)) {
3216 NewI = B.CreateSelect(SI->getCondition(),
3217 ShrinkOperand(SI->getTrueValue()),
3218 ShrinkOperand(SI->getFalseValue()));
3219 } else if (auto *CI = dyn_cast<CastInst>(I)) {
3220 switch (CI->getOpcode()) {
3221 default:
3222 llvm_unreachable("Unhandled cast!")::llvm::llvm_unreachable_internal("Unhandled cast!", "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3222)
;
3223 case Instruction::Trunc:
3224 NewI = ShrinkOperand(CI->getOperand(0));
3225 break;
3226 case Instruction::SExt:
3227 NewI = B.CreateSExtOrTrunc(
3228 CI->getOperand(0),
3229 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3230 break;
3231 case Instruction::ZExt:
3232 NewI = B.CreateZExtOrTrunc(
3233 CI->getOperand(0),
3234 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3235 break;
3236 }
3237 } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {
3238 auto Elements0 = SI->getOperand(0)->getType()->getVectorNumElements();
3239 auto *O0 = B.CreateZExtOrTrunc(
3240 SI->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements0));
3241 auto Elements1 = SI->getOperand(1)->getType()->getVectorNumElements();
3242 auto *O1 = B.CreateZExtOrTrunc(
3243 SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
3244
3245 NewI = B.CreateShuffleVector(O0, O1, SI->getMask());
3246 } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
3247 // Don't do anything with the operands, just extend the result.
3248 continue;
3249 } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
3250 auto Elements = IE->getOperand(0)->getType()->getVectorNumElements();
3251 auto *O0 = B.CreateZExtOrTrunc(
3252 IE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));
3253 auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);
3254 NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));
3255 } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
3256 auto Elements = EE->getOperand(0)->getType()->getVectorNumElements();
3257 auto *O0 = B.CreateZExtOrTrunc(
3258 EE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));
3259 NewI = B.CreateExtractElement(O0, EE->getOperand(2));
3260 } else {
3261 // If we don't know what to do, be conservative and don't do anything.
3262 continue;
3263 }
3264
3265 // Lastly, extend the result.
3266 NewI->takeName(cast<Instruction>(I));
3267 Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
3268 I->replaceAllUsesWith(Res);
3269 cast<Instruction>(I)->eraseFromParent();
3270 Erased.insert(I);
3271 VectorLoopValueMap.resetVectorValue(KV.first, Part, Res);
3272 }
3273 }
3274
3275 // We'll have created a bunch of ZExts that are now parentless. Clean up.
3276 for (const auto &KV : Cost->getMinimalBitwidths()) {
3277 // If the value wasn't vectorized, we must maintain the original scalar
3278 // type. The absence of the value from VectorLoopValueMap indicates that it
3279 // wasn't vectorized.
3280 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))
3281 continue;
3282 for (unsigned Part = 0; Part < UF; ++Part) {
3283 Value *I = getOrCreateVectorValue(KV.first, Part);
3284 ZExtInst *Inst = dyn_cast<ZExtInst>(I);
3285 if (Inst && Inst->use_empty()) {
3286 Value *NewI = Inst->getOperand(0);
3287 Inst->eraseFromParent();
3288 VectorLoopValueMap.resetVectorValue(KV.first, Part, NewI);
3289 }
3290 }
3291 }
3292}
3293
3294void InnerLoopVectorizer::fixVectorizedLoop() {
3295 // Insert truncates and extends for any truncated instructions as hints to
3296 // InstCombine.
3297 if (VF > 1)
3298 truncateToMinimalBitwidths();
3299
3300 // Fix widened non-induction PHIs by setting up the PHI operands.
3301 if (OrigPHIsToFix.size()) {
3302 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3303, __PRETTY_FUNCTION__))
3303 "Unexpected non-induction PHIs for fixup in non VPlan-native path")((EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3303, __PRETTY_FUNCTION__))
;
3304 fixNonInductionPHIs();
3305 }
3306
3307 // At this point every instruction in the original loop is widened to a
3308 // vector form. Now we need to fix the recurrences in the loop. These PHI
3309 // nodes are currently empty because we did not want to introduce cycles.
3310 // This is the second stage of vectorizing recurrences.
3311 fixCrossIterationPHIs();
3312
3313 // Update the dominator tree.
3314 //
3315 // FIXME: After creating the structure of the new loop, the dominator tree is
3316 // no longer up-to-date, and it remains that way until we update it
3317 // here. An out-of-date dominator tree is problematic for SCEV,
3318 // because SCEVExpander uses it to guide code generation. The
3319 // vectorizer use SCEVExpanders in several places. Instead, we should
3320 // keep the dominator tree up-to-date as we go.
3321 updateAnalysis();
3322
3323 // Fix-up external users of the induction variables.
3324 for (auto &Entry : *Legal->getInductionVars())
3325 fixupIVUsers(Entry.first, Entry.second,
3326 getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)),
3327 IVEndValues[Entry.first], LoopMiddleBlock);
3328
3329 fixLCSSAPHIs();
3330 for (Instruction *PI : PredicatedInstructions)
3331 sinkScalarOperands(&*PI);
3332
3333 // Remove redundant induction instructions.
3334 cse(LoopVectorBody);
3335}
3336
3337void InnerLoopVectorizer::fixCrossIterationPHIs() {
3338 // In order to support recurrences we need to be able to vectorize Phi nodes.
3339 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
3340 // stage #2: We now need to fix the recurrences by adding incoming edges to
3341 // the currently empty PHI nodes. At this point every instruction in the
3342 // original loop is widened to a vector form so we can use them to construct
3343 // the incoming edges.
3344 for (PHINode &Phi : OrigLoop->getHeader()->phis()) {
3345 // Handle first-order recurrences and reductions that need to be fixed.
3346 if (Legal->isFirstOrderRecurrence(&Phi))
3347 fixFirstOrderRecurrence(&Phi);
3348 else if (Legal->isReductionVariable(&Phi))
3349 fixReduction(&Phi);
3350 }
3351}
3352
3353void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi) {
3354 // This is the second phase of vectorizing first-order recurrences. An
3355 // overview of the transformation is described below. Suppose we have the
3356 // following loop.
3357 //
3358 // for (int i = 0; i < n; ++i)
3359 // b[i] = a[i] - a[i - 1];
3360 //
3361 // There is a first-order recurrence on "a". For this loop, the shorthand
3362 // scalar IR looks like:
3363 //
3364 // scalar.ph:
3365 // s_init = a[-1]
3366 // br scalar.body
3367 //
3368 // scalar.body:
3369 // i = phi [0, scalar.ph], [i+1, scalar.body]
3370 // s1 = phi [s_init, scalar.ph], [s2, scalar.body]
3371 // s2 = a[i]
3372 // b[i] = s2 - s1
3373 // br cond, scalar.body, ...
3374 //
3375 // In this example, s1 is a recurrence because it's value depends on the
3376 // previous iteration. In the first phase of vectorization, we created a
3377 // temporary value for s1. We now complete the vectorization and produce the
3378 // shorthand vector IR shown below (for VF = 4, UF = 1).
3379 //
3380 // vector.ph:
3381 // v_init = vector(..., ..., ..., a[-1])
3382 // br vector.body
3383 //
3384 // vector.body
3385 // i = phi [0, vector.ph], [i+4, vector.body]
3386 // v1 = phi [v_init, vector.ph], [v2, vector.body]
3387 // v2 = a[i, i+1, i+2, i+3];
3388 // v3 = vector(v1(3), v2(0, 1, 2))
3389 // b[i, i+1, i+2, i+3] = v2 - v3
3390 // br cond, vector.body, middle.block
3391 //
3392 // middle.block:
3393 // x = v2(3)
3394 // br scalar.ph
3395 //
3396 // scalar.ph:
3397 // s_init = phi [x, middle.block], [a[-1], otherwise]
3398 // br scalar.body
3399 //
3400 // After execution completes the vector loop, we extract the next value of
3401 // the recurrence (x) to use as the initial value in the scalar loop.
3402
3403 // Get the original loop preheader and single loop latch.
3404 auto *Preheader = OrigLoop->getLoopPreheader();
3405 auto *Latch = OrigLoop->getLoopLatch();
3406
3407 // Get the initial and previous values of the scalar recurrence.
3408 auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader);
3409 auto *Previous = Phi->getIncomingValueForBlock(Latch);
3410
3411 // Create a vector from the initial value.
3412 auto *VectorInit = ScalarInit;
3413 if (VF > 1) {
3414 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
3415 VectorInit = Builder.CreateInsertElement(
3416 UndefValue::get(VectorType::get(VectorInit->getType(), VF)), VectorInit,
3417 Builder.getInt32(VF - 1), "vector.recur.init");
3418 }
3419
3420 // We constructed a temporary phi node in the first phase of vectorization.
3421 // This phi node will eventually be deleted.
3422 Builder.SetInsertPoint(
3423 cast<Instruction>(VectorLoopValueMap.getVectorValue(Phi, 0)));
3424
3425 // Create a phi node for the new recurrence. The current value will either be
3426 // the initial value inserted into a vector or loop-varying vector value.
3427 auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur");
3428 VecPhi->addIncoming(VectorInit, LoopVectorPreHeader);
3429
3430 // Get the vectorized previous value of the last part UF - 1. It appears last
3431 // among all unrolled iterations, due to the order of their construction.
3432 Value *PreviousLastPart = getOrCreateVectorValue(Previous, UF - 1);
3433
3434 // Set the insertion point after the previous value if it is an instruction.
3435 // Note that the previous value may have been constant-folded so it is not
3436 // guaranteed to be an instruction in the vector loop. Also, if the previous
3437 // value is a phi node, we should insert after all the phi nodes to avoid
3438 // breaking basic block verification.
3439 if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart) ||
3440 isa<PHINode>(PreviousLastPart))
3441 Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());
3442 else
3443 Builder.SetInsertPoint(
3444 &*++BasicBlock::iterator(cast<Instruction>(PreviousLastPart)));
3445
3446 // We will construct a vector for the recurrence by combining the values for
3447 // the current and previous iterations. This is the required shuffle mask.
3448 SmallVector<Constant *, 8> ShuffleMask(VF);
3449 ShuffleMask[0] = Builder.getInt32(VF - 1);
3450 for (unsigned I = 1; I < VF; ++I)
3451 ShuffleMask[I] = Builder.getInt32(I + VF - 1);
3452
3453 // The vector from which to take the initial value for the current iteration
3454 // (actual or unrolled). Initially, this is the vector phi node.
3455 Value *Incoming = VecPhi;
3456
3457 // Shuffle the current and previous vector and update the vector parts.
3458 for (unsigned Part = 0; Part < UF; ++Part) {
3459 Value *PreviousPart = getOrCreateVectorValue(Previous, Part);
3460 Value *PhiPart = VectorLoopValueMap.getVectorValue(Phi, Part);
3461 auto *Shuffle =
3462 VF > 1 ? Builder.CreateShuffleVector(Incoming, PreviousPart,
3463 ConstantVector::get(ShuffleMask))
3464 : Incoming;
3465 PhiPart->replaceAllUsesWith(Shuffle);
3466 cast<Instruction>(PhiPart)->eraseFromParent();
3467 VectorLoopValueMap.resetVectorValue(Phi, Part, Shuffle);
3468 Incoming = PreviousPart;
3469 }
3470
3471 // Fix the latch value of the new recurrence in the vector loop.
3472 VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
3473
3474 // Extract the last vector element in the middle block. This will be the
3475 // initial value for the recurrence when jumping to the scalar loop.
3476 auto *ExtractForScalar = Incoming;
3477 if (VF > 1) {
3478 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
3479 ExtractForScalar = Builder.CreateExtractElement(
3480 ExtractForScalar, Builder.getInt32(VF - 1), "vector.recur.extract");
3481 }
3482 // Extract the second last element in the middle block if the
3483 // Phi is used outside the loop. We need to extract the phi itself
3484 // and not the last element (the phi update in the current iteration). This
3485 // will be the value when jumping to the exit block from the LoopMiddleBlock,
3486 // when the scalar loop is not run at all.
3487 Value *ExtractForPhiUsedOutsideLoop = nullptr;
3488 if (VF > 1)
3489 ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(
3490 Incoming, Builder.getInt32(VF - 2), "vector.recur.extract.for.phi");
3491 // When loop is unrolled without vectorizing, initialize
3492 // ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value of
3493 // `Incoming`. This is analogous to the vectorized case above: extracting the
3494 // second last element when VF > 1.
3495 else if (UF > 1)
3496 ExtractForPhiUsedOutsideLoop = getOrCreateVectorValue(Previous, UF - 2);
3497
3498 // Fix the initial value of the original recurrence in the scalar loop.
3499 Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
3500 auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
3501 for (auto *BB : predecessors(LoopScalarPreHeader)) {
3502 auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
3503 Start->addIncoming(Incoming, BB);
3504 }
3505
3506 Phi->setIncomingValue(Phi->getBasicBlockIndex(LoopScalarPreHeader), Start);
3507 Phi->setName("scalar.recur");
3508
3509 // Finally, fix users of the recurrence outside the loop. The users will need
3510 // either the last value of the scalar recurrence or the last value of the
3511 // vector recurrence we extracted in the middle block. Since the loop is in
3512 // LCSSA form, we just need to find all the phi nodes for the original scalar
3513 // recurrence in the exit block, and then add an edge for the middle block.
3514 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
3515 if (LCSSAPhi.getIncomingValue(0) == Phi) {
3516 LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
3517 }
3518 }
3519}
3520
3521void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
3522 Constant *Zero = Builder.getInt32(0);
3523
3524 // Get it's reduction variable descriptor.
3525 assert(Legal->isReductionVariable(Phi) &&((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3526, __PRETTY_FUNCTION__))
3526 "Unable to find the reduction variable")((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3526, __PRETTY_FUNCTION__))
;
3527 RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[Phi];
3528
3529 RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
3530 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
3531 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
3532 RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
3533 RdxDesc.getMinMaxRecurrenceKind();
3534 setDebugLocFromInst(Builder, ReductionStartValue);
3535
3536 // We need to generate a reduction vector from the incoming scalar.
3537 // To do so, we need to generate the 'identity' vector and override
3538 // one of the elements with the incoming scalar reduction. We need
3539 // to do it in the vector-loop preheader.
3540 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
3541
3542 // This is the vector-clone of the value that leaves the loop.
3543 Type *VecTy = getOrCreateVectorValue(LoopExitInst, 0)->getType();
3544
3545 // Find the reduction identity variable. Zero for addition, or, xor,
3546 // one for multiplication, -1 for And.
3547 Value *Identity;
3548 Value *VectorStart;
3549 if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
3550 RK == RecurrenceDescriptor::RK_FloatMinMax) {
3551 // MinMax reduction have the start value as their identify.
3552 if (VF == 1) {
3553 VectorStart = Identity = ReductionStartValue;
3554 } else {
3555 VectorStart = Identity =
3556 Builder.CreateVectorSplat(VF, ReductionStartValue, "minmax.ident");
3557 }
3558 } else {
3559 // Handle other reduction kinds:
3560 Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
3561 RK, VecTy->getScalarType());
3562 if (VF == 1) {
3563 Identity = Iden;
3564 // This vector is the Identity vector where the first element is the
3565 // incoming scalar reduction.
3566 VectorStart = ReductionStartValue;
3567 } else {
3568 Identity = ConstantVector::getSplat(VF, Iden);
3569
3570 // This vector is the Identity vector where the first element is the
3571 // incoming scalar reduction.
3572 VectorStart =
3573 Builder.CreateInsertElement(Identity, ReductionStartValue, Zero);
3574 }
3575 }
3576
3577 // Fix the vector-loop phi.
3578
3579 // Reductions do not have to start at zero. They can start with
3580 // any loop invariant values.
3581 BasicBlock *Latch = OrigLoop->getLoopLatch();
3582 Value *LoopVal = Phi->getIncomingValueForBlock(Latch);
3583 for (unsigned Part = 0; Part < UF; ++Part) {
3584 Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part);
3585 Value *Val = getOrCreateVectorValue(LoopVal, Part);
3586 // Make sure to add the reduction stat value only to the
3587 // first unroll part.
3588 Value *StartVal = (Part == 0) ? VectorStart : Identity;
3589 cast<PHINode>(VecRdxPhi)->addIncoming(StartVal, LoopVectorPreHeader);
3590 cast<PHINode>(VecRdxPhi)
3591 ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
3592 }
3593
3594 // Before each round, move the insertion point right between
3595 // the PHIs and the values we are going to write.
3596 // This allows us to write both PHINodes and the extractelement
3597 // instructions.
3598 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
3599
3600 setDebugLocFromInst(Builder, LoopExitInst);
3601
3602 // If the vector reduction can be performed in a smaller type, we truncate
3603 // then extend the loop exit value to enable InstCombine to evaluate the
3604 // entire expression in the smaller type.
3605 if (VF > 1 && Phi->getType() != RdxDesc.getRecurrenceType()) {
3606 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
3607 Builder.SetInsertPoint(
3608 LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
3609 VectorParts RdxParts(UF);
3610 for (unsigned Part = 0; Part < UF; ++Part) {
3611 RdxParts[Part] = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
3612 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
3613 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
3614 : Builder.CreateZExt(Trunc, VecTy);
3615 for (Value::user_iterator UI = RdxParts[Part]->user_begin();
3616 UI != RdxParts[Part]->user_end();)
3617 if (*UI != Trunc) {
3618 (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd);
3619 RdxParts[Part] = Extnd;
3620 } else {
3621 ++UI;
3622 }
3623 }
3624 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
3625 for (unsigned Part = 0; Part < UF; ++Part) {
3626 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
3627 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, RdxParts[Part]);
3628 }
3629 }
3630
3631 // Reduce all of the unrolled parts into a single vector.
3632 Value *ReducedPartRdx = VectorLoopValueMap.getVectorValue(LoopExitInst, 0);
3633 unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK);
3634 setDebugLocFromInst(Builder, ReducedPartRdx);
3635 for (unsigned Part = 1; Part < UF; ++Part) {
3636 Value *RdxPart = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
3637 if (Op != Instruction::ICmp && Op != Instruction::FCmp)
3638 // Floating point operations had to be 'fast' to enable the reduction.
3639 ReducedPartRdx = addFastMathFlag(
3640 Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxPart,
3641 ReducedPartRdx, "bin.rdx"),
3642 RdxDesc.getFastMathFlags());
3643 else
3644 ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
3645 RdxPart);
3646 }
3647
3648 if (VF > 1) {
3649 bool NoNaN = Legal->hasFunNoNaNAttr();
3650 ReducedPartRdx =
3651 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, NoNaN);
3652 // If the reduction can be performed in a smaller type, we need to extend
3653 // the reduction to the wider type before we branch to the original loop.
3654 if (Phi->getType() != RdxDesc.getRecurrenceType())
3655 ReducedPartRdx =
3656 RdxDesc.isSigned()
3657 ? Builder.CreateSExt(ReducedPartRdx, Phi->getType())
3658 : Builder.CreateZExt(ReducedPartRdx, Phi->getType());
3659 }
3660
3661 // Create a phi node that merges control-flow from the backedge-taken check
3662 // block and the middle block.
3663 PHINode *BCBlockPhi = PHINode::Create(Phi->getType(), 2, "bc.merge.rdx",
3664 LoopScalarPreHeader->getTerminator());
3665 for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
3666 BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
3667 BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);
3668
3669 // Now, we need to fix the users of the reduction variable
3670 // inside and outside of the scalar remainder loop.
3671 // We know that the loop is in LCSSA form. We need to update the
3672 // PHI nodes in the exit blocks.
3673 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
3674 // All PHINodes need to have a single entry edge, or two if
3675 // we already fixed them.
3676 assert(LCSSAPhi.getNumIncomingValues() < 3 && "Invalid LCSSA PHI")((LCSSAPhi.getNumIncomingValues() < 3 && "Invalid LCSSA PHI"
) ? static_cast<void> (0) : __assert_fail ("LCSSAPhi.getNumIncomingValues() < 3 && \"Invalid LCSSA PHI\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3676, __PRETTY_FUNCTION__))
;
3677
3678 // We found a reduction value exit-PHI. Update it with the
3679 // incoming bypass edge.
3680 if (LCSSAPhi.getIncomingValue(0) == LoopExitInst)
3681 LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);
3682 } // end of the LCSSA phi scan.
3683
3684 // Fix the scalar loop reduction variable with the incoming reduction sum
3685 // from the vector body and from the backedge value.
3686 int IncomingEdgeBlockIdx =
3687 Phi->getBasicBlockIndex(OrigLoop->getLoopLatch());
3688 assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index")((IncomingEdgeBlockIdx >= 0 && "Invalid block index"
) ? static_cast<void> (0) : __assert_fail ("IncomingEdgeBlockIdx >= 0 && \"Invalid block index\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3688, __PRETTY_FUNCTION__))
;
3689 // Pick the other block.
3690 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
3691 Phi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
3692 Phi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
3693}
3694
3695void InnerLoopVectorizer::fixLCSSAPHIs() {
3696 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
3697 if (LCSSAPhi.getNumIncomingValues() == 1) {
3698 auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
3699 // Non-instruction incoming values will have only one value.
3700 unsigned LastLane = 0;
3701 if (isa<Instruction>(IncomingValue))
3702 LastLane = Cost->isUniformAfterVectorization(
3703 cast<Instruction>(IncomingValue), VF)
3704 ? 0
3705 : VF - 1;
3706 // Can be a loop invariant incoming value or the last scalar value to be
3707 // extracted from the vectorized loop.
3708 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
3709 Value *lastIncomingValue =
3710 getOrCreateScalarValue(IncomingValue, { UF - 1, LastLane });
3711 LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock);
3712 }
3713 }
3714}
3715
3716void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
3717 // The basic block and loop containing the predicated instruction.
3718 auto *PredBB = PredInst->getParent();
3719 auto *VectorLoop = LI->getLoopFor(PredBB);
3720
3721 // Initialize a worklist with the operands of the predicated instruction.
3722 SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());
3723
3724 // Holds instructions that we need to analyze again. An instruction may be
3725 // reanalyzed if we don't yet know if we can sink it or not.
3726 SmallVector<Instruction *, 8> InstsToReanalyze;
3727
3728 // Returns true if a given use occurs in the predicated block. Phi nodes use
3729 // their operands in their corresponding predecessor blocks.
3730 auto isBlockOfUsePredicated = [&](Use &U) -> bool {
3731 auto *I = cast<Instruction>(U.getUser());
3732 BasicBlock *BB = I->getParent();
3733 if (auto *Phi = dyn_cast<PHINode>(I))
3734 BB = Phi->getIncomingBlock(
3735 PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
3736 return BB == PredBB;
3737 };
3738
3739 // Iteratively sink the scalarized operands of the predicated instruction
3740 // into the block we created for it. When an instruction is sunk, it's
3741 // operands are then added to the worklist. The algorithm ends after one pass
3742 // through the worklist doesn't sink a single instruction.
3743 bool Changed;
3744 do {
3745 // Add the instructions that need to be reanalyzed to the worklist, and
3746 // reset the changed indicator.
3747 Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());
3748 InstsToReanalyze.clear();
3749 Changed = false;
3750
3751 while (!Worklist.empty()) {
3752 auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());
3753
3754 // We can't sink an instruction if it is a phi node, is already in the
3755 // predicated block, is not in the loop, or may have side effects.
3756 if (!I || isa<PHINode>(I) || I->getParent() == PredBB ||
3757 !VectorLoop->contains(I) || I->mayHaveSideEffects())
3758 continue;
3759
3760 // It's legal to sink the instruction if all its uses occur in the
3761 // predicated block. Otherwise, there's nothing to do yet, and we may
3762 // need to reanalyze the instruction.
3763 if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) {
3764 InstsToReanalyze.push_back(I);
3765 continue;
3766 }
3767
3768 // Move the instruction to the beginning of the predicated block, and add
3769 // it's operands to the worklist.
3770 I->moveBefore(&*PredBB->getFirstInsertionPt());
3771 Worklist.insert(I->op_begin(), I->op_end());
3772
3773 // The sinking may have enabled other instructions to be sunk, so we will
3774 // need to iterate.
3775 Changed = true;
3776 }
3777 } while (Changed);
3778}
3779
3780void InnerLoopVectorizer::fixNonInductionPHIs() {
3781 for (PHINode *OrigPhi : OrigPHIsToFix) {
3782 PHINode *NewPhi =
3783 cast<PHINode>(VectorLoopValueMap.getVectorValue(OrigPhi, 0));
3784 unsigned NumIncomingValues = OrigPhi->getNumIncomingValues();
3785
3786 SmallVector<BasicBlock *, 2> ScalarBBPredecessors(
3787 predecessors(OrigPhi->getParent()));
3788 SmallVector<BasicBlock *, 2> VectorBBPredecessors(
3789 predecessors(NewPhi->getParent()));
3790 assert(ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&((ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&
"Scalar and Vector BB should have the same number of predecessors"
) ? static_cast<void> (0) : __assert_fail ("ScalarBBPredecessors.size() == VectorBBPredecessors.size() && \"Scalar and Vector BB should have the same number of predecessors\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3791, __PRETTY_FUNCTION__))
3791 "Scalar and Vector BB should have the same number of predecessors")((ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&
"Scalar and Vector BB should have the same number of predecessors"
) ? static_cast<void> (0) : __assert_fail ("ScalarBBPredecessors.size() == VectorBBPredecessors.size() && \"Scalar and Vector BB should have the same number of predecessors\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3791, __PRETTY_FUNCTION__))
;
3792
3793 // The insertion point in Builder may be invalidated by the time we get
3794 // here. Force the Builder insertion point to something valid so that we do
3795 // not run into issues during insertion point restore in
3796 // getOrCreateVectorValue calls below.
3797 Builder.SetInsertPoint(NewPhi);
3798
3799 // The predecessor order is preserved and we can rely on mapping between
3800 // scalar and vector block predecessors.
3801 for (unsigned i = 0; i < NumIncomingValues; ++i) {
3802 BasicBlock *NewPredBB = VectorBBPredecessors[i];
3803
3804 // When looking up the new scalar/vector values to fix up, use incoming
3805 // values from original phi.
3806 Value *ScIncV =
3807 OrigPhi->getIncomingValueForBlock(ScalarBBPredecessors[i]);
3808
3809 // Scalar incoming value may need a broadcast
3810 Value *NewIncV = getOrCreateVectorValue(ScIncV, 0);
3811 NewPhi->addIncoming(NewIncV, NewPredBB);
3812 }
3813 }
3814}
3815
3816void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, unsigned UF,
3817 unsigned VF) {
3818 PHINode *P = cast<PHINode>(PN);
3819 if (EnableVPlanNativePath) {
3820 // Currently we enter here in the VPlan-native path for non-induction
3821 // PHIs where all control flow is uniform. We simply widen these PHIs.
3822 // Create a vector phi with no operands - the vector phi operands will be
3823 // set at the end of vector code generation.
3824 Type *VecTy =
3825 (VF == 1) ? PN->getType() : VectorType::get(PN->getType(), VF);
3826 Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
3827 VectorLoopValueMap.setVectorValue(P, 0, VecPhi);
3828 OrigPHIsToFix.push_back(P);
3829
3830 return;
3831 }
3832
3833 assert(PN->getParent() == OrigLoop->getHeader() &&((PN->getParent() == OrigLoop->getHeader() && "Non-header phis should have been handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3834, __PRETTY_FUNCTION__))
3834 "Non-header phis should have been handled elsewhere")((PN->getParent() == OrigLoop->getHeader() && "Non-header phis should have been handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3834, __PRETTY_FUNCTION__))
;
3835
3836 // In order to support recurrences we need to be able to vectorize Phi nodes.
3837 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
3838 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
3839 // this value when we vectorize all of the instructions that use the PHI.
3840 if (Legal->isReductionVariable(P) || Legal->isFirstOrderRecurrence(P)) {
3841 for (unsigned Part = 0; Part < UF; ++Part) {
3842 // This is phase one of vectorizing PHIs.
3843 Type *VecTy =
3844 (VF == 1) ? PN->getType() : VectorType::get(PN->getType(), VF);
3845 Value *EntryPart = PHINode::Create(
3846 VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
3847 VectorLoopValueMap.setVectorValue(P, Part, EntryPart);
3848 }
3849 return;
3850 }
3851
3852 setDebugLocFromInst(Builder, P);
3853
3854 // This PHINode must be an induction variable.
3855 // Make sure that we know about it.
3856 assert(Legal->getInductionVars()->count(P) && "Not an induction variable")((Legal->getInductionVars()->count(P) && "Not an induction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->getInductionVars()->count(P) && \"Not an induction variable\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3856, __PRETTY_FUNCTION__))
;
3857
3858 InductionDescriptor II = Legal->getInductionVars()->lookup(P);
3859 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
3860
3861 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
3862 // which can be found from the original scalar operations.
3863 switch (II.getKind()) {
3864 case InductionDescriptor::IK_NoInduction:
3865 llvm_unreachable("Unknown induction")::llvm::llvm_unreachable_internal("Unknown induction", "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3865)
;
3866 case InductionDescriptor::IK_IntInduction:
3867 case InductionDescriptor::IK_FpInduction:
3868 llvm_unreachable("Integer/fp induction is handled elsewhere.")::llvm::llvm_unreachable_internal("Integer/fp induction is handled elsewhere."
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3868)
;
3869 case InductionDescriptor::IK_PtrInduction: {
3870 // Handle the pointer induction variable case.
3871 assert(P->getType()->isPointerTy() && "Unexpected type.")((P->getType()->isPointerTy() && "Unexpected type."
) ? static_cast<void> (0) : __assert_fail ("P->getType()->isPointerTy() && \"Unexpected type.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3871, __PRETTY_FUNCTION__))
;
3872 // This is the normalized GEP that starts counting at zero.
3873 Value *PtrInd = Induction;
3874 PtrInd = Builder.CreateSExtOrTrunc(PtrInd, II.getStep()->getType());
3875 // Determine the number of scalars we need to generate for each unroll
3876 // iteration. If the instruction is uniform, we only need to generate the
3877 // first lane. Otherwise, we generate all VF values.
3878 unsigned Lanes = Cost->isUniformAfterVectorization(P, VF) ? 1 : VF;
3879 // These are the scalar results. Notice that we don't generate vector GEPs
3880 // because scalar GEPs result in better code.
3881 for (unsigned Part = 0; Part < UF; ++Part) {
3882 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
3883 Constant *Idx = ConstantInt::get(PtrInd->getType(), Lane + Part * VF);
3884 Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
3885 Value *SclrGep =
3886 emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
3887 SclrGep->setName("next.gep");
3888 VectorLoopValueMap.setScalarValue(P, {Part, Lane}, SclrGep);
3889 }
3890 }
3891 return;
3892 }
3893 }
3894}
3895
3896/// A helper function for checking whether an integer division-related
3897/// instruction may divide by zero (in which case it must be predicated if
3898/// executed conditionally in the scalar code).
3899/// TODO: It may be worthwhile to generalize and check isKnownNonZero().
3900/// Non-zero divisors that are non compile-time constants will not be
3901/// converted into multiplication, so we will still end up scalarizing
3902/// the division, but can do so w/o predication.
3903static bool mayDivideByZero(Instruction &I) {
3904 assert((I.getOpcode() == Instruction::UDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3908, __PRETTY_FUNCTION__))
3905 I.getOpcode() == Instruction::SDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3908, __PRETTY_FUNCTION__))
3906 I.getOpcode() == Instruction::URem ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3908, __PRETTY_FUNCTION__))
3907 I.getOpcode() == Instruction::SRem) &&(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3908, __PRETTY_FUNCTION__))
3908 "Unexpected instruction")(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3908, __PRETTY_FUNCTION__))
;
3909 Value *Divisor = I.getOperand(1);
3910 auto *CInt = dyn_cast<ConstantInt>(Divisor);
3911 return !CInt || CInt->isZero();
3912}
3913
3914void InnerLoopVectorizer::widenInstruction(Instruction &I) {
3915 switch (I.getOpcode()) {
3916 case Instruction::Br:
3917 case Instruction::PHI:
3918 llvm_unreachable("This instruction is handled by a different recipe.")::llvm::llvm_unreachable_internal("This instruction is handled by a different recipe."
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3918)
;
3919 case Instruction::GetElementPtr: {
3920 // Construct a vector GEP by widening the operands of the scalar GEP as
3921 // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
3922 // results in a vector of pointers when at least one operand of the GEP
3923 // is vector-typed. Thus, to keep the representation compact, we only use
3924 // vector-typed operands for loop-varying values.
3925 auto *GEP = cast<GetElementPtrInst>(&I);
3926
3927 if (VF > 1 && OrigLoop->hasLoopInvariantOperands(GEP)) {
3928 // If we are vectorizing, but the GEP has only loop-invariant operands,
3929 // the GEP we build (by only using vector-typed operands for
3930 // loop-varying values) would be a scalar pointer. Thus, to ensure we
3931 // produce a vector of pointers, we need to either arbitrarily pick an
3932 // operand to broadcast, or broadcast a clone of the original GEP.
3933 // Here, we broadcast a clone of the original.
3934 //
3935 // TODO: If at some point we decide to scalarize instructions having
3936 // loop-invariant operands, this special case will no longer be
3937 // required. We would add the scalarization decision to
3938 // collectLoopScalars() and teach getVectorValue() to broadcast
3939 // the lane-zero scalar value.
3940 auto *Clone = Builder.Insert(GEP->clone());
3941 for (unsigned Part = 0; Part < UF; ++Part) {
3942 Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);
3943 VectorLoopValueMap.setVectorValue(&I, Part, EntryPart);
3944 addMetadata(EntryPart, GEP);
3945 }
3946 } else {
3947 // If the GEP has at least one loop-varying operand, we are sure to
3948 // produce a vector of pointers. But if we are only unrolling, we want
3949 // to produce a scalar GEP for each unroll part. Thus, the GEP we
3950 // produce with the code below will be scalar (if VF == 1) or vector
3951 // (otherwise). Note that for the unroll-only case, we still maintain
3952 // values in the vector mapping with initVector, as we do for other
3953 // instructions.
3954 for (unsigned Part = 0; Part < UF; ++Part) {
3955 // The pointer operand of the new GEP. If it's loop-invariant, we
3956 // won't broadcast it.
3957 auto *Ptr =
3958 OrigLoop->isLoopInvariant(GEP->getPointerOperand())
3959 ? GEP->getPointerOperand()
3960 : getOrCreateVectorValue(GEP->getPointerOperand(), Part);
3961
3962 // Collect all the indices for the new GEP. If any index is
3963 // loop-invariant, we won't broadcast it.
3964 SmallVector<Value *, 4> Indices;
3965 for (auto &U : make_range(GEP->idx_begin(), GEP->idx_end())) {
3966 if (OrigLoop->isLoopInvariant(U.get()))
3967 Indices.push_back(U.get());
3968 else
3969 Indices.push_back(getOrCreateVectorValue(U.get(), Part));
3970 }
3971
3972 // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
3973 // but it should be a vector, otherwise.
3974 auto *NewGEP =
3975 GEP->isInBounds()
3976 ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
3977 Indices)
3978 : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
3979 assert((VF == 1 || NewGEP->getType()->isVectorTy()) &&(((VF == 1 || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF == 1 || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3980, __PRETTY_FUNCTION__))
3980 "NewGEP is not a pointer vector")(((VF == 1 || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF == 1 || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3980, __PRETTY_FUNCTION__))
;
3981 VectorLoopValueMap.setVectorValue(&I, Part, NewGEP);
3982 addMetadata(NewGEP, GEP);
3983 }
3984 }
3985
3986 break;
3987 }
3988 case Instruction::UDiv:
3989 case Instruction::SDiv:
3990 case Instruction::SRem:
3991 case Instruction::URem:
3992 case Instruction::Add:
3993 case Instruction::FAdd:
3994 case Instruction::Sub:
3995 case Instruction::FSub:
3996 case Instruction::Mul:
3997 case Instruction::FMul:
3998 case Instruction::FDiv:
3999 case Instruction::FRem:
4000 case Instruction::Shl:
4001 case Instruction::LShr:
4002 case Instruction::AShr:
4003 case Instruction::And:
4004 case Instruction::Or:
4005 case Instruction::Xor: {
4006 // Just widen binops.
4007 auto *BinOp = cast<BinaryOperator>(&I);
4008 setDebugLocFromInst(Builder, BinOp);
4009
4010 for (unsigned Part = 0; Part < UF; ++Part) {
4011 Value *A = getOrCreateVectorValue(BinOp->getOperand(0), Part);
4012 Value *B = getOrCreateVectorValue(BinOp->getOperand(1), Part);
4013 Value *V = Builder.CreateBinOp(BinOp->getOpcode(), A, B);
4014
4015 if (BinaryOperator *VecOp = dyn_cast<BinaryOperator>(V))
4016 VecOp->copyIRFlags(BinOp);
4017
4018 // Use this vector value for all users of the original instruction.
4019 VectorLoopValueMap.setVectorValue(&I, Part, V);
4020 addMetadata(V, BinOp);
4021 }
4022
4023 break;
4024 }
4025 case Instruction::Select: {
4026 // Widen selects.
4027 // If the selector is loop invariant we can create a select
4028 // instruction with a scalar condition. Otherwise, use vector-select.
4029 auto *SE = PSE.getSE();
4030 bool InvariantCond =
4031 SE->isLoopInvariant(PSE.getSCEV(I.getOperand(0)), OrigLoop);
4032 setDebugLocFromInst(Builder, &I);
4033
4034 // The condition can be loop invariant but still defined inside the
4035 // loop. This means that we can't just use the original 'cond' value.
4036 // We have to take the 'vectorized' value and pick the first lane.
4037 // Instcombine will make this a no-op.
4038
4039 auto *ScalarCond = getOrCreateScalarValue(I.getOperand(0), {0, 0});
4040
4041 for (unsigned Part = 0; Part < UF; ++Part) {
4042 Value *Cond = getOrCreateVectorValue(I.getOperand(0), Part);
4043 Value *Op0 = getOrCreateVectorValue(I.getOperand(1), Part);
4044 Value *Op1 = getOrCreateVectorValue(I.getOperand(2), Part);
4045 Value *Sel =
4046 Builder.CreateSelect(InvariantCond ? ScalarCond : Cond, Op0, Op1);
4047 VectorLoopValueMap.setVectorValue(&I, Part, Sel);
4048 addMetadata(Sel, &I);
4049 }
4050
4051 break;
4052 }
4053
4054 case Instruction::ICmp:
4055 case Instruction::FCmp: {
4056 // Widen compares. Generate vector compares.
4057 bool FCmp = (I.getOpcode() == Instruction::FCmp);
4058 auto *Cmp = dyn_cast<CmpInst>(&I);
4059 setDebugLocFromInst(Builder, Cmp);
4060 for (unsigned Part = 0; Part < UF; ++Part) {
4061 Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part);
4062 Value *B = getOrCreateVectorValue(Cmp->getOperand(1), Part);
4063 Value *C = nullptr;
4064 if (FCmp) {
4065 // Propagate fast math flags.
4066 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
4067 Builder.setFastMathFlags(Cmp->getFastMathFlags());
4068 C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
4069 } else {
4070 C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
4071 }
4072 VectorLoopValueMap.setVectorValue(&I, Part, C);
4073 addMetadata(C, &I);
4074 }
4075
4076 break;
4077 }
4078
4079 case Instruction::ZExt:
4080 case Instruction::SExt:
4081 case Instruction::FPToUI:
4082 case Instruction::FPToSI:
4083 case Instruction::FPExt:
4084 case Instruction::PtrToInt:
4085 case Instruction::IntToPtr:
4086 case Instruction::SIToFP:
4087 case Instruction::UIToFP:
4088 case Instruction::Trunc:
4089 case Instruction::FPTrunc:
4090 case Instruction::BitCast: {
4091 auto *CI = dyn_cast<CastInst>(&I);
4092 setDebugLocFromInst(Builder, CI);
4093
4094 /// Vectorize casts.
4095 Type *DestTy =
4096 (VF == 1) ? CI->getType() : VectorType::get(CI->getType(), VF);
4097
4098 for (unsigned Part = 0; Part < UF; ++Part) {
4099 Value *A = getOrCreateVectorValue(CI->getOperand(0), Part);
4100 Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
4101 VectorLoopValueMap.setVectorValue(&I, Part, Cast);
4102 addMetadata(Cast, &I);
4103 }
4104 break;
4105 }
4106
4107 case Instruction::Call: {
4108 // Ignore dbg intrinsics.
4109 if (isa<DbgInfoIntrinsic>(I))
4110 break;
4111 setDebugLocFromInst(Builder, &I);
4112
4113 Module *M = I.getParent()->getParent()->getParent();
4114 auto *CI = cast<CallInst>(&I);
4115
4116 StringRef FnName = CI->getCalledFunction()->getName();
4117 Function *F = CI->getCalledFunction();
4118 Type *RetTy = ToVectorTy(CI->getType(), VF);
4119 SmallVector<Type *, 4> Tys;
4120 for (Value *ArgOperand : CI->arg_operands())
4121 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
4122
4123 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4124
4125 // The flag shows whether we use Intrinsic or a usual Call for vectorized
4126 // version of the instruction.
4127 // Is it beneficial to perform intrinsic call compared to lib call?
4128 bool NeedToScalarize;
4129 unsigned CallCost = getVectorCallCost(CI, VF, *TTI, TLI, NeedToScalarize);
4130 bool UseVectorIntrinsic =
4131 ID && getVectorIntrinsicCost(CI, VF, *TTI, TLI) <= CallCost;
4132 assert((UseVectorIntrinsic || !NeedToScalarize) &&(((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."
) ? static_cast<void> (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4133, __PRETTY_FUNCTION__))
4133 "Instruction should be scalarized elsewhere.")(((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."
) ? static_cast<void> (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4133, __PRETTY_FUNCTION__))
;
4134
4135 for (unsigned Part = 0; Part < UF; ++Part) {
4136 SmallVector<Value *, 4> Args;
4137 for (unsigned i = 0, ie = CI->getNumArgOperands(); i != ie; ++i) {
4138 Value *Arg = CI->getArgOperand(i);
4139 // Some intrinsics have a scalar argument - don't replace it with a
4140 // vector.
4141 if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, i))
4142 Arg = getOrCreateVectorValue(CI->getArgOperand(i), Part);
4143 Args.push_back(Arg);
4144 }
4145
4146 Function *VectorF;
4147 if (UseVectorIntrinsic) {
4148 // Use vector version of the intrinsic.
4149 Type *TysForDecl[] = {CI->getType()};
4150 if (VF > 1)
4151 TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
4152 VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
4153 } else {
4154 // Use vector version of the library call.
4155 StringRef VFnName = TLI->getVectorizedFunction(FnName, VF);
4156 assert(!VFnName.empty() && "Vector function name is empty.")((!VFnName.empty() && "Vector function name is empty."
) ? static_cast<void> (0) : __assert_fail ("!VFnName.empty() && \"Vector function name is empty.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4156, __PRETTY_FUNCTION__))
;
4157 VectorF = M->getFunction(VFnName);
4158 if (!VectorF) {
4159 // Generate a declaration
4160 FunctionType *FTy = FunctionType::get(RetTy, Tys, false);
4161 VectorF =
4162 Function::Create(FTy, Function::ExternalLinkage, VFnName, M);
4163 VectorF->copyAttributesFrom(F);
4164 }
4165 }
4166 assert(VectorF && "Can't create vector function.")((VectorF && "Can't create vector function.") ? static_cast
<void> (0) : __assert_fail ("VectorF && \"Can't create vector function.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4166, __PRETTY_FUNCTION__))
;
4167
4168 SmallVector<OperandBundleDef, 1> OpBundles;
4169 CI->getOperandBundlesAsDefs(OpBundles);
4170 CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
4171
4172 if (isa<FPMathOperator>(V))
4173 V->copyFastMathFlags(CI);
4174
4175 VectorLoopValueMap.setVectorValue(&I, Part, V);
4176 addMetadata(V, &I);
4177 }
4178
4179 break;
4180 }
4181
4182 default:
4183 // This instruction is not vectorized by simple widening.
4184 LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an unhandled instruction: "
<< I; } } while (false)
;
4185 llvm_unreachable("Unhandled instruction!")::llvm::llvm_unreachable_internal("Unhandled instruction!", "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4185)
;
4186 } // end of switch.
4187}
4188
4189void InnerLoopVectorizer::updateAnalysis() {
4190 // Forget the original basic block.
4191 PSE.getSE()->forgetLoop(OrigLoop);
4192
4193 // DT is not kept up-to-date for outer loop vectorization
4194 if (EnableVPlanNativePath)
4195 return;
4196
4197 // Update the dominator tree information.
4198 assert(DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) &&((DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock
) && "Entry does not dominate exit.") ? static_cast<
void> (0) : __assert_fail ("DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) && \"Entry does not dominate exit.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4199, __PRETTY_FUNCTION__))
4199 "Entry does not dominate exit.")((DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock
) && "Entry does not dominate exit.") ? static_cast<
void> (0) : __assert_fail ("DT->properlyDominates(LoopBypassBlocks.front(), LoopExitBlock) && \"Entry does not dominate exit.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4199, __PRETTY_FUNCTION__))
;
4200
4201 DT->addNewBlock(LoopMiddleBlock,
4202 LI->getLoopFor(LoopVectorBody)->getLoopLatch());
4203 DT->addNewBlock(LoopScalarPreHeader, LoopBypassBlocks[0]);
4204 DT->changeImmediateDominator(LoopScalarBody, LoopScalarPreHeader);
4205 DT->changeImmediateDominator(LoopExitBlock, LoopBypassBlocks[0]);
4206 assert(DT->verify(DominatorTree::VerificationLevel::Fast))((DT->verify(DominatorTree::VerificationLevel::Fast)) ? static_cast
<void> (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast)"
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4206, __PRETTY_FUNCTION__))
;
4207}
4208
4209void LoopVectorizationCostModel::collectLoopScalars(unsigned VF) {
4210 // We should not collect Scalars more than once per VF. Right now, this
4211 // function is called from collectUniformsAndScalars(), which already does
4212 // this check. Collecting Scalars for VF=1 does not make any sense.
4213 assert(VF >= 2 && Scalars.find(VF) == Scalars.end() &&((VF >= 2 && Scalars.find(VF) == Scalars.end() &&
"This function should not be visited twice for the same VF")
? static_cast<void> (0) : __assert_fail ("VF >= 2 && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4214, __PRETTY_FUNCTION__))
4214 "This function should not be visited twice for the same VF")((VF >= 2 && Scalars.find(VF) == Scalars.end() &&
"This function should not be visited twice for the same VF")
? static_cast<void> (0) : __assert_fail ("VF >= 2 && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4214, __PRETTY_FUNCTION__))
;
4215
4216 SmallSetVector<Instruction *, 8> Worklist;
4217
4218 // These sets are used to seed the analysis with pointers used by memory
4219 // accesses that will remain scalar.
4220 SmallSetVector<Instruction *, 8> ScalarPtrs;
4221 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
4222
4223 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
4224 // The pointer operands of loads and stores will be scalar as long as the
4225 // memory access is not a gather or scatter operation. The value operand of a
4226 // store will remain scalar if the store is scalarized.
4227 auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
4228 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
4229 assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4230, __PRETTY_FUNCTION__))
4230 "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4230, __PRETTY_FUNCTION__))
;
4231 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
4232 if (Ptr == Store->getValueOperand())
4233 return WideningDecision == CM_Scalarize;
4234 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&((Ptr == getLoadStorePointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4235, __PRETTY_FUNCTION__))
4235 "Ptr is neither a value or pointer operand")((Ptr == getLoadStorePointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4235, __PRETTY_FUNCTION__))
;
4236 return WideningDecision != CM_GatherScatter;
4237 };
4238
4239 // A helper that returns true if the given value is a bitcast or
4240 // getelementptr instruction contained in the loop.
4241 auto isLoopVaryingBitCastOrGEP = [&](Value *V) {
4242 return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||
4243 isa<GetElementPtrInst>(V)) &&
4244 !TheLoop->isLoopInvariant(V);
4245 };
4246
4247 // A helper that evaluates a memory access's use of a pointer. If the use
4248 // will be a scalar use, and the pointer is only used by memory accesses, we
4249 // place the pointer in ScalarPtrs. Otherwise, the pointer is placed in
4250 // PossibleNonScalarPtrs.
4251 auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
4252 // We only care about bitcast and getelementptr instructions contained in
4253 // the loop.
4254 if (!isLoopVaryingBitCastOrGEP(Ptr))
4255 return;
4256
4257 // If the pointer has already been identified as scalar (e.g., if it was
4258 // also identified as uniform), there's nothing to do.
4259 auto *I = cast<Instruction>(Ptr);
4260 if (Worklist.count(I))
4261 return;
4262
4263 // If the use of the pointer will be a scalar use, and all users of the
4264 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
4265 // place the pointer in PossibleNonScalarPtrs.
4266 if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
4267 return isa<LoadInst>(U) || isa<StoreInst>(U);
4268 }))
4269 ScalarPtrs.insert(I);
4270 else
4271 PossibleNonScalarPtrs.insert(I);
4272 };
4273
4274 // We seed the scalars analysis with three classes of instructions: (1)
4275 // instructions marked uniform-after-vectorization, (2) bitcast and
4276 // getelementptr instructions used by memory accesses requiring a scalar use,
4277 // and (3) pointer induction variables and their update instructions (we
4278 // currently only scalarize these).
4279 //
4280 // (1) Add to the worklist all instructions that have been identified as
4281 // uniform-after-vectorization.
4282 Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());
4283
4284 // (2) Add to the worklist all bitcast and getelementptr instructions used by
4285 // memory accesses requiring a scalar use. The pointer operands of loads and
4286 // stores will be scalar as long as the memory accesses is not a gather or
4287 // scatter operation. The value operand of a store will remain scalar if the
4288 // store is scalarized.
4289 for (auto *BB : TheLoop->blocks())
4290 for (auto &I : *BB) {
4291 if (auto *Load = dyn_cast<LoadInst>(&I)) {
4292 evaluatePtrUse(Load, Load->getPointerOperand());
4293 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
4294 evaluatePtrUse(Store, Store->getPointerOperand());
4295 evaluatePtrUse(Store, Store->getValueOperand());
4296 }
4297 }
4298 for (auto *I : ScalarPtrs)
4299 if (PossibleNonScalarPtrs.find(I) == PossibleNonScalarPtrs.end()) {
4300 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *I << "\n"; } } while (false)
;
4301 Worklist.insert(I);
4302 }
4303
4304 // (3) Add to the worklist all pointer induction variables and their update
4305 // instructions.
4306 //
4307 // TODO: Once we are able to vectorize pointer induction variables we should
4308 // no longer insert them into the worklist here.
4309 auto *Latch = TheLoop->getLoopLatch();
4310 for (auto &Induction : *Legal->getInductionVars()) {
4311 auto *Ind = Induction.first;
4312 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4313 if (Induction.second.getKind() != InductionDescriptor::IK_PtrInduction)
4314 continue;
4315 Worklist.insert(Ind);
4316 Worklist.insert(IndUpdate);
4317 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Ind << "\n"; } } while (false)
;
4318 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
4319 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
;
4320 }
4321
4322 // Insert the forced scalars.
4323 // FIXME: Currently widenPHIInstruction() often creates a dead vector
4324 // induction variable when the PHI user is scalarized.
4325 auto ForcedScalar = ForcedScalars.find(VF);
4326 if (ForcedScalar != ForcedScalars.end())
4327 for (auto *I : ForcedScalar->second)
4328 Worklist.insert(I);
4329
4330 // Expand the worklist by looking through any bitcasts and getelementptr
4331 // instructions we've already identified as scalar. This is similar to the
4332 // expansion step in collectLoopUniforms(); however, here we're only
4333 // expanding to include additional bitcasts and getelementptr instructions.
4334 unsigned Idx = 0;
4335 while (Idx != Worklist.size()) {
4336 Instruction *Dst = Worklist[Idx++];
4337 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
4338 continue;
4339 auto *Src = cast<Instruction>(Dst->getOperand(0));
4340 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
4341 auto *J = cast<Instruction>(U);
4342 return !TheLoop->contains(J) || Worklist.count(J) ||
4343 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
4344 isScalarUse(J, Src));
4345 })) {
4346 Worklist.insert(Src);
4347 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Src << "\n"; } } while (false)
;
4348 }
4349 }
4350
4351 // An induction variable will remain scalar if all users of the induction
4352 // variable and induction variable update remain scalar.
4353 for (auto &Induction : *Legal->getInductionVars()) {
4354 auto *Ind = Induction.first;
4355 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4356
4357 // We already considered pointer induction variables, so there's no reason
4358 // to look at their users again.
4359 //
4360 // TODO: Once we are able to vectorize pointer induction variables we
4361 // should no longer skip over them here.
4362 if (Induction.second.getKind() == InductionDescriptor::IK_PtrInduction)
4363 continue;
4364
4365 // Determine if all users of the induction variable are scalar after
4366 // vectorization.
4367 auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
4368 auto *I = cast<Instruction>(U);
4369 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I);
4370 });
4371 if (!ScalarInd)
4372 continue;
4373
4374 // Determine if all users of the induction variable update instruction are
4375 // scalar after vectorization.
4376 auto ScalarIndUpdate =
4377 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
4378 auto *I = cast<Instruction>(U);
4379 return I == Ind || !TheLoop->contains(I) || Worklist.count(I);
4380 });
4381 if (!ScalarIndUpdate)
4382 continue;
4383
4384 // The induction variable and its update instruction will remain scalar.
4385 Worklist.insert(Ind);
4386 Worklist.insert(IndUpdate);
4387 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Ind << "\n"; } } while (false)
;
4388 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
4389 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
;
4390 }
4391
4392 Scalars[VF].insert(Worklist.begin(), Worklist.end());
4393}
4394
4395bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I, unsigned VF) {
4396 if (!blockNeedsPredication(I->getParent()))
4397 return false;
4398 switch(I->getOpcode()) {
4399 default:
4400 break;
4401 case Instruction::Load:
4402 case Instruction::Store: {
4403 if (!Legal->isMaskRequired(I))
4404 return false;
4405 auto *Ptr = getLoadStorePointerOperand(I);
4406 auto *Ty = getMemInstValueType(I);
4407 // We have already decided how to vectorize this instruction, get that
4408 // result.
4409 if (VF > 1) {
4410 InstWidening WideningDecision = getWideningDecision(I, VF);
4411 assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4412, __PRETTY_FUNCTION__))
4412 "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4412, __PRETTY_FUNCTION__))
;
4413 return WideningDecision == CM_Scalarize;
4414 }
4415 return isa<LoadInst>(I) ?
4416 !(isLegalMaskedLoad(Ty, Ptr) || isLegalMaskedGather(Ty))
4417 : !(isLegalMaskedStore(Ty, Ptr) || isLegalMaskedScatter(Ty));
4418 }
4419 case Instruction::UDiv:
4420 case Instruction::SDiv:
4421 case Instruction::SRem:
4422 case Instruction::URem:
4423 return mayDivideByZero(*I);
4424 }
4425 return false;
4426}
4427
4428bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(Instruction *I,
4429 unsigned VF) {
4430 assert(isAccessInterleaved(I) && "Expecting interleaved access.")((isAccessInterleaved(I) && "Expecting interleaved access."
) ? static_cast<void> (0) : __assert_fail ("isAccessInterleaved(I) && \"Expecting interleaved access.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4430, __PRETTY_FUNCTION__))
;
4431 assert(getWideningDecision(I, VF) == CM_Unknown &&((getWideningDecision(I, VF) == CM_Unknown && "Decision should not be set yet."
) ? static_cast<void> (0) : __assert_fail ("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4432, __PRETTY_FUNCTION__))
4432 "Decision should not be set yet.")((getWideningDecision(I, VF) == CM_Unknown && "Decision should not be set yet."
) ? static_cast<void> (0) : __assert_fail ("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4432, __PRETTY_FUNCTION__))
;
4433 auto *Group = getInterleavedAccessGroup(I);
4434 assert(Group && "Must have a group.")((Group && "Must have a group.") ? static_cast<void
> (0) : __assert_fail ("Group && \"Must have a group.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4434, __PRETTY_FUNCTION__))
;
4435
4436 // Check if masking is required.
4437 // A Group may need masking for one of two reasons: it resides in a block that
4438 // needs predication, or it was decided to use masking to deal with gaps.
4439 bool PredicatedAccessRequiresMasking =
4440 Legal->blockNeedsPredication(I->getParent()) && Legal->isMaskRequired(I);
4441 bool AccessWithGapsRequiresMasking =
4442 Group->requiresScalarEpilogue() && !IsScalarEpilogueAllowed;
4443 if (!PredicatedAccessRequiresMasking && !AccessWithGapsRequiresMasking)
4444 return true;
4445
4446 // If masked interleaving is required, we expect that the user/target had
4447 // enabled it, because otherwise it either wouldn't have been created or
4448 // it should have been invalidated by the CostModel.
4449 assert(useMaskedInterleavedAccesses(TTI) &&((useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4450, __PRETTY_FUNCTION__))
4450 "Masked interleave-groups for predicated accesses are not enabled.")((useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4450, __PRETTY_FUNCTION__))
;
4451
4452 auto *Ty = getMemInstValueType(I);
4453 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty)
4454 : TTI.isLegalMaskedStore(Ty);
4455}
4456
4457bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
4458 unsigned VF) {
4459 // Get and ensure we have a valid memory instruction.
4460 LoadInst *LI = dyn_cast<LoadInst>(I);
4461 StoreInst *SI = dyn_cast<StoreInst>(I);
4462 assert((LI || SI) && "Invalid memory instruction")(((LI || SI) && "Invalid memory instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid memory instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4462, __PRETTY_FUNCTION__))
;
4463
4464 auto *Ptr = getLoadStorePointerOperand(I);
4465
4466 // In order to be widened, the pointer should be consecutive, first of all.
4467 if (!Legal->isConsecutivePtr(Ptr))
4468 return false;
4469
4470 // If the instruction is a store located in a predicated block, it will be
4471 // scalarized.
4472 if (isScalarWithPredication(I))
4473 return false;
4474
4475 // If the instruction's allocated size doesn't equal it's type size, it
4476 // requires padding and will be scalarized.
4477 auto &DL = I->getModule()->getDataLayout();
4478 auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
4479 if (hasIrregularType(ScalarTy, DL, VF))
4480 return false;
4481
4482 return true;
4483}
4484
4485void LoopVectorizationCostModel::collectLoopUniforms(unsigned VF) {
4486 // We should not collect Uniforms more than once per VF. Right now,
4487 // this function is called from collectUniformsAndScalars(), which
4488 // already does this check. Collecting Uniforms for VF=1 does not make any
4489 // sense.
4490
4491 assert(VF >= 2 && Uniforms.find(VF) == Uniforms.end() &&((VF >= 2 && Uniforms.find(VF) == Uniforms.end() &&
"This function should not be visited twice for the same VF")
? static_cast<void> (0) : __assert_fail ("VF >= 2 && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4492, __PRETTY_FUNCTION__))
4492 "This function should not be visited twice for the same VF")((VF >= 2 && Uniforms.find(VF) == Uniforms.end() &&
"This function should not be visited twice for the same VF")
? static_cast<void> (0) : __assert_fail ("VF >= 2 && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4492, __PRETTY_FUNCTION__))
;
4493
4494 // Visit the list of Uniforms. If we'll not find any uniform value, we'll
4495 // not analyze again. Uniforms.count(VF) will return 1.
4496 Uniforms[VF].clear();
4497
4498 // We now know that the loop is vectorizable!
4499 // Collect instructions inside the loop that will remain uniform after
4500 // vectorization.
4501
4502 // Global values, params and instructions outside of current loop are out of
4503 // scope.
4504 auto isOutOfScope = [&](Value *V) -> bool {
4505 Instruction *I = dyn_cast<Instruction>(V);
4506 return (!I || !TheLoop->contains(I));
4507 };
4508
4509 SetVector<Instruction *> Worklist;
4510 BasicBlock *Latch = TheLoop->getLoopLatch();
4511
4512 // Start with the conditional branch. If the branch condition is an
4513 // instruction contained in the loop that is only used by the branch, it is
4514 // uniform.
4515 auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));
4516 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse()) {
4517 Worklist.insert(Cmp);
4518 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *Cmp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *Cmp << "\n"; } } while (false)
;
4519 }
4520
4521 // Holds consecutive and consecutive-like pointers. Consecutive-like pointers
4522 // are pointers that are treated like consecutive pointers during
4523 // vectorization. The pointer operands of interleaved accesses are an
4524 // example.
4525 SmallSetVector<Instruction *, 8> ConsecutiveLikePtrs;
4526
4527 // Holds pointer operands of instructions that are possibly non-uniform.
4528 SmallPtrSet<Instruction *, 8> PossibleNonUniformPtrs;
4529
4530 auto isUniformDecision = [&](Instruction *I, unsigned VF) {
4531 InstWidening WideningDecision = getWideningDecision(I, VF);
4532 assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4533, __PRETTY_FUNCTION__))
4533 "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4533, __PRETTY_FUNCTION__))
;
4534
4535 return (WideningDecision == CM_Widen ||
4536 WideningDecision == CM_Widen_Reverse ||
4537 WideningDecision == CM_Interleave);
4538 };
4539 // Iterate over the instructions in the loop, and collect all
4540 // consecutive-like pointer operands in ConsecutiveLikePtrs. If it's possible
4541 // that a consecutive-like pointer operand will be scalarized, we collect it
4542 // in PossibleNonUniformPtrs instead. We use two sets here because a single
4543 // getelementptr instruction can be used by both vectorized and scalarized
4544 // memory instructions. For example, if a loop loads and stores from the same
4545 // location, but the store is conditional, the store will be scalarized, and
4546 // the getelementptr won't remain uniform.
4547 for (auto *BB : TheLoop->blocks())
4548 for (auto &I : *BB) {
4549 // If there's no pointer operand, there's nothing to do.
4550 auto *Ptr = dyn_cast_or_null<Instruction>(getLoadStorePointerOperand(&I));
4551 if (!Ptr)
4552 continue;
4553
4554 // True if all users of Ptr are memory accesses that have Ptr as their
4555 // pointer operand.
4556 auto UsersAreMemAccesses =
4557 llvm::all_of(Ptr->users(), [&](User *U) -> bool {
4558 return getLoadStorePointerOperand(U) == Ptr;
4559 });
4560
4561 // Ensure the memory instruction will not be scalarized or used by
4562 // gather/scatter, making its pointer operand non-uniform. If the pointer
4563 // operand is used by any instruction other than a memory access, we
4564 // conservatively assume the pointer operand may be non-uniform.
4565 if (!UsersAreMemAccesses || !isUniformDecision(&I, VF))
4566 PossibleNonUniformPtrs.insert(Ptr);
4567
4568 // If the memory instruction will be vectorized and its pointer operand
4569 // is consecutive-like, or interleaving - the pointer operand should
4570 // remain uniform.
4571 else
4572 ConsecutiveLikePtrs.insert(Ptr);
4573 }
4574
4575 // Add to the Worklist all consecutive and consecutive-like pointers that
4576 // aren't also identified as possibly non-uniform.
4577 for (auto *V : ConsecutiveLikePtrs)
4578 if (PossibleNonUniformPtrs.find(V) == PossibleNonUniformPtrs.end()) {
4579 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *V << "\n"; } } while (false)
;
4580 Worklist.insert(V);
4581 }
4582
4583 // Expand Worklist in topological order: whenever a new instruction
4584 // is added , its users should be already inside Worklist. It ensures
4585 // a uniform instruction will only be used by uniform instructions.
4586 unsigned idx = 0;
4587 while (idx != Worklist.size()) {
4588 Instruction *I = Worklist[idx++];
4589
4590 for (auto OV : I->operand_values()) {
4591 // isOutOfScope operands cannot be uniform instructions.
4592 if (isOutOfScope(OV))
4593 continue;
4594 // First order recurrence Phi's should typically be considered
4595 // non-uniform.
4596 auto *OP = dyn_cast<PHINode>(OV);
4597 if (OP && Legal->isFirstOrderRecurrence(OP))
4598 continue;
4599 // If all the users of the operand are uniform, then add the
4600 // operand into the uniform worklist.
4601 auto *OI = cast<Instruction>(OV);
4602 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
4603 auto *J = cast<Instruction>(U);
4604 return Worklist.count(J) ||
4605 (OI == getLoadStorePointerOperand(J) &&
4606 isUniformDecision(J, VF));
4607 })) {
4608 Worklist.insert(OI);
4609 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *OI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *OI << "\n"; } } while (false)
;
4610 }
4611 }
4612 }
4613
4614 // Returns true if Ptr is the pointer operand of a memory access instruction
4615 // I, and I is known to not require scalarization.
4616 auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
4617 return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
4618 };
4619
4620 // For an instruction to be added into Worklist above, all its users inside
4621 // the loop should also be in Worklist. However, this condition cannot be
4622 // true for phi nodes that form a cyclic dependence. We must process phi
4623 // nodes separately. An induction variable will remain uniform if all users
4624 // of the induction variable and induction variable update remain uniform.
4625 // The code below handles both pointer and non-pointer induction variables.
4626 for (auto &Induction : *Legal->getInductionVars()) {
4627 auto *Ind = Induction.first;
4628 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4629
4630 // Determine if all users of the induction variable are uniform after
4631 // vectorization.
4632 auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
4633 auto *I = cast<Instruction>(U);
4634 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
4635 isVectorizedMemAccessUse(I, Ind);
4636 });
4637 if (!UniformInd)
4638 continue;
4639
4640 // Determine if all users of the induction variable update instruction are
4641 // uniform after vectorization.
4642 auto UniformIndUpdate =
4643 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
4644 auto *I = cast<Instruction>(U);
4645 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
4646 isVectorizedMemAccessUse(I, IndUpdate);
4647 });
4648 if (!UniformIndUpdate)
4649 continue;
4650
4651 // The induction variable and its update instruction will remain uniform.
4652 Worklist.insert(Ind);
4653 Worklist.insert(IndUpdate);
4654 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *Ind << "\n"; } } while (false)
;
4655 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *IndUpdatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *IndUpdate << "\n"; } } while (false)
4656 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *IndUpdate << "\n"; } } while (false)
;
4657 }
4658
4659 Uniforms[VF].insert(Worklist.begin(), Worklist.end());
4660}
4661
4662Optional<unsigned> LoopVectorizationCostModel::computeMaxVF(bool OptForSize) {
4663 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
4664 // TODO: It may by useful to do since it's still likely to be dynamically
4665 // uniform if the target can skip.
4666 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not inserting runtime ptr check for divergent target"
; } } while (false)
4667 dbgs() << "LV: Not inserting runtime ptr check for divergent target")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not inserting runtime ptr check for divergent target"
; } } while (false)
;
4668
4669 ORE->emit(
4670 createMissedAnalysis("CantVersionLoopWithDivergentTarget")
4671 << "runtime pointer checks needed. Not enabled for divergent target");
4672
4673 return None;
4674 }
4675
4676 unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
4677 if (!OptForSize) // Remaining checks deal with scalar loop when OptForSize.
4678 return computeFeasibleMaxVF(OptForSize, TC);
4679
4680 if (Legal->getRuntimePointerChecking()->Need) {
4681 ORE->emit(createMissedAnalysis("CantVersionLoopWithOptForSize")
4682 << "runtime pointer checks needed. Enable vectorization of this "
4683 "loop with '#pragma clang loop vectorize(enable)' when "
4684 "compiling with -Os/-Oz");
4685 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n"
; } } while (false)
4686 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n"
; } } while (false)
4687 << "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime ptr check is required with -Os/-Oz.\n"
; } } while (false)
;
4688 return None;
4689 }
4690
4691 if (!PSE.getUnionPredicate().getPredicates().empty()) {
4692 ORE->emit(createMissedAnalysis("CantVersionLoopWithOptForSize")
4693 << "runtime SCEV checks needed. Enable vectorization of this "
4694 "loop with '#pragma clang loop vectorize(enable)' when "
4695 "compiling with -Os/-Oz");
4696 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime SCEV check is required with -Os/-Oz.\n"
; } } while (false)
4697 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime SCEV check is required with -Os/-Oz.\n"
; } } while (false)
4698 << "LV: Aborting. Runtime SCEV check is required with -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime SCEV check is required with -Os/-Oz.\n"
; } } while (false)
;
4699 return None;
4700 }
4701
4702 // FIXME: Avoid specializing for stride==1 instead of bailing out.
4703 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
4704 ORE->emit(createMissedAnalysis("CantVersionLoopWithOptForSize")
4705 << "runtime stride == 1 checks needed. Enable vectorization of "
4706 "this loop with '#pragma clang loop vectorize(enable)' when "
4707 "compiling with -Os/-Oz");
4708 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime stride check is required with -Os/-Oz.\n"
; } } while (false)
4709 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime stride check is required with -Os/-Oz.\n"
; } } while (false)
4710 << "LV: Aborting. Runtime stride check is required with -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting. Runtime stride check is required with -Os/-Oz.\n"
; } } while (false)
;
4711 return None;
4712 }
4713
4714 // If we optimize the program for size, avoid creating the tail loop.
4715 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found trip count: "
<< TC << '\n'; } } while (false)
;
4716
4717 if (TC == 1) {
4718 ORE->emit(createMissedAnalysis("SingleIterationLoop")
4719 << "loop trip count is one, irrelevant for vectorization");
4720 LLVM_DEBUG(dbgs() << "LV: Aborting, single iteration (non) loop.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Aborting, single iteration (non) loop.\n"
; } } while (false)
;
4721 return None;
4722 }
4723
4724 // Record that scalar epilogue is not allowed.
4725 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false)
;
4726
4727 IsScalarEpilogueAllowed = !OptForSize;
4728
4729 // We don't create an epilogue when optimizing for size.
4730 // Invalidate interleave groups that require an epilogue if we can't mask
4731 // the interleave-group.
4732 if (!useMaskedInterleavedAccesses(TTI))
4733 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
4734
4735 unsigned MaxVF = computeFeasibleMaxVF(OptForSize, TC);
4736
4737 if (TC > 0 && TC % MaxVF == 0) {
4738 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No tail will remain for any chosen VF.\n"
; } } while (false)
;
4739 return MaxVF;
4740 }
4741
4742 // If we don't know the precise trip count, or if the trip count that we
4743 // found modulo the vectorization factor is not zero, try to fold the tail
4744 // by masking.
4745 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
4746 if (Legal->canFoldTailByMasking()) {
4747 FoldTailByMasking = true;
4748 return MaxVF;
4749 }
4750
4751 if (TC == 0) {
4752 ORE->emit(
4753 createMissedAnalysis("UnknownLoopCountComplexCFG")
4754 << "unable to calculate the loop count due to complex control flow");
4755 return None;
4756 }
4757
4758 ORE->emit(createMissedAnalysis("NoTailLoopWithOptForSize")
4759 << "cannot optimize for size and vectorize at the same time. "
4760 "Enable vectorization of this loop with '#pragma clang loop "
4761 "vectorize(enable)' when compiling with -Os/-Oz");
4762 return None;
4763}
4764
4765unsigned
4766LoopVectorizationCostModel::computeFeasibleMaxVF(bool OptForSize,
4767 unsigned ConstTripCount) {
4768 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
4769 unsigned SmallestType, WidestType;
4770 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
4771 unsigned WidestRegister = TTI.getRegisterBitWidth(true);
4772
4773 // Get the maximum safe dependence distance in bits computed by LAA.
4774 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
4775 // the memory accesses that is most restrictive (involved in the smallest
4776 // dependence distance).
4777 unsigned MaxSafeRegisterWidth = Legal->getMaxSafeRegisterWidth();
4778
4779 WidestRegister = std::min(WidestRegister, MaxSafeRegisterWidth);
4780
4781 unsigned MaxVectorSize = WidestRegister / WidestType;
4782
4783 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestTypedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
<< SmallestType << " / " << WidestType <<
" bits.\n"; } } while (false)
4784 << " / " << WidestType << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
<< SmallestType << " / " << WidestType <<
" bits.\n"; } } while (false)
;
4785 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register safe to use is: "
<< WidestRegister << " bits.\n"; } } while (false
)
4786 << WidestRegister << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register safe to use is: "
<< WidestRegister << " bits.\n"; } } while (false
)
;
4787
4788 assert(MaxVectorSize <= 256 && "Did not expect to pack so many elements"((MaxVectorSize <= 256 && "Did not expect to pack so many elements"
" into one vector!") ? static_cast<void> (0) : __assert_fail
("MaxVectorSize <= 256 && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4789, __PRETTY_FUNCTION__))
4789 " into one vector!")((MaxVectorSize <= 256 && "Did not expect to pack so many elements"
" into one vector!") ? static_cast<void> (0) : __assert_fail
("MaxVectorSize <= 256 && \"Did not expect to pack so many elements\" \" into one vector!\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4789, __PRETTY_FUNCTION__))
;
4790 if (MaxVectorSize == 0) {
4791 LLVM_DEBUG(dbgs() << "LV: The target has no vector registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has no vector registers.\n"
; } } while (false)
;
4792 MaxVectorSize = 1;
4793 return MaxVectorSize;
4794 } else if (ConstTripCount && ConstTripCount < MaxVectorSize &&
4795 isPowerOf2_32(ConstTripCount)) {
4796 // We need to clamp the VF to be the ConstTripCount. There is no point in
4797 // choosing a higher viable VF as done in the loop below.
4798 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to the constant trip count: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
<< ConstTripCount << "\n"; } } while (false)
4799 << ConstTripCount << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to the constant trip count: "
<< ConstTripCount << "\n"; } } while (false)
;
4800 MaxVectorSize = ConstTripCount;
4801 return MaxVectorSize;
4802 }
4803
4804 unsigned MaxVF = MaxVectorSize;
4805 if (TTI.shouldMaximizeVectorBandwidth(OptForSize) ||
4806 (MaximizeBandwidth && !OptForSize)) {
4807 // Collect all viable vectorization factors larger than the default MaxVF
4808 // (i.e. MaxVectorSize).
4809 SmallVector<unsigned, 8> VFs;
4810 unsigned NewMaxVectorSize = WidestRegister / SmallestType;
4811 for (unsigned VS = MaxVectorSize * 2; VS <= NewMaxVectorSize; VS *= 2)
4812 VFs.push_back(VS);
4813
4814 // For each VF calculate its register usage.
4815 auto RUs = calculateRegisterUsage(VFs);
4816
4817 // Select the largest VF which doesn't require more registers than existing
4818 // ones.
4819 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(true);
4820 for (int i = RUs.size() - 1; i >= 0; --i) {
4821 if (RUs[i].MaxLocalUsers <= TargetNumRegisters) {
4822 MaxVF = VFs[i];
4823 break;
4824 }
4825 }
4826 if (unsigned MinVF = TTI.getMinimumVF(SmallestType)) {
4827 if (MaxVF < MinVF) {
4828 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Overriding calculated MaxVF("
<< MaxVF << ") with target's minimum: " <<
MinVF << '\n'; } } while (false)
4829 << ") with target's minimum: " << MinVF << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Overriding calculated MaxVF("
<< MaxVF << ") with target's minimum: " <<
MinVF << '\n'; } } while (false)
;
4830 MaxVF = MinVF;
4831 }
4832 }
4833 }
4834 return MaxVF;
4835}
4836
4837VectorizationFactor
4838LoopVectorizationCostModel::selectVectorizationFactor(unsigned MaxVF) {
4839 float Cost = expectedCost(1).first;
4840 const float ScalarCost = Cost;
4841 unsigned Width = 1;
4842 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << (int)ScalarCost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalar loop costs: "
<< (int)ScalarCost << ".\n"; } } while (false)
;
4843
4844 bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
4845 if (ForceVectorization && MaxVF > 1) {
4846 // Ignore scalar width, because the user explicitly wants vectorization.
4847 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
4848 // evaluation.
4849 Cost = std::numeric_limits<float>::max();
4850 }
4851
4852 for (unsigned i = 2; i <= MaxVF; i *= 2) {
4853 // Notice that the vector loop needs to be executed less times, so
4854 // we need to divide the cost of the vector loops by the width of
4855 // the vector elements.
4856 VectorizationCostTy C = expectedCost(i);
4857 float VectorCost = C.first / (float)i;
4858 LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vector loop of width "
<< i << " costs: " << (int)VectorCost <<
".\n"; } } while (false)
4859 << " costs: " << (int)VectorCost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Vector loop of width "
<< i << " costs: " << (int)VectorCost <<
".\n"; } } while (false)
;
4860 if (!C.second && !ForceVectorization) {
4861 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
<< i << " because it will not generate any vector instructions.\n"
; } } while (false)
4862 dbgs() << "LV: Not considering vector loop of width " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
<< i << " because it will not generate any vector instructions.\n"
; } } while (false)
4863 << " because it will not generate any vector instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not considering vector loop of width "
<< i << " because it will not generate any vector instructions.\n"
; } } while (false)
;
4864 continue;
4865 }
4866 if (VectorCost < Cost) {
4867 Cost = VectorCost;
4868 Width = i;
4869 }
4870 }
4871
4872 if (!EnableCondStoresVectorization && NumPredStores) {
4873 ORE->emit(createMissedAnalysis("ConditionalStore")
4874 << "store that is conditionally executed prevents vectorization");
4875 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No vectorization. There are conditional stores.\n"
; } } while (false)
4876 dbgs() << "LV: No vectorization. There are conditional stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No vectorization. There are conditional stores.\n"
; } } while (false)
;
4877 Width = 1;
4878 Cost = ScalarCost;
4879 }
4880
4881 LLVM_DEBUG(if (ForceVectorization && Width > 1 && Cost >= ScalarCost) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && Width
> 1 && Cost >= ScalarCost) dbgs() << "LV: Vectorization seems to be not beneficial, "
<< "but was forced by a user.\n"; } } while (false)
4882 << "LV: Vectorization seems to be not beneficial, "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && Width
> 1 && Cost >= ScalarCost) dbgs() << "LV: Vectorization seems to be not beneficial, "
<< "but was forced by a user.\n"; } } while (false)
4883 << "but was forced by a user.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { if (ForceVectorization && Width
> 1 && Cost >= ScalarCost) dbgs() << "LV: Vectorization seems to be not beneficial, "
<< "but was forced by a user.\n"; } } while (false)
;
4884 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << Width << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Selecting VF: " <<
Width << ".\n"; } } while (false)
;
4885 VectorizationFactor Factor = {Width, (unsigned)(Width * Cost)};
4886 return Factor;
4887}
4888
4889std::pair<unsigned, unsigned>
4890LoopVectorizationCostModel::getSmallestAndWidestTypes() {
4891 unsigned MinWidth = -1U;
4892 unsigned MaxWidth = 8;
4893 const DataLayout &DL = TheFunction->getParent()->getDataLayout();
4894
4895 // For each block.
4896 for (BasicBlock *BB : TheLoop->blocks()) {
4897 // For each instruction in the loop.
4898 for (Instruction &I : BB->instructionsWithoutDebug()) {
4899 Type *T = I.getType();
4900
4901 // Skip ignored values.
4902 if (ValuesToIgnore.find(&I) != ValuesToIgnore.end())
4903 continue;
4904
4905 // Only examine Loads, Stores and PHINodes.
4906 if (!isa<LoadInst>(I) && !isa<StoreInst>(I) && !isa<PHINode>(I))
4907 continue;
4908
4909 // Examine PHI nodes that are reduction variables. Update the type to
4910 // account for the recurrence type.
4911 if (auto *PN = dyn_cast<PHINode>(&I)) {
4912 if (!Legal->isReductionVariable(PN))
4913 continue;
4914 RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[PN];
4915 T = RdxDesc.getRecurrenceType();
4916 }
4917
4918 // Examine the stored values.
4919 if (auto *ST = dyn_cast<StoreInst>(&I))
4920 T = ST->getValueOperand()->getType();
4921
4922 // Ignore loaded pointer types and stored pointer types that are not
4923 // vectorizable.
4924 //
4925 // FIXME: The check here attempts to predict whether a load or store will
4926 // be vectorized. We only know this for certain after a VF has
4927 // been selected. Here, we assume that if an access can be
4928 // vectorized, it will be. We should also look at extending this
4929 // optimization to non-pointer types.
4930 //
4931 if (T->isPointerTy() && !isConsecutiveLoadOrStore(&I) &&
4932 !isAccessInterleaved(&I) && !isLegalGatherOrScatter(&I))
4933 continue;
4934
4935 MinWidth = std::min(MinWidth,
4936 (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
4937 MaxWidth = std::max(MaxWidth,
4938 (unsigned)DL.getTypeSizeInBits(T->getScalarType()));
4939 }
4940 }
4941
4942 return {MinWidth, MaxWidth};
4943}
4944
4945unsigned LoopVectorizationCostModel::selectInterleaveCount(bool OptForSize,
4946 unsigned VF,
4947 unsigned LoopCost) {
4948 // -- The interleave heuristics --
4949 // We interleave the loop in order to expose ILP and reduce the loop overhead.
4950 // There are many micro-architectural considerations that we can't predict
4951 // at this level. For example, frontend pressure (on decode or fetch) due to
4952 // code size, or the number and capabilities of the execution ports.
4953 //
4954 // We use the following heuristics to select the interleave count:
4955 // 1. If the code has reductions, then we interleave to break the cross
4956 // iteration dependency.
4957 // 2. If the loop is really small, then we interleave to reduce the loop
4958 // overhead.
4959 // 3. We don't interleave if we think that we will spill registers to memory
4960 // due to the increased register pressure.
4961
4962 // When we optimize for size, we don't interleave.
4963 if (OptForSize)
4964 return 1;
4965
4966 // We used the distance for the interleave count.
4967 if (Legal->getMaxSafeDepDistBytes() != -1U)
4968 return 1;
4969
4970 // Do not interleave loops with a relatively small trip count.
4971 unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
4972 if (TC > 1 && TC < TinyTripCountInterleaveThreshold)
4973 return 1;
4974
4975 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(VF > 1);
4976 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegistersdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
TargetNumRegisters << " registers\n"; } } while (false
)
4977 << " registers\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has " <<
TargetNumRegisters << " registers\n"; } } while (false
)
;
4978
4979 if (VF == 1) {
4980 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
4981 TargetNumRegisters = ForceTargetNumScalarRegs;
4982 } else {
4983 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
4984 TargetNumRegisters = ForceTargetNumVectorRegs;
4985 }
4986
4987 RegisterUsage R = calculateRegisterUsage({VF})[0];
4988 // We divide by these constants so assume that we have at least one
4989 // instruction that uses at least one register.
4990 R.MaxLocalUsers = std::max(R.MaxLocalUsers, 1U);
4991
4992 // We calculate the interleave count using the following formula.
4993 // Subtract the number of loop invariants from the number of available
4994 // registers. These registers are used by all of the interleaved instances.
4995 // Next, divide the remaining registers by the number of registers that is
4996 // required by the loop, in order to estimate how many parallel instances
4997 // fit without causing spills. All of this is rounded down if necessary to be
4998 // a power of two. We want power of two interleave count to simplify any
4999 // addressing operations or alignment considerations.
5000 // We also want power of two interleave counts to ensure that the induction
5001 // variable of the vector loop wraps to zero, when tail is folded by masking;
5002 // this currently happens when OptForSize, in which case IC is set to 1 above.
5003 unsigned IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs) /
5004 R.MaxLocalUsers);
5005
5006 // Don't count the induction variable as interleaved.
5007 if (EnableIndVarRegisterHeur)
5008 IC = PowerOf2Floor((TargetNumRegisters - R.LoopInvariantRegs - 1) /
5009 std::max(1U, (R.MaxLocalUsers - 1)));
5010
5011 // Clamp the interleave ranges to reasonable counts.
5012 unsigned MaxInterleaveCount = TTI.getMaxInterleaveFactor(VF);
5013
5014 // Check if the user has overridden the max.
5015 if (VF == 1) {
5016 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
5017 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
5018 } else {
5019 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
5020 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
5021 }
5022
5023 // If we did not calculate the cost for VF (because the user selected the VF)
5024 // then we calculate the cost of VF here.
5025 if (LoopCost == 0)
5026 LoopCost = expectedCost(VF).first;
5027
5028 // Clamp the calculated IC to be between the 1 and the max interleave count
5029 // that the target allows.
5030 if (IC > MaxInterleaveCount)
5031 IC = MaxInterleaveCount;
5032 else if (IC < 1)
5033 IC = 1;
5034
5035 // Interleave if we vectorized this loop and there is a reduction that could
5036 // benefit from interleaving.
5037 if (VF > 1 && !Legal->getReductionVars()->empty()) {
5038 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving because of reductions.\n"
; } } while (false)
;
5039 return IC;
5040 }
5041
5042 // Note that if we've already vectorized the loop we will have done the
5043 // runtime check and so interleaving won't require further checks.
5044 bool InterleavingRequiresRuntimePointerCheck =
5045 (VF == 1 && Legal->getRuntimePointerChecking()->Need);
5046
5047 // We want to interleave small loops in order to reduce the loop overhead and
5048 // potentially expose ILP opportunities.
5049 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop cost is " <<
LoopCost << '\n'; } } while (false)
;
5050 if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
5051 // We assume that the cost overhead is 1 and we use the cost model
5052 // to estimate the cost of the loop and interleave until the cost of the
5053 // loop overhead is about 5% of the cost of the loop.
5054 unsigned SmallIC =
5055 std::min(IC, (unsigned)PowerOf2Floor(SmallLoopCost / LoopCost));
5056
5057 // Interleave until store/load ports (estimated by max interleave count) are
5058 // saturated.
5059 unsigned NumStores = Legal->getNumStores();
5060 unsigned NumLoads = Legal->getNumLoads();
5061 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
5062 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
5063
5064 // If we have a scalar reduction (vector reductions are already dealt with
5065 // by this point), we can increase the critical path length if the loop
5066 // we're interleaving is inside another loop. Limit, by default to 2, so the
5067 // critical path only gets increased by one reduction operation.
5068 if (!Legal->getReductionVars()->empty() && TheLoop->getLoopDepth() > 1) {
5069 unsigned F = static_cast<unsigned>(MaxNestedScalarReductionIC);
5070 SmallIC = std::min(SmallIC, F);
5071 StoresIC = std::min(StoresIC, F);
5072 LoadsIC = std::min(LoadsIC, F);
5073 }
5074
5075 if (EnableLoadStoreRuntimeInterleave &&
5076 std::max(StoresIC, LoadsIC) > SmallIC) {
5077 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to saturate store or load ports.\n"
; } } while (false)
5078 dbgs() << "LV: Interleaving to saturate store or load ports.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to saturate store or load ports.\n"
; } } while (false)
;
5079 return std::max(StoresIC, LoadsIC);
5080 }
5081
5082 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to reduce branch cost.\n"
; } } while (false)
;
5083 return SmallIC;
5084 }
5085
5086 // Interleave if this is a large loop (small loops are already dealt with by
5087 // this point) that could benefit from interleaving.
5088 bool HasReductions = !Legal->getReductionVars()->empty();
5089 if (TTI.enableAggressiveInterleaving(HasReductions)) {
5090 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Interleaving to expose ILP.\n"
; } } while (false)
;
5091 return IC;
5092 }
5093
5094 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not Interleaving.\n"
; } } while (false)
;
5095 return 1;
5096}
5097
5098SmallVector<LoopVectorizationCostModel::RegisterUsage, 8>
5099LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef<unsigned> VFs) {
5100 // This function calculates the register usage by measuring the highest number
5101 // of values that are alive at a single location. Obviously, this is a very
5102 // rough estimation. We scan the loop in a topological order in order and
5103 // assign a number to each instruction. We use RPO to ensure that defs are
5104 // met before their users. We assume that each instruction that has in-loop
5105 // users starts an interval. We record every time that an in-loop value is
5106 // used, so we have a list of the first and last occurrences of each
5107 // instruction. Next, we transpose this data structure into a multi map that
5108 // holds the list of intervals that *end* at a specific location. This multi
5109 // map allows us to perform a linear search. We scan the instructions linearly
5110 // and record each time that a new interval starts, by placing it in a set.
5111 // If we find this value in the multi-map then we remove it from the set.
5112 // The max register usage is the maximum size of the set.
5113 // We also search for instructions that are defined outside the loop, but are
5114 // used inside the loop. We need this number separately from the max-interval
5115 // usage number because when we unroll, loop-invariant values do not take
5116 // more register.
5117 LoopBlocksDFS DFS(TheLoop);
5118 DFS.perform(LI);
5119
5120 RegisterUsage RU;
5121
5122 // Each 'key' in the map opens a new interval. The values
5123 // of the map are the index of the 'last seen' usage of the
5124 // instruction that is the key.
5125 using IntervalMap = DenseMap<Instruction *, unsigned>;
5126
5127 // Maps instruction to its index.
5128 SmallVector<Instruction *, 64> IdxToInstr;
5129 // Marks the end of each interval.
5130 IntervalMap EndPoint;
5131 // Saves the list of instruction indices that are used in the loop.
5132 SmallPtrSet<Instruction *, 8> Ends;
5133 // Saves the list of values that are used in the loop but are
5134 // defined outside the loop, such as arguments and constants.
5135 SmallPtrSet<Value *, 8> LoopInvariants;
5136
5137 for (BasicBlock *BB : make_range(DFS.beginRPO(), DFS.endRPO())) {
5138 for (Instruction &I : BB->instructionsWithoutDebug()) {
5139 IdxToInstr.push_back(&I);
5140
5141 // Save the end location of each USE.
5142 for (Value *U : I.operands()) {
5143 auto *Instr = dyn_cast<Instruction>(U);
5144
5145 // Ignore non-instruction values such as arguments, constants, etc.
5146 if (!Instr)
5147 continue;
5148
5149 // If this instruction is outside the loop then record it and continue.
5150 if (!TheLoop->contains(Instr)) {
5151 LoopInvariants.insert(Instr);
5152 continue;
5153 }
5154
5155 // Overwrite previous end points.
5156 EndPoint[Instr] = IdxToInstr.size();
5157 Ends.insert(Instr);
5158 }
5159 }
5160 }
5161
5162 // Saves the list of intervals that end with the index in 'key'.
5163 using InstrList = SmallVector<Instruction *, 2>;
5164 DenseMap<unsigned, InstrList> TransposeEnds;
5165
5166 // Transpose the EndPoints to a list of values that end at each index.
5167 for (auto &Interval : EndPoint)
5168 TransposeEnds[Interval.second].push_back(Interval.first);
5169
5170 SmallPtrSet<Instruction *, 8> OpenIntervals;
5171
5172 // Get the size of the widest register.
5173 unsigned MaxSafeDepDist = -1U;
5174 if (Legal->getMaxSafeDepDistBytes() != -1U)
5175 MaxSafeDepDist = Legal->getMaxSafeDepDistBytes() * 8;
5176 unsigned WidestRegister =
5177 std::min(TTI.getRegisterBitWidth(true), MaxSafeDepDist);
5178 const DataLayout &DL = TheFunction->getParent()->getDataLayout();
5179
5180 SmallVector<RegisterUsage, 8> RUs(VFs.size());
5181 SmallVector<unsigned, 8> MaxUsages(VFs.size(), 0);
5182
5183 LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Calculating max register usage:\n"
; } } while (false)
;
5184
5185 // A lambda that gets the register usage for the given type and VF.
5186 auto GetRegUsage = [&DL, WidestRegister](Type *Ty, unsigned VF) {
5187 if (Ty->isTokenTy())
5188 return 0U;
5189 unsigned TypeSize = DL.getTypeSizeInBits(Ty->getScalarType());
5190 return std::max<unsigned>(1, VF * TypeSize / WidestRegister);
5191 };
5192
5193 for (unsigned int i = 0, s = IdxToInstr.size(); i < s; ++i) {
5194 Instruction *I = IdxToInstr[i];
5195
5196 // Remove all of the instructions that end at this location.
5197 InstrList &List = TransposeEnds[i];
5198 for (Instruction *ToRemove : List)
5199 OpenIntervals.erase(ToRemove);
5200
5201 // Ignore instructions that are never used within the loop.
5202 if (Ends.find(I) == Ends.end())
5203 continue;
5204
5205 // Skip ignored values.
5206 if (ValuesToIgnore.find(I) != ValuesToIgnore.end())
5207 continue;
5208
5209 // For each VF find the maximum usage of registers.
5210 for (unsigned j = 0, e = VFs.size(); j < e; ++j) {
5211 if (VFs[j] == 1) {
5212 MaxUsages[j] = std::max(MaxUsages[j], OpenIntervals.size());
5213 continue;
5214 }
5215 collectUniformsAndScalars(VFs[j]);
5216 // Count the number of live intervals.
5217 unsigned RegUsage = 0;
5218 for (auto Inst : OpenIntervals) {
5219 // Skip ignored values for VF > 1.
5220 if (VecValuesToIgnore.find(Inst) != VecValuesToIgnore.end() ||
5221 isScalarAfterVectorization(Inst, VFs[j]))
5222 continue;
5223 RegUsage += GetRegUsage(Inst->getType(), VFs[j]);
5224 }
5225 MaxUsages[j] = std::max(MaxUsages[j], RegUsage);
5226 }
5227
5228 LLVM_DEBUG(dbgs() << "LV(REG): At #" << i << " Interval # "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): At #" <<
i << " Interval # " << OpenIntervals.size() <<
'\n'; } } while (false)
5229 << OpenIntervals.size() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): At #" <<
i << " Interval # " << OpenIntervals.size() <<
'\n'; } } while (false)
;
5230
5231 // Add the current instruction to the list of open intervals.
5232 OpenIntervals.insert(I);
5233 }
5234
5235 for (unsigned i = 0, e = VFs.size(); i < e; ++i) {
5236 unsigned Invariant = 0;
5237 if (VFs[i] == 1)
5238 Invariant = LoopInvariants.size();
5239 else {
5240 for (auto Inst : LoopInvariants)
5241 Invariant += GetRegUsage(Inst->getType(), VFs[i]);
5242 }
5243
5244 LLVM_DEBUG(dbgs() << "LV(REG): VF = " << VFs[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): VF = " <<
VFs[i] << '\n'; } } while (false)
;
5245 LLVM_DEBUG(dbgs() << "LV(REG): Found max usage: " << MaxUsages[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Found max usage: "
<< MaxUsages[i] << '\n'; } } while (false)
;
5246 LLVM_DEBUG(dbgs() << "LV(REG): Found invariant usage: " << Invariantdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Found invariant usage: "
<< Invariant << '\n'; } } while (false)
5247 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV(REG): Found invariant usage: "
<< Invariant << '\n'; } } while (false)
;
5248
5249 RU.LoopInvariantRegs = Invariant;
5250 RU.MaxLocalUsers = MaxUsages[i];
5251 RUs[i] = RU;
5252 }
5253
5254 return RUs;
5255}
5256
5257bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I){
5258 // TODO: Cost model for emulated masked load/store is completely
5259 // broken. This hack guides the cost model to use an artificially
5260 // high enough value to practically disable vectorization with such
5261 // operations, except where previously deployed legality hack allowed
5262 // using very low cost values. This is to avoid regressions coming simply
5263 // from moving "masked load/store" check from legality to cost model.
5264 // Masked Load/Gather emulation was previously never allowed.
5265 // Limited number of Masked Store/Scatter emulation was allowed.
5266 assert(isPredicatedInst(I) && "Expecting a scalar emulated instruction")((isPredicatedInst(I) && "Expecting a scalar emulated instruction"
) ? static_cast<void> (0) : __assert_fail ("isPredicatedInst(I) && \"Expecting a scalar emulated instruction\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5266, __PRETTY_FUNCTION__))
;
5267 return isa<LoadInst>(I) ||
5268 (isa<StoreInst>(I) &&
5269 NumPredStores > NumberOfStoresToPredicate);
5270}
5271
5272void LoopVectorizationCostModel::collectInstsToScalarize(unsigned VF) {
5273 // If we aren't vectorizing the loop, or if we've already collected the
5274 // instructions to scalarize, there's nothing to do. Collection may already
5275 // have occurred if we have a user-selected VF and are now computing the
5276 // expected cost for interleaving.
5277 if (VF < 2 || InstsToScalarize.find(VF) != InstsToScalarize.end())
5278 return;
5279
5280 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
5281 // not profitable to scalarize any instructions, the presence of VF in the
5282 // map will indicate that we've analyzed it already.
5283 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
5284
5285 // Find all the instructions that are scalar with predication in the loop and
5286 // determine if it would be better to not if-convert the blocks they are in.
5287 // If so, we also record the instructions to scalarize.
5288 for (BasicBlock *BB : TheLoop->blocks()) {
5289 if (!blockNeedsPredication(BB))
5290 continue;
5291 for (Instruction &I : *BB)
5292 if (isScalarWithPredication(&I)) {
5293 ScalarCostsTy ScalarCosts;
5294 // Do not apply discount logic if hacked cost is needed
5295 // for emulated masked memrefs.
5296 if (!useEmulatedMaskMemRefHack(&I) &&
5297 computePredInstDiscount(&I, ScalarCosts, VF) >= 0)
5298 ScalarCostsVF.insert(ScalarCosts.begin(), ScalarCosts.end());
5299 // Remember that BB will remain after vectorization.
5300 PredicatedBBsAfterVectorization.insert(BB);
5301 }
5302 }
5303}
5304
5305int LoopVectorizationCostModel::computePredInstDiscount(
5306 Instruction *PredInst, DenseMap<Instruction *, unsigned> &ScalarCosts,
5307 unsigned VF) {
5308 assert(!isUniformAfterVectorization(PredInst, VF) &&((!isUniformAfterVectorization(PredInst, VF) && "Instruction marked uniform-after-vectorization will be predicated"
) ? static_cast<void> (0) : __assert_fail ("!isUniformAfterVectorization(PredInst, VF) && \"Instruction marked uniform-after-vectorization will be predicated\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5309, __PRETTY_FUNCTION__))
5309 "Instruction marked uniform-after-vectorization will be predicated")((!isUniformAfterVectorization(PredInst, VF) && "Instruction marked uniform-after-vectorization will be predicated"
) ? static_cast<void> (0) : __assert_fail ("!isUniformAfterVectorization(PredInst, VF) && \"Instruction marked uniform-after-vectorization will be predicated\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5309, __PRETTY_FUNCTION__))
;
5310
5311 // Initialize the discount to zero, meaning that the scalar version and the
5312 // vector version cost the same.
5313 int Discount = 0;
5314
5315 // Holds instructions to analyze. The instructions we visit are mapped in
5316 // ScalarCosts. Those instructions are the ones that would be scalarized if
5317 // we find that the scalar version costs less.
5318 SmallVector<Instruction *, 8> Worklist;
5319
5320 // Returns true if the given instruction can be scalarized.
5321 auto canBeScalarized = [&](Instruction *I) -> bool {
5322 // We only attempt to scalarize instructions forming a single-use chain
5323 // from the original predicated block that would otherwise be vectorized.
5324 // Although not strictly necessary, we give up on instructions we know will
5325 // already be scalar to avoid traversing chains that are unlikely to be
5326 // beneficial.
5327 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
5328 isScalarAfterVectorization(I, VF))
5329 return false;
5330
5331 // If the instruction is scalar with predication, it will be analyzed
5332 // separately. We ignore it within the context of PredInst.
5333 if (isScalarWithPredication(I))
5334 return false;
5335
5336 // If any of the instruction's operands are uniform after vectorization,
5337 // the instruction cannot be scalarized. This prevents, for example, a
5338 // masked load from being scalarized.
5339 //
5340 // We assume we will only emit a value for lane zero of an instruction
5341 // marked uniform after vectorization, rather than VF identical values.
5342 // Thus, if we scalarize an instruction that uses a uniform, we would
5343 // create uses of values corresponding to the lanes we aren't emitting code
5344 // for. This behavior can be changed by allowing getScalarValue to clone
5345 // the lane zero values for uniforms rather than asserting.
5346 for (Use &U : I->operands())
5347 if (auto *J = dyn_cast<Instruction>(U.get()))
5348 if (isUniformAfterVectorization(J, VF))
5349 return false;
5350
5351 // Otherwise, we can scalarize the instruction.
5352 return true;
5353 };
5354
5355 // Returns true if an operand that cannot be scalarized must be extracted
5356 // from a vector. We will account for this scalarization overhead below. Note
5357 // that the non-void predicated instructions are placed in their own blocks,
5358 // and their return values are inserted into vectors. Thus, an extract would
5359 // still be required.
5360 auto needsExtract = [&](Instruction *I) -> bool {
5361 return TheLoop->contains(I) && !isScalarAfterVectorization(I, VF);
5362 };
5363
5364 // Compute the expected cost discount from scalarizing the entire expression
5365 // feeding the predicated instruction. We currently only consider expressions
5366 // that are single-use instruction chains.
5367 Worklist.push_back(PredInst);
5368 while (!Worklist.empty()) {
5369 Instruction *I = Worklist.pop_back_val();
5370
5371 // If we've already analyzed the instruction, there's nothing to do.
5372 if (ScalarCosts.find(I) != ScalarCosts.end())
5373 continue;
5374
5375 // Compute the cost of the vector instruction. Note that this cost already
5376 // includes the scalarization overhead of the predicated instruction.
5377 unsigned VectorCost = getInstructionCost(I, VF).first;
5378
5379 // Compute the cost of the scalarized instruction. This cost is the cost of
5380 // the instruction as if it wasn't if-converted and instead remained in the
5381 // predicated block. We will scale this cost by block probability after
5382 // computing the scalarization overhead.
5383 unsigned ScalarCost = VF * getInstructionCost(I, 1).first;
5384
5385 // Compute the scalarization overhead of needed insertelement instructions
5386 // and phi nodes.
5387 if (isScalarWithPredication(I) && !I->getType()->isVoidTy()) {
5388 ScalarCost += TTI.getScalarizationOverhead(ToVectorTy(I->getType(), VF),
5389 true, false);
5390 ScalarCost += VF * TTI.getCFInstrCost(Instruction::PHI);
5391 }
5392
5393 // Compute the scalarization overhead of needed extractelement
5394 // instructions. For each of the instruction's operands, if the operand can
5395 // be scalarized, add it to the worklist; otherwise, account for the
5396 // overhead.
5397 for (Use &U : I->operands())
5398 if (auto *J = dyn_cast<Instruction>(U.get())) {
5399 assert(VectorType::isValidElementType(J->getType()) &&((VectorType::isValidElementType(J->getType()) && "Instruction has non-scalar type"
) ? static_cast<void> (0) : __assert_fail ("VectorType::isValidElementType(J->getType()) && \"Instruction has non-scalar type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5400, __PRETTY_FUNCTION__))
5400 "Instruction has non-scalar type")((VectorType::isValidElementType(J->getType()) && "Instruction has non-scalar type"
) ? static_cast<void> (0) : __assert_fail ("VectorType::isValidElementType(J->getType()) && \"Instruction has non-scalar type\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5400, __PRETTY_FUNCTION__))
;
5401 if (canBeScalarized(J))
5402 Worklist.push_back(J);
5403 else if (needsExtract(J))
5404 ScalarCost += TTI.getScalarizationOverhead(
5405 ToVectorTy(J->getType(),VF), false, true);
5406 }
5407
5408 // Scale the total scalar cost by block probability.
5409 ScalarCost /= getReciprocalPredBlockProb();
5410
5411 // Compute the discount. A non-negative discount means the vector version
5412 // of the instruction costs more, and scalarizing would be beneficial.
5413 Discount += VectorCost - ScalarCost;
5414 ScalarCosts[I] = ScalarCost;
5415 }
5416
5417 return Discount;
5418}
5419
5420LoopVectorizationCostModel::VectorizationCostTy
5421LoopVectorizationCostModel::expectedCost(unsigned VF) {
5422 VectorizationCostTy Cost;
5423
5424 // For each block.
5425 for (BasicBlock *BB : TheLoop->blocks()) {
5426 VectorizationCostTy BlockCost;
5427
5428 // For each instruction in the old loop.
5429 for (Instruction &I : BB->instructionsWithoutDebug()) {
5430 // Skip ignored values.
5431 if (ValuesToIgnore.find(&I) != ValuesToIgnore.end() ||
5432 (VF > 1 && VecValuesToIgnore.find(&I) != VecValuesToIgnore.end()))
5433 continue;
5434
5435 VectorizationCostTy C = getInstructionCost(&I, VF);
5436
5437 // Check if we should override the cost.
5438 if (ForceTargetInstructionCost.getNumOccurrences() > 0)
5439 C.first = ForceTargetInstructionCost;
5440
5441 BlockCost.first += C.first;
5442 BlockCost.second |= C.second;
5443 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C.firstdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
<< C.first << " for VF " << VF << " For instruction: "
<< I << '\n'; } } while (false)
5444 << " for VF " << VF << " For instruction: " << Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
<< C.first << " for VF " << VF << " For instruction: "
<< I << '\n'; } } while (false)
5445 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an estimated cost of "
<< C.first << " for VF " << VF << " For instruction: "
<< I << '\n'; } } while (false)
;
5446 }
5447
5448 // If we are vectorizing a predicated block, it will have been
5449 // if-converted. This means that the block's instructions (aside from
5450 // stores and instructions that may divide by zero) will now be
5451 // unconditionally executed. For the scalar case, we may not always execute
5452 // the predicated block. Thus, scale the block's cost by the probability of
5453 // executing it.
5454 if (VF == 1 && blockNeedsPredication(BB))
5455 BlockCost.first /= getReciprocalPredBlockProb();
5456
5457 Cost.first += BlockCost.first;
5458 Cost.second |= BlockCost.second;
5459 }
5460
5461 return Cost;
5462}
5463
5464/// Gets Address Access SCEV after verifying that the access pattern
5465/// is loop invariant except the induction variable dependence.
5466///
5467/// This SCEV can be sent to the Target in order to estimate the address
5468/// calculation cost.
5469static const SCEV *getAddressAccessSCEV(
5470 Value *Ptr,
5471 LoopVectorizationLegality *Legal,
5472 PredicatedScalarEvolution &PSE,
5473 const Loop *TheLoop) {
5474
5475 auto *Gep = dyn_cast<GetElementPtrInst>(Ptr);
5476 if (!Gep)
5477 return nullptr;
5478
5479 // We are looking for a gep with all loop invariant indices except for one
5480 // which should be an induction variable.
5481 auto SE = PSE.getSE();
5482 unsigned NumOperands = Gep->getNumOperands();
5483 for (unsigned i = 1; i < NumOperands; ++i) {
5484 Value *Opd = Gep->getOperand(i);
5485 if (!SE->isLoopInvariant(SE->getSCEV(Opd), TheLoop) &&
5486 !Legal->isInductionVariable(Opd))
5487 return nullptr;
5488 }
5489
5490 // Now we know we have a GEP ptr, %inv, %ind, %inv. return the Ptr SCEV.
5491 return PSE.getSCEV(Ptr);
5492}
5493
5494static bool isStrideMul(Instruction *I, LoopVectorizationLegality *Legal) {
5495 return Legal->hasStride(I->getOperand(0)) ||
5496 Legal->hasStride(I->getOperand(1));
5497}
5498
5499unsigned LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
5500 unsigned VF) {
5501 assert(VF > 1 && "Scalarization cost of instruction implies vectorization.")((VF > 1 && "Scalarization cost of instruction implies vectorization."
) ? static_cast<void> (0) : __assert_fail ("VF > 1 && \"Scalarization cost of instruction implies vectorization.\""
, "/build/llvm-toolchain-snapshot-9~svn358860/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5501, __PRETTY_FUNCTION__))
;
5502 Type *ValTy = getMemInstValueType(I);
5503 auto SE = PSE.getSE();
5504
5505 unsigned Alignment = getLoadStoreAlignment(I);
5506 unsigned AS = getLoadStoreAddressSpace(I);
5507 Value *Ptr = getLoadStorePointerOperand(I);
5508 Type *PtrTy = ToVectorTy(Ptr->getType(), VF);
5509
5510 // Figure out whether the access is strided and get the stride value
5511 // if it's known in compile time
5512 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);
5513
5514 // Get the cost of the scalar memory instruction and address computation.
5515 unsigned Cost = VF * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);
5516
5517 // Don't pass *I here, since it is scalar but will actually be part of a
5518 // vectorized loop where the user of it is a vectorized instruction.
5519 Cost += VF *
5520 TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(), Alignment,
5521 AS);
5522
5523