Bug Summary

File:llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Warning:line 7321, column 35
Potential leak of memory pointed to by 'BlockMask'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LoopVectorize.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-11-24-172238-38865-1 -x c++ /build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/Proposal/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanHCFGBuilder.h"
61#include "VPlanPredicator.h"
62#include "VPlanTransforms.h"
63#include "llvm/ADT/APInt.h"
64#include "llvm/ADT/ArrayRef.h"
65#include "llvm/ADT/DenseMap.h"
66#include "llvm/ADT/DenseMapInfo.h"
67#include "llvm/ADT/Hashing.h"
68#include "llvm/ADT/MapVector.h"
69#include "llvm/ADT/None.h"
70#include "llvm/ADT/Optional.h"
71#include "llvm/ADT/STLExtras.h"
72#include "llvm/ADT/SetVector.h"
73#include "llvm/ADT/SmallPtrSet.h"
74#include "llvm/ADT/SmallVector.h"
75#include "llvm/ADT/Statistic.h"
76#include "llvm/ADT/StringRef.h"
77#include "llvm/ADT/Twine.h"
78#include "llvm/ADT/iterator_range.h"
79#include "llvm/Analysis/AssumptionCache.h"
80#include "llvm/Analysis/BasicAliasAnalysis.h"
81#include "llvm/Analysis/BlockFrequencyInfo.h"
82#include "llvm/Analysis/CFG.h"
83#include "llvm/Analysis/CodeMetrics.h"
84#include "llvm/Analysis/DemandedBits.h"
85#include "llvm/Analysis/GlobalsModRef.h"
86#include "llvm/Analysis/LoopAccessAnalysis.h"
87#include "llvm/Analysis/LoopAnalysisManager.h"
88#include "llvm/Analysis/LoopInfo.h"
89#include "llvm/Analysis/LoopIterator.h"
90#include "llvm/Analysis/MemorySSA.h"
91#include "llvm/Analysis/OptimizationRemarkEmitter.h"
92#include "llvm/Analysis/ProfileSummaryInfo.h"
93#include "llvm/Analysis/ScalarEvolution.h"
94#include "llvm/Analysis/ScalarEvolutionExpressions.h"
95#include "llvm/Analysis/TargetLibraryInfo.h"
96#include "llvm/Analysis/TargetTransformInfo.h"
97#include "llvm/Analysis/VectorUtils.h"
98#include "llvm/IR/Attributes.h"
99#include "llvm/IR/BasicBlock.h"
100#include "llvm/IR/CFG.h"
101#include "llvm/IR/Constant.h"
102#include "llvm/IR/Constants.h"
103#include "llvm/IR/DataLayout.h"
104#include "llvm/IR/DebugInfoMetadata.h"
105#include "llvm/IR/DebugLoc.h"
106#include "llvm/IR/DerivedTypes.h"
107#include "llvm/IR/DiagnosticInfo.h"
108#include "llvm/IR/Dominators.h"
109#include "llvm/IR/Function.h"
110#include "llvm/IR/IRBuilder.h"
111#include "llvm/IR/InstrTypes.h"
112#include "llvm/IR/Instruction.h"
113#include "llvm/IR/Instructions.h"
114#include "llvm/IR/IntrinsicInst.h"
115#include "llvm/IR/Intrinsics.h"
116#include "llvm/IR/LLVMContext.h"
117#include "llvm/IR/Metadata.h"
118#include "llvm/IR/Module.h"
119#include "llvm/IR/Operator.h"
120#include "llvm/IR/Type.h"
121#include "llvm/IR/Use.h"
122#include "llvm/IR/User.h"
123#include "llvm/IR/Value.h"
124#include "llvm/IR/ValueHandle.h"
125#include "llvm/IR/Verifier.h"
126#include "llvm/InitializePasses.h"
127#include "llvm/Pass.h"
128#include "llvm/Support/Casting.h"
129#include "llvm/Support/CommandLine.h"
130#include "llvm/Support/Compiler.h"
131#include "llvm/Support/Debug.h"
132#include "llvm/Support/ErrorHandling.h"
133#include "llvm/Support/MathExtras.h"
134#include "llvm/Support/raw_ostream.h"
135#include "llvm/Transforms/Utils/BasicBlockUtils.h"
136#include "llvm/Transforms/Utils/InjectTLIMappings.h"
137#include "llvm/Transforms/Utils/LoopSimplify.h"
138#include "llvm/Transforms/Utils/LoopUtils.h"
139#include "llvm/Transforms/Utils/LoopVersioning.h"
140#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
141#include "llvm/Transforms/Utils/SizeOpts.h"
142#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
143#include <algorithm>
144#include <cassert>
145#include <cstdint>
146#include <cstdlib>
147#include <functional>
148#include <iterator>
149#include <limits>
150#include <memory>
151#include <string>
152#include <tuple>
153#include <utility>
154
155using namespace llvm;
156
157#define LV_NAME"loop-vectorize" "loop-vectorize"
158#define DEBUG_TYPE"loop-vectorize" LV_NAME"loop-vectorize"
159
160/// @{
161/// Metadata attribute names
162static const char *const LLVMLoopVectorizeFollowupAll =
163 "llvm.loop.vectorize.followup_all";
164static const char *const LLVMLoopVectorizeFollowupVectorized =
165 "llvm.loop.vectorize.followup_vectorized";
166static const char *const LLVMLoopVectorizeFollowupEpilogue =
167 "llvm.loop.vectorize.followup_epilogue";
168/// @}
169
170STATISTIC(LoopsVectorized, "Number of loops vectorized")static llvm::Statistic LoopsVectorized = {"loop-vectorize", "LoopsVectorized"
, "Number of loops vectorized"}
;
171STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization")static llvm::Statistic LoopsAnalyzed = {"loop-vectorize", "LoopsAnalyzed"
, "Number of loops analyzed for vectorization"}
;
172
173/// Loops with a known constant trip count below this number are vectorized only
174/// if no scalar iteration overheads are incurred.
175static cl::opt<unsigned> TinyTripCountVectorThreshold(
176 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
177 cl::desc("Loops with a constant trip count that is smaller than this "
178 "value are vectorized only if no scalar iteration overheads "
179 "are incurred."));
180
181// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
182// that predication is preferred, and this lists all options. I.e., the
183// vectorizer will try to fold the tail-loop (epilogue) into the vector body
184// and predicate the instructions accordingly. If tail-folding fails, there are
185// different fallback strategies depending on these values:
186namespace PreferPredicateTy {
187 enum Option {
188 ScalarEpilogue = 0,
189 PredicateElseScalarEpilogue,
190 PredicateOrDontVectorize
191 };
192} // namespace PreferPredicateTy
193
194static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
195 "prefer-predicate-over-epilogue",
196 cl::init(PreferPredicateTy::ScalarEpilogue),
197 cl::Hidden,
198 cl::desc("Tail-folding and predication preferences over creating a scalar "
199 "epilogue loop."),
200 cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
201 "scalar-epilogue",llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
202 "Don't tail-predicate loops, create scalar epilogue")llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
,
203 clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
204 "predicate-else-scalar-epilogue",llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
205 "prefer tail-folding, create scalar epilogue if tail "llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
206 "folding fails.")llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
,
207 clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
208 "predicate-dont-vectorize",llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
209 "prefers tail-folding, don't attempt vectorization if "llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
210 "tail-folding fails.")llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
));
211
212static cl::opt<bool> MaximizeBandwidth(
213 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
214 cl::desc("Maximize bandwidth when selecting vectorization factor which "
215 "will be determined by the smallest type in loop."));
216
217static cl::opt<bool> EnableInterleavedMemAccesses(
218 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
219 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
220
221/// An interleave-group may need masking if it resides in a block that needs
222/// predication, or in order to mask away gaps.
223static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
224 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
225 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
226
227static cl::opt<unsigned> TinyTripCountInterleaveThreshold(
228 "tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden,
229 cl::desc("We don't interleave loops with a estimated constant trip count "
230 "below this number"));
231
232static cl::opt<unsigned> ForceTargetNumScalarRegs(
233 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
234 cl::desc("A flag that overrides the target's number of scalar registers."));
235
236static cl::opt<unsigned> ForceTargetNumVectorRegs(
237 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
238 cl::desc("A flag that overrides the target's number of vector registers."));
239
240static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
241 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
242 cl::desc("A flag that overrides the target's max interleave factor for "
243 "scalar loops."));
244
245static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
246 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
247 cl::desc("A flag that overrides the target's max interleave factor for "
248 "vectorized loops."));
249
250static cl::opt<unsigned> ForceTargetInstructionCost(
251 "force-target-instruction-cost", cl::init(0), cl::Hidden,
252 cl::desc("A flag that overrides the target's expected cost for "
253 "an instruction to a single constant value. Mostly "
254 "useful for getting consistent testing."));
255
256static cl::opt<unsigned> SmallLoopCost(
257 "small-loop-cost", cl::init(20), cl::Hidden,
258 cl::desc(
259 "The cost of a loop that is considered 'small' by the interleaver."));
260
261static cl::opt<bool> LoopVectorizeWithBlockFrequency(
262 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
263 cl::desc("Enable the use of the block frequency analysis to access PGO "
264 "heuristics minimizing code growth in cold regions and being more "
265 "aggressive in hot regions."));
266
267// Runtime interleave loops for load/store throughput.
268static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
269 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
270 cl::desc(
271 "Enable runtime interleaving until load/store ports are saturated"));
272
273/// Interleave small loops with scalar reductions.
274static cl::opt<bool> InterleaveSmallLoopScalarReduction(
275 "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden,
276 cl::desc("Enable interleaving for loops with small iteration counts that "
277 "contain scalar reductions to expose ILP."));
278
279/// The number of stores in a loop that are allowed to need predication.
280static cl::opt<unsigned> NumberOfStoresToPredicate(
281 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
282 cl::desc("Max number of stores to be predicated behind an if."));
283
284static cl::opt<bool> EnableIndVarRegisterHeur(
285 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
286 cl::desc("Count the induction variable only once when interleaving"));
287
288static cl::opt<bool> EnableCondStoresVectorization(
289 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
290 cl::desc("Enable if predication of stores during vectorization."));
291
292static cl::opt<unsigned> MaxNestedScalarReductionIC(
293 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
294 cl::desc("The maximum interleave count to use when interleaving a scalar "
295 "reduction in a nested loop."));
296
297static cl::opt<bool>
298 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
299 cl::Hidden,
300 cl::desc("Prefer in-loop vector reductions, "
301 "overriding the targets preference."));
302
303static cl::opt<bool> PreferPredicatedReductionSelect(
304 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
305 cl::desc(
306 "Prefer predicating a reduction operation over an after loop select."));
307
308cl::opt<bool> EnableVPlanNativePath(
309 "enable-vplan-native-path", cl::init(false), cl::Hidden,
310 cl::desc("Enable VPlan-native vectorization path with "
311 "support for outer loop vectorization."));
312
313// FIXME: Remove this switch once we have divergence analysis. Currently we
314// assume divergent non-backedge branches when this switch is true.
315cl::opt<bool> EnableVPlanPredication(
316 "enable-vplan-predication", cl::init(false), cl::Hidden,
317 cl::desc("Enable VPlan-native vectorization path predicator with "
318 "support for outer loop vectorization."));
319
320// This flag enables the stress testing of the VPlan H-CFG construction in the
321// VPlan-native vectorization path. It must be used in conjuction with
322// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
323// verification of the H-CFGs built.
324static cl::opt<bool> VPlanBuildStressTest(
325 "vplan-build-stress-test", cl::init(false), cl::Hidden,
326 cl::desc(
327 "Build VPlan for every supported loop nest in the function and bail "
328 "out right after the build (stress test the VPlan H-CFG construction "
329 "in the VPlan-native vectorization path)."));
330
331cl::opt<bool> llvm::EnableLoopInterleaving(
332 "interleave-loops", cl::init(true), cl::Hidden,
333 cl::desc("Enable loop interleaving in Loop vectorization passes"));
334cl::opt<bool> llvm::EnableLoopVectorization(
335 "vectorize-loops", cl::init(true), cl::Hidden,
336 cl::desc("Run the Loop vectorization passes"));
337
338/// A helper function that returns the type of loaded or stored value.
339static Type *getMemInstValueType(Value *I) {
340 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction") ? static_cast<void>
(0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 341, __PRETTY_FUNCTION__))
341 "Expected Load or Store instruction")(((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction") ? static_cast<void>
(0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Expected Load or Store instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 341, __PRETTY_FUNCTION__))
;
342 if (auto *LI = dyn_cast<LoadInst>(I))
343 return LI->getType();
344 return cast<StoreInst>(I)->getValueOperand()->getType();
345}
346
347/// A helper function that returns true if the given type is irregular. The
348/// type is irregular if its allocated size doesn't equal the store size of an
349/// element of the corresponding vector type at the given vectorization factor.
350static bool hasIrregularType(Type *Ty, const DataLayout &DL, ElementCount VF) {
351 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 351, __PRETTY_FUNCTION__))
;
352 // Determine if an array of VF elements of type Ty is "bitcast compatible"
353 // with a <VF x Ty> vector.
354 if (VF.isVector()) {
355 auto *VectorTy = VectorType::get(Ty, VF);
356 return TypeSize::get(VF.getKnownMinValue() *
357 DL.getTypeAllocSize(Ty).getFixedValue(),
358 VF.isScalable()) != DL.getTypeStoreSize(VectorTy);
359 }
360
361 // If the vectorization factor is one, we just check if an array of type Ty
362 // requires padding between elements.
363 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
364}
365
366/// A helper function that returns the reciprocal of the block probability of
367/// predicated blocks. If we return X, we are assuming the predicated block
368/// will execute once for every X iterations of the loop header.
369///
370/// TODO: We should use actual block probability here, if available. Currently,
371/// we always assume predicated blocks have a 50% chance of executing.
372static unsigned getReciprocalPredBlockProb() { return 2; }
373
374/// A helper function that adds a 'fast' flag to floating-point operations.
375static Value *addFastMathFlag(Value *V) {
376 if (isa<FPMathOperator>(V))
377 cast<Instruction>(V)->setFastMathFlags(FastMathFlags::getFast());
378 return V;
379}
380
381static Value *addFastMathFlag(Value *V, FastMathFlags FMF) {
382 if (isa<FPMathOperator>(V))
383 cast<Instruction>(V)->setFastMathFlags(FMF);
384 return V;
385}
386
387/// A helper function that returns an integer or floating-point constant with
388/// value C.
389static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
390 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
391 : ConstantFP::get(Ty, C);
392}
393
394/// Returns "best known" trip count for the specified loop \p L as defined by
395/// the following procedure:
396/// 1) Returns exact trip count if it is known.
397/// 2) Returns expected trip count according to profile data if any.
398/// 3) Returns upper bound estimate if it is known.
399/// 4) Returns None if all of the above failed.
400static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) {
401 // Check if exact trip count is known.
402 if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L))
403 return ExpectedTC;
404
405 // Check if there is an expected trip count available from profile data.
406 if (LoopVectorizeWithBlockFrequency)
407 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
408 return EstimatedTC;
409
410 // Check if upper bound estimate is known.
411 if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
412 return ExpectedTC;
413
414 return None;
415}
416
417namespace llvm {
418
419/// InnerLoopVectorizer vectorizes loops which contain only one basic
420/// block to a specified vectorization factor (VF).
421/// This class performs the widening of scalars into vectors, or multiple
422/// scalars. This class also implements the following features:
423/// * It inserts an epilogue loop for handling loops that don't have iteration
424/// counts that are known to be a multiple of the vectorization factor.
425/// * It handles the code generation for reduction variables.
426/// * Scalarization (implementation using scalars) of un-vectorizable
427/// instructions.
428/// InnerLoopVectorizer does not perform any vectorization-legality
429/// checks, and relies on the caller to check for the different legality
430/// aspects. The InnerLoopVectorizer relies on the
431/// LoopVectorizationLegality class to provide information about the induction
432/// and reduction variables that were found to a given vectorization factor.
433class InnerLoopVectorizer {
434public:
435 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
436 LoopInfo *LI, DominatorTree *DT,
437 const TargetLibraryInfo *TLI,
438 const TargetTransformInfo *TTI, AssumptionCache *AC,
439 OptimizationRemarkEmitter *ORE, ElementCount VecWidth,
440 unsigned UnrollFactor, LoopVectorizationLegality *LVL,
441 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
442 ProfileSummaryInfo *PSI)
443 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
444 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
445 Builder(PSE.getSE()->getContext()),
446 VectorLoopValueMap(UnrollFactor, VecWidth), Legal(LVL), Cost(CM),
447 BFI(BFI), PSI(PSI) {
448 // Query this against the original loop and save it here because the profile
449 // of the original loop header may change as the transformation happens.
450 OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
451 OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
452 }
453
454 virtual ~InnerLoopVectorizer() = default;
455
456 /// Create a new empty loop that will contain vectorized instructions later
457 /// on, while the old loop will be used as the scalar remainder. Control flow
458 /// is generated around the vectorized (and scalar epilogue) loops consisting
459 /// of various checks and bypasses. Return the pre-header block of the new
460 /// loop.
461 BasicBlock *createVectorizedLoopSkeleton();
462
463 /// Widen a single instruction within the innermost loop.
464 void widenInstruction(Instruction &I, VPUser &Operands,
465 VPTransformState &State);
466
467 /// Widen a single call instruction within the innermost loop.
468 void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands,
469 VPTransformState &State);
470
471 /// Widen a single select instruction within the innermost loop.
472 void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands,
473 bool InvariantCond, VPTransformState &State);
474
475 /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
476 void fixVectorizedLoop();
477
478 // Return true if any runtime check is added.
479 bool areSafetyChecksAdded() { return AddedSafetyChecks; }
480
481 /// A type for vectorized values in the new loop. Each value from the
482 /// original loop, when vectorized, is represented by UF vector values in the
483 /// new unrolled loop, where UF is the unroll factor.
484 using VectorParts = SmallVector<Value *, 2>;
485
486 /// Vectorize a single GetElementPtrInst based on information gathered and
487 /// decisions taken during planning.
488 void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices,
489 unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
490 SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
491
492 /// Vectorize a single PHINode in a block. This method handles the induction
493 /// variable canonicalization. It supports both VF = 1 for unrolled loops and
494 /// arbitrary length vectors.
495 void widenPHIInstruction(Instruction *PN, unsigned UF, ElementCount VF);
496
497 /// A helper function to scalarize a single Instruction in the innermost loop.
498 /// Generates a sequence of scalar instances for each lane between \p MinLane
499 /// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
500 /// inclusive. Uses the VPValue operands from \p Operands instead of \p
501 /// Instr's operands.
502 void scalarizeInstruction(Instruction *Instr, VPUser &Operands,
503 const VPIteration &Instance, bool IfPredicateInstr,
504 VPTransformState &State);
505
506 /// Widen an integer or floating-point induction variable \p IV. If \p Trunc
507 /// is provided, the integer induction variable will first be truncated to
508 /// the corresponding type.
509 void widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc = nullptr);
510
511 /// getOrCreateVectorValue and getOrCreateScalarValue coordinate to generate a
512 /// vector or scalar value on-demand if one is not yet available. When
513 /// vectorizing a loop, we visit the definition of an instruction before its
514 /// uses. When visiting the definition, we either vectorize or scalarize the
515 /// instruction, creating an entry for it in the corresponding map. (In some
516 /// cases, such as induction variables, we will create both vector and scalar
517 /// entries.) Then, as we encounter uses of the definition, we derive values
518 /// for each scalar or vector use unless such a value is already available.
519 /// For example, if we scalarize a definition and one of its uses is vector,
520 /// we build the required vector on-demand with an insertelement sequence
521 /// when visiting the use. Otherwise, if the use is scalar, we can use the
522 /// existing scalar definition.
523 ///
524 /// Return a value in the new loop corresponding to \p V from the original
525 /// loop at unroll index \p Part. If the value has already been vectorized,
526 /// the corresponding vector entry in VectorLoopValueMap is returned. If,
527 /// however, the value has a scalar entry in VectorLoopValueMap, we construct
528 /// a new vector value on-demand by inserting the scalar values into a vector
529 /// with an insertelement sequence. If the value has been neither vectorized
530 /// nor scalarized, it must be loop invariant, so we simply broadcast the
531 /// value into a vector.
532 Value *getOrCreateVectorValue(Value *V, unsigned Part);
533
534 void setVectorValue(Value *Scalar, unsigned Part, Value *Vector) {
535 VectorLoopValueMap.setVectorValue(Scalar, Part, Vector);
536 }
537
538 /// Return a value in the new loop corresponding to \p V from the original
539 /// loop at unroll and vector indices \p Instance. If the value has been
540 /// vectorized but not scalarized, the necessary extractelement instruction
541 /// will be generated.
542 Value *getOrCreateScalarValue(Value *V, const VPIteration &Instance);
543
544 /// Construct the vector value of a scalarized value \p V one lane at a time.
545 void packScalarIntoVectorValue(Value *V, const VPIteration &Instance);
546
547 /// Try to vectorize interleaved access group \p Group with the base address
548 /// given in \p Addr, optionally masking the vector operations if \p
549 /// BlockInMask is non-null. Use \p State to translate given VPValues to IR
550 /// values in the vectorized loop.
551 void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
552 VPTransformState &State, VPValue *Addr,
553 VPValue *BlockInMask = nullptr);
554
555 /// Vectorize Load and Store instructions with the base address given in \p
556 /// Addr, optionally masking the vector operations if \p BlockInMask is
557 /// non-null. Use \p State to translate given VPValues to IR values in the
558 /// vectorized loop.
559 void vectorizeMemoryInstruction(Instruction *Instr, VPTransformState &State,
560 VPValue *Def, VPValue *Addr,
561 VPValue *StoredValue, VPValue *BlockInMask);
562
563 /// Set the debug location in the builder using the debug location in
564 /// the instruction.
565 void setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr);
566
567 /// Fix the non-induction PHIs in the OrigPHIsToFix vector.
568 void fixNonInductionPHIs(void);
569
570protected:
571 friend class LoopVectorizationPlanner;
572
573 /// A small list of PHINodes.
574 using PhiVector = SmallVector<PHINode *, 4>;
575
576 /// A type for scalarized values in the new loop. Each value from the
577 /// original loop, when scalarized, is represented by UF x VF scalar values
578 /// in the new unrolled loop, where UF is the unroll factor and VF is the
579 /// vectorization factor.
580 using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
581
582 /// Set up the values of the IVs correctly when exiting the vector loop.
583 void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
584 Value *CountRoundDown, Value *EndValue,
585 BasicBlock *MiddleBlock);
586
587 /// Create a new induction variable inside L.
588 PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
589 Value *Step, Instruction *DL);
590
591 /// Handle all cross-iteration phis in the header.
592 void fixCrossIterationPHIs();
593
594 /// Fix a first-order recurrence. This is the second phase of vectorizing
595 /// this phi node.
596 void fixFirstOrderRecurrence(PHINode *Phi);
597
598 /// Fix a reduction cross-iteration phi. This is the second phase of
599 /// vectorizing this phi node.
600 void fixReduction(PHINode *Phi);
601
602 /// Clear NSW/NUW flags from reduction instructions if necessary.
603 void clearReductionWrapFlags(RecurrenceDescriptor &RdxDesc);
604
605 /// The Loop exit block may have single value PHI nodes with some
606 /// incoming value. While vectorizing we only handled real values
607 /// that were defined inside the loop and we should have one value for
608 /// each predecessor of its parent basic block. See PR14725.
609 void fixLCSSAPHIs();
610
611 /// Iteratively sink the scalarized operands of a predicated instruction into
612 /// the block that was created for it.
613 void sinkScalarOperands(Instruction *PredInst);
614
615 /// Shrinks vector element sizes to the smallest bitwidth they can be legally
616 /// represented as.
617 void truncateToMinimalBitwidths();
618
619 /// Create a broadcast instruction. This method generates a broadcast
620 /// instruction (shuffle) for loop invariant values and for the induction
621 /// value. If this is the induction variable then we extend it to N, N+1, ...
622 /// this is needed because each iteration in the loop corresponds to a SIMD
623 /// element.
624 virtual Value *getBroadcastInstrs(Value *V);
625
626 /// This function adds (StartIdx, StartIdx + Step, StartIdx + 2*Step, ...)
627 /// to each vector element of Val. The sequence starts at StartIndex.
628 /// \p Opcode is relevant for FP induction variable.
629 virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step,
630 Instruction::BinaryOps Opcode =
631 Instruction::BinaryOpsEnd);
632
633 /// Compute scalar induction steps. \p ScalarIV is the scalar induction
634 /// variable on which to base the steps, \p Step is the size of the step, and
635 /// \p EntryVal is the value from the original loop that maps to the steps.
636 /// Note that \p EntryVal doesn't have to be an induction variable - it
637 /// can also be a truncate instruction.
638 void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal,
639 const InductionDescriptor &ID);
640
641 /// Create a vector induction phi node based on an existing scalar one. \p
642 /// EntryVal is the value from the original loop that maps to the vector phi
643 /// node, and \p Step is the loop-invariant step. If \p EntryVal is a
644 /// truncate instruction, instead of widening the original IV, we widen a
645 /// version of the IV truncated to \p EntryVal's type.
646 void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
647 Value *Step, Instruction *EntryVal);
648
649 /// Returns true if an instruction \p I should be scalarized instead of
650 /// vectorized for the chosen vectorization factor.
651 bool shouldScalarizeInstruction(Instruction *I) const;
652
653 /// Returns true if we should generate a scalar version of \p IV.
654 bool needsScalarInduction(Instruction *IV) const;
655
656 /// If there is a cast involved in the induction variable \p ID, which should
657 /// be ignored in the vectorized loop body, this function records the
658 /// VectorLoopValue of the respective Phi also as the VectorLoopValue of the
659 /// cast. We had already proved that the casted Phi is equal to the uncasted
660 /// Phi in the vectorized loop (under a runtime guard), and therefore
661 /// there is no need to vectorize the cast - the same value can be used in the
662 /// vector loop for both the Phi and the cast.
663 /// If \p VectorLoopValue is a scalarized value, \p Lane is also specified,
664 /// Otherwise, \p VectorLoopValue is a widened/vectorized value.
665 ///
666 /// \p EntryVal is the value from the original loop that maps to the vector
667 /// phi node and is used to distinguish what is the IV currently being
668 /// processed - original one (if \p EntryVal is a phi corresponding to the
669 /// original IV) or the "newly-created" one based on the proof mentioned above
670 /// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the
671 /// latter case \p EntryVal is a TruncInst and we must not record anything for
672 /// that IV, but it's error-prone to expect callers of this routine to care
673 /// about that, hence this explicit parameter.
674 void recordVectorLoopValueForInductionCast(const InductionDescriptor &ID,
675 const Instruction *EntryVal,
676 Value *VectorLoopValue,
677 unsigned Part,
678 unsigned Lane = UINT_MAX(2147483647 *2U +1U));
679
680 /// Generate a shuffle sequence that will reverse the vector Vec.
681 virtual Value *reverseVector(Value *Vec);
682
683 /// Returns (and creates if needed) the original loop trip count.
684 Value *getOrCreateTripCount(Loop *NewLoop);
685
686 /// Returns (and creates if needed) the trip count of the widened loop.
687 Value *getOrCreateVectorTripCount(Loop *NewLoop);
688
689 /// Returns a bitcasted value to the requested vector type.
690 /// Also handles bitcasts of vector<float> <-> vector<pointer> types.
691 Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
692 const DataLayout &DL);
693
694 /// Emit a bypass check to see if the vector trip count is zero, including if
695 /// it overflows.
696 void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);
697
698 /// Emit a bypass check to see if all of the SCEV assumptions we've
699 /// had to make are correct.
700 void emitSCEVChecks(Loop *L, BasicBlock *Bypass);
701
702 /// Emit bypass checks to check any memory assumptions we may have made.
703 void emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);
704
705 /// Compute the transformed value of Index at offset StartValue using step
706 /// StepValue.
707 /// For integer induction, returns StartValue + Index * StepValue.
708 /// For pointer induction, returns StartValue[Index * StepValue].
709 /// FIXME: The newly created binary instructions should contain nsw/nuw
710 /// flags, which can be found from the original scalar operations.
711 Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
712 const DataLayout &DL,
713 const InductionDescriptor &ID) const;
714
715 /// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
716 /// vector loop preheader, middle block and scalar preheader. Also
717 /// allocate a loop object for the new vector loop and return it.
718 Loop *createVectorLoopSkeleton(StringRef Prefix);
719
720 /// Create new phi nodes for the induction variables to resume iteration count
721 /// in the scalar epilogue, from where the vectorized loop left off (given by
722 /// \p VectorTripCount).
723 void createInductionResumeValues(Loop *L, Value *VectorTripCount);
724
725 /// Complete the loop skeleton by adding debug MDs, creating appropriate
726 /// conditional branches in the middle block, preparing the builder and
727 /// running the verifier. Take in the vector loop \p L as argument, and return
728 /// the preheader of the completed vector loop.
729 BasicBlock *completeLoopSkeleton(Loop *L, MDNode *OrigLoopID);
730
731 /// Add additional metadata to \p To that was not present on \p Orig.
732 ///
733 /// Currently this is used to add the noalias annotations based on the
734 /// inserted memchecks. Use this for instructions that are *cloned* into the
735 /// vector loop.
736 void addNewMetadata(Instruction *To, const Instruction *Orig);
737
738 /// Add metadata from one instruction to another.
739 ///
740 /// This includes both the original MDs from \p From and additional ones (\see
741 /// addNewMetadata). Use this for *newly created* instructions in the vector
742 /// loop.
743 void addMetadata(Instruction *To, Instruction *From);
744
745 /// Similar to the previous function but it adds the metadata to a
746 /// vector of instructions.
747 void addMetadata(ArrayRef<Value *> To, Instruction *From);
748
749 /// The original loop.
750 Loop *OrigLoop;
751
752 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
753 /// dynamic knowledge to simplify SCEV expressions and converts them to a
754 /// more usable form.
755 PredicatedScalarEvolution &PSE;
756
757 /// Loop Info.
758 LoopInfo *LI;
759
760 /// Dominator Tree.
761 DominatorTree *DT;
762
763 /// Alias Analysis.
764 AAResults *AA;
765
766 /// Target Library Info.
767 const TargetLibraryInfo *TLI;
768
769 /// Target Transform Info.
770 const TargetTransformInfo *TTI;
771
772 /// Assumption Cache.
773 AssumptionCache *AC;
774
775 /// Interface to emit optimization remarks.
776 OptimizationRemarkEmitter *ORE;
777
778 /// LoopVersioning. It's only set up (non-null) if memchecks were
779 /// used.
780 ///
781 /// This is currently only used to add no-alias metadata based on the
782 /// memchecks. The actually versioning is performed manually.
783 std::unique_ptr<LoopVersioning> LVer;
784
785 /// The vectorization SIMD factor to use. Each vector will have this many
786 /// vector elements.
787 ElementCount VF;
788
789 /// The vectorization unroll factor to use. Each scalar is vectorized to this
790 /// many different vector instructions.
791 unsigned UF;
792
793 /// The builder that we use
794 IRBuilder<> Builder;
795
796 // --- Vectorization state ---
797
798 /// The vector-loop preheader.
799 BasicBlock *LoopVectorPreHeader;
800
801 /// The scalar-loop preheader.
802 BasicBlock *LoopScalarPreHeader;
803
804 /// Middle Block between the vector and the scalar.
805 BasicBlock *LoopMiddleBlock;
806
807 /// The ExitBlock of the scalar loop.
808 BasicBlock *LoopExitBlock;
809
810 /// The vector loop body.
811 BasicBlock *LoopVectorBody;
812
813 /// The scalar loop body.
814 BasicBlock *LoopScalarBody;
815
816 /// A list of all bypass blocks. The first block is the entry of the loop.
817 SmallVector<BasicBlock *, 4> LoopBypassBlocks;
818
819 /// The new Induction variable which was added to the new block.
820 PHINode *Induction = nullptr;
821
822 /// The induction variable of the old basic block.
823 PHINode *OldInduction = nullptr;
824
825 /// Maps values from the original loop to their corresponding values in the
826 /// vectorized loop. A key value can map to either vector values, scalar
827 /// values or both kinds of values, depending on whether the key was
828 /// vectorized and scalarized.
829 VectorizerValueMap VectorLoopValueMap;
830
831 /// Store instructions that were predicated.
832 SmallVector<Instruction *, 4> PredicatedInstructions;
833
834 /// Trip count of the original loop.
835 Value *TripCount = nullptr;
836
837 /// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
838 Value *VectorTripCount = nullptr;
839
840 /// The legality analysis.
841 LoopVectorizationLegality *Legal;
842
843 /// The profitablity analysis.
844 LoopVectorizationCostModel *Cost;
845
846 // Record whether runtime checks are added.
847 bool AddedSafetyChecks = false;
848
849 // Holds the end values for each induction variable. We save the end values
850 // so we can later fix-up the external users of the induction variables.
851 DenseMap<PHINode *, Value *> IVEndValues;
852
853 // Vector of original scalar PHIs whose corresponding widened PHIs need to be
854 // fixed up at the end of vector code generation.
855 SmallVector<PHINode *, 8> OrigPHIsToFix;
856
857 /// BFI and PSI are used to check for profile guided size optimizations.
858 BlockFrequencyInfo *BFI;
859 ProfileSummaryInfo *PSI;
860
861 // Whether this loop should be optimized for size based on profile guided size
862 // optimizatios.
863 bool OptForSizeBasedOnProfile;
864};
865
866class InnerLoopUnroller : public InnerLoopVectorizer {
867public:
868 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
869 LoopInfo *LI, DominatorTree *DT,
870 const TargetLibraryInfo *TLI,
871 const TargetTransformInfo *TTI, AssumptionCache *AC,
872 OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
873 LoopVectorizationLegality *LVL,
874 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
875 ProfileSummaryInfo *PSI)
876 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
877 ElementCount::getFixed(1), UnrollFactor, LVL, CM,
878 BFI, PSI) {}
879
880private:
881 Value *getBroadcastInstrs(Value *V) override;
882 Value *getStepVector(Value *Val, int StartIdx, Value *Step,
883 Instruction::BinaryOps Opcode =
884 Instruction::BinaryOpsEnd) override;
885 Value *reverseVector(Value *Vec) override;
886};
887
888} // end namespace llvm
889
890/// Look for a meaningful debug location on the instruction or it's
891/// operands.
892static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
893 if (!I)
894 return I;
895
896 DebugLoc Empty;
897 if (I->getDebugLoc() != Empty)
898 return I;
899
900 for (User::op_iterator OI = I->op_begin(), OE = I->op_end(); OI != OE; ++OI) {
901 if (Instruction *OpInst = dyn_cast<Instruction>(*OI))
902 if (OpInst->getDebugLoc() != Empty)
903 return OpInst;
904 }
905
906 return I;
907}
908
909void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr) {
910 if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) {
911 const DILocation *DIL = Inst->getDebugLoc();
912 if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
913 !isa<DbgInfoIntrinsic>(Inst)) {
914 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 914, __PRETTY_FUNCTION__))
;
915 auto NewDIL =
916 DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue());
917 if (NewDIL)
918 B.SetCurrentDebugLocation(NewDIL.getValue());
919 else
920 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
921 << "Failed to create new discriminator: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
922 << DIL->getFilename() << " Line: " << DIL->getLine())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
;
923 }
924 else
925 B.SetCurrentDebugLocation(DIL);
926 } else
927 B.SetCurrentDebugLocation(DebugLoc());
928}
929
930/// Write a record \p DebugMsg about vectorization failure to the debug
931/// output stream. If \p I is passed, it is an instruction that prevents
932/// vectorization.
933#ifndef NDEBUG
934static void debugVectorizationFailure(const StringRef DebugMsg,
935 Instruction *I) {
936 dbgs() << "LV: Not vectorizing: " << DebugMsg;
937 if (I != nullptr)
938 dbgs() << " " << *I;
939 else
940 dbgs() << '.';
941 dbgs() << '\n';
942}
943#endif
944
945/// Create an analysis remark that explains why vectorization failed
946///
947/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
948/// RemarkName is the identifier for the remark. If \p I is passed it is an
949/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
950/// the location of the remark. \return the remark object that can be
951/// streamed to.
952static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
953 StringRef RemarkName, Loop *TheLoop, Instruction *I) {
954 Value *CodeRegion = TheLoop->getHeader();
955 DebugLoc DL = TheLoop->getStartLoc();
956
957 if (I) {
958 CodeRegion = I->getParent();
959 // If there is no debug location attached to the instruction, revert back to
960 // using the loop's.
961 if (I->getDebugLoc())
962 DL = I->getDebugLoc();
963 }
964
965 OptimizationRemarkAnalysis R(PassName, RemarkName, DL, CodeRegion);
966 R << "loop not vectorized: ";
967 return R;
968}
969
970namespace llvm {
971
972void reportVectorizationFailure(const StringRef DebugMsg,
973 const StringRef OREMsg, const StringRef ORETag,
974 OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I) {
975 LLVM_DEBUG(debugVectorizationFailure(DebugMsg, I))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { debugVectorizationFailure(DebugMsg, I);
} } while (false)
;
976 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
977 ORE->emit(createLVAnalysis(Hints.vectorizeAnalysisPassName(),
978 ORETag, TheLoop, I) << OREMsg);
979}
980
981} // end namespace llvm
982
983#ifndef NDEBUG
984/// \return string containing a file name and a line # for the given loop.
985static std::string getDebugLocString(const Loop *L) {
986 std::string Result;
987 if (L) {
988 raw_string_ostream OS(Result);
989 if (const DebugLoc LoopDbgLoc = L->getStartLoc())
990 LoopDbgLoc.print(OS);
991 else
992 // Just print the module name.
993 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
994 OS.flush();
995 }
996 return Result;
997}
998#endif
999
1000void InnerLoopVectorizer::addNewMetadata(Instruction *To,
1001 const Instruction *Orig) {
1002 // If the loop was versioned with memchecks, add the corresponding no-alias
1003 // metadata.
1004 if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
1005 LVer->annotateInstWithNoAlias(To, Orig);
1006}
1007
1008void InnerLoopVectorizer::addMetadata(Instruction *To,
1009 Instruction *From) {
1010 propagateMetadata(To, From);
1011 addNewMetadata(To, From);
1012}
1013
1014void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,
1015 Instruction *From) {
1016 for (Value *V : To) {
1017 if (Instruction *I = dyn_cast<Instruction>(V))
1018 addMetadata(I, From);
1019 }
1020}
1021
1022namespace llvm {
1023
1024// Loop vectorization cost-model hints how the scalar epilogue loop should be
1025// lowered.
1026enum ScalarEpilogueLowering {
1027
1028 // The default: allowing scalar epilogues.
1029 CM_ScalarEpilogueAllowed,
1030
1031 // Vectorization with OptForSize: don't allow epilogues.
1032 CM_ScalarEpilogueNotAllowedOptSize,
1033
1034 // A special case of vectorisation with OptForSize: loops with a very small
1035 // trip count are considered for vectorization under OptForSize, thereby
1036 // making sure the cost of their loop body is dominant, free of runtime
1037 // guards and scalar iteration overheads.
1038 CM_ScalarEpilogueNotAllowedLowTripLoop,
1039
1040 // Loop hint predicate indicating an epilogue is undesired.
1041 CM_ScalarEpilogueNotNeededUsePredicate
1042};
1043
1044/// LoopVectorizationCostModel - estimates the expected speedups due to
1045/// vectorization.
1046/// In many cases vectorization is not profitable. This can happen because of
1047/// a number of reasons. In this class we mainly attempt to predict the
1048/// expected speedup/slowdowns due to the supported instruction set. We use the
1049/// TargetTransformInfo to query the different backends for the cost of
1050/// different operations.
1051class LoopVectorizationCostModel {
1052public:
1053 LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L,
1054 PredicatedScalarEvolution &PSE, LoopInfo *LI,
1055 LoopVectorizationLegality *Legal,
1056 const TargetTransformInfo &TTI,
1057 const TargetLibraryInfo *TLI, DemandedBits *DB,
1058 AssumptionCache *AC,
1059 OptimizationRemarkEmitter *ORE, const Function *F,
1060 const LoopVectorizeHints *Hints,
1061 InterleavedAccessInfo &IAI)
1062 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
1063 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
1064 Hints(Hints), InterleaveInfo(IAI) {}
1065
1066 /// \return An upper bound for the vectorization factor, or None if
1067 /// vectorization and interleaving should be avoided up front.
1068 Optional<ElementCount> computeMaxVF(ElementCount UserVF, unsigned UserIC);
1069
1070 /// \return True if runtime checks are required for vectorization, and false
1071 /// otherwise.
1072 bool runtimeChecksRequired();
1073
1074 /// \return The most profitable vectorization factor and the cost of that VF.
1075 /// This method checks every power of two up to MaxVF. If UserVF is not ZERO
1076 /// then this vectorization factor will be selected if vectorization is
1077 /// possible.
1078 VectorizationFactor selectVectorizationFactor(ElementCount MaxVF);
1079
1080 /// Setup cost-based decisions for user vectorization factor.
1081 void selectUserVectorizationFactor(ElementCount UserVF) {
1082 collectUniformsAndScalars(UserVF);
1083 collectInstsToScalarize(UserVF);
1084 }
1085
1086 /// \return The size (in bits) of the smallest and widest types in the code
1087 /// that needs to be vectorized. We ignore values that remain scalar such as
1088 /// 64 bit loop indices.
1089 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
1090
1091 /// \return The desired interleave count.
1092 /// If interleave count has been specified by metadata it will be returned.
1093 /// Otherwise, the interleave count is computed and returned. VF and LoopCost
1094 /// are the selected vectorization factor and the cost of the selected VF.
1095 unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost);
1096
1097 /// Memory access instruction may be vectorized in more than one way.
1098 /// Form of instruction after vectorization depends on cost.
1099 /// This function takes cost-based decisions for Load/Store instructions
1100 /// and collects them in a map. This decisions map is used for building
1101 /// the lists of loop-uniform and loop-scalar instructions.
1102 /// The calculated cost is saved with widening decision in order to
1103 /// avoid redundant calculations.
1104 void setCostBasedWideningDecision(ElementCount VF);
1105
1106 /// A struct that represents some properties of the register usage
1107 /// of a loop.
1108 struct RegisterUsage {
1109 /// Holds the number of loop invariant values that are used in the loop.
1110 /// The key is ClassID of target-provided register class.
1111 SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
1112 /// Holds the maximum number of concurrent live intervals in the loop.
1113 /// The key is ClassID of target-provided register class.
1114 SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
1115 };
1116
1117 /// \return Returns information about the register usages of the loop for the
1118 /// given vectorization factors.
1119 SmallVector<RegisterUsage, 8>
1120 calculateRegisterUsage(ArrayRef<ElementCount> VFs);
1121
1122 /// Collect values we want to ignore in the cost model.
1123 void collectValuesToIgnore();
1124
1125 /// Split reductions into those that happen in the loop, and those that happen
1126 /// outside. In loop reductions are collected into InLoopReductionChains.
1127 void collectInLoopReductions();
1128
1129 /// \returns The smallest bitwidth each instruction can be represented with.
1130 /// The vector equivalents of these instructions should be truncated to this
1131 /// type.
1132 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
1133 return MinBWs;
1134 }
1135
1136 /// \returns True if it is more profitable to scalarize instruction \p I for
1137 /// vectorization factor \p VF.
1138 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
1139 assert(VF.isVector() &&((VF.isVector() && "Profitable to scalarize relevant only for VF > 1."
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Profitable to scalarize relevant only for VF > 1.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1140, __PRETTY_FUNCTION__))
1140 "Profitable to scalarize relevant only for VF > 1.")((VF.isVector() && "Profitable to scalarize relevant only for VF > 1."
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && \"Profitable to scalarize relevant only for VF > 1.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1140, __PRETTY_FUNCTION__))
;
1141
1142 // Cost model is not run in the VPlan-native path - return conservative
1143 // result until this changes.
1144 if (EnableVPlanNativePath)
1145 return false;
1146
1147 auto Scalars = InstsToScalarize.find(VF);
1148 assert(Scalars != InstsToScalarize.end() &&((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1149, __PRETTY_FUNCTION__))
1149 "VF not yet analyzed for scalarization profitability")((Scalars != InstsToScalarize.end() && "VF not yet analyzed for scalarization profitability"
) ? static_cast<void> (0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1149, __PRETTY_FUNCTION__))
;
1150 return Scalars->second.find(I) != Scalars->second.end();
1151 }
1152
1153 /// Returns true if \p I is known to be uniform after vectorization.
1154 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
1155 if (VF.isScalar())
1156 return true;
1157
1158 // Cost model is not run in the VPlan-native path - return conservative
1159 // result until this changes.
1160 if (EnableVPlanNativePath)
1161 return false;
1162
1163 auto UniformsPerVF = Uniforms.find(VF);
1164 assert(UniformsPerVF != Uniforms.end() &&((UniformsPerVF != Uniforms.end() && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1165, __PRETTY_FUNCTION__))
1165 "VF not yet analyzed for uniformity")((UniformsPerVF != Uniforms.end() && "VF not yet analyzed for uniformity"
) ? static_cast<void> (0) : __assert_fail ("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1165, __PRETTY_FUNCTION__))
;
1166 return UniformsPerVF->second.count(I);
1167 }
1168
1169 /// Returns true if \p I is known to be scalar after vectorization.
1170 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
1171 if (VF.isScalar())
1172 return true;
1173
1174 // Cost model is not run in the VPlan-native path - return conservative
1175 // result until this changes.
1176 if (EnableVPlanNativePath)
1177 return false;
1178
1179 auto ScalarsPerVF = Scalars.find(VF);
1180 assert(ScalarsPerVF != Scalars.end() &&((ScalarsPerVF != Scalars.end() && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1181, __PRETTY_FUNCTION__))
1181 "Scalar values are not calculated for VF")((ScalarsPerVF != Scalars.end() && "Scalar values are not calculated for VF"
) ? static_cast<void> (0) : __assert_fail ("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1181, __PRETTY_FUNCTION__))
;
1182 return ScalarsPerVF->second.count(I);
1183 }
1184
1185 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1186 /// for vectorization factor \p VF.
1187 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
1188 return VF.isVector() && MinBWs.find(I) != MinBWs.end() &&
1189 !isProfitableToScalarize(I, VF) &&
1190 !isScalarAfterVectorization(I, VF);
1191 }
1192
1193 /// Decision that was taken during cost calculation for memory instruction.
1194 enum InstWidening {
1195 CM_Unknown,
1196 CM_Widen, // For consecutive accesses with stride +1.
1197 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1198 CM_Interleave,
1199 CM_GatherScatter,
1200 CM_Scalarize
1201 };
1202
1203 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1204 /// instruction \p I and vector width \p VF.
1205 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
1206 unsigned Cost) {
1207 assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1207, __PRETTY_FUNCTION__))
;
1208 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1209 }
1210
1211 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1212 /// interleaving group \p Grp and vector width \p VF.
1213 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
1214 ElementCount VF, InstWidening W, unsigned Cost) {
1215 assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1215, __PRETTY_FUNCTION__))
;
1216 /// Broadcast this decicion to all instructions inside the group.
1217 /// But the cost will be assigned to one instruction only.
1218 for (unsigned i = 0; i < Grp->getFactor(); ++i) {
1219 if (auto *I = Grp->getMember(i)) {
1220 if (Grp->getInsertPos() == I)
1221 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1222 else
1223 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
1224 }
1225 }
1226 }
1227
1228 /// Return the cost model decision for the given instruction \p I and vector
1229 /// width \p VF. Return CM_Unknown if this instruction did not pass
1230 /// through the cost modeling.
1231 InstWidening getWideningDecision(Instruction *I, ElementCount VF) {
1232 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1232, __PRETTY_FUNCTION__))
;
1233 assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1233, __PRETTY_FUNCTION__))
;
1234
1235 // Cost model is not run in the VPlan-native path - return conservative
1236 // result until this changes.
1237 if (EnableVPlanNativePath)
1238 return CM_GatherScatter;
1239
1240 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1241 auto Itr = WideningDecisions.find(InstOnVF);
1242 if (Itr == WideningDecisions.end())
1243 return CM_Unknown;
1244 return Itr->second.first;
1245 }
1246
1247 /// Return the vectorization cost for the given instruction \p I and vector
1248 /// width \p VF.
1249 unsigned getWideningCost(Instruction *I, ElementCount VF) {
1250 assert(VF.isVector() && "Expected VF >=2")((VF.isVector() && "Expected VF >=2") ? static_cast
<void> (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1250, __PRETTY_FUNCTION__))
;
1251 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1252 assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() &&((WideningDecisions.find(InstOnVF) != WideningDecisions.end()
&& "The cost is not calculated") ? static_cast<void
> (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1253, __PRETTY_FUNCTION__))
1253 "The cost is not calculated")((WideningDecisions.find(InstOnVF) != WideningDecisions.end()
&& "The cost is not calculated") ? static_cast<void
> (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1253, __PRETTY_FUNCTION__))
;
1254 return WideningDecisions[InstOnVF].second;
1255 }
1256
1257 /// Return True if instruction \p I is an optimizable truncate whose operand
1258 /// is an induction variable. Such a truncate will be removed by adding a new
1259 /// induction variable with the destination type.
1260 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
1261 // If the instruction is not a truncate, return false.
1262 auto *Trunc = dyn_cast<TruncInst>(I);
1263 if (!Trunc)
1264 return false;
1265
1266 // Get the source and destination types of the truncate.
1267 Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);
1268 Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);
1269
1270 // If the truncate is free for the given types, return false. Replacing a
1271 // free truncate with an induction variable would add an induction variable
1272 // update instruction to each iteration of the loop. We exclude from this
1273 // check the primary induction variable since it will need an update
1274 // instruction regardless.
1275 Value *Op = Trunc->getOperand(0);
1276 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1277 return false;
1278
1279 // If the truncated value is not an induction variable, return false.
1280 return Legal->isInductionPhi(Op);
1281 }
1282
1283 /// Collects the instructions to scalarize for each predicated instruction in
1284 /// the loop.
1285 void collectInstsToScalarize(ElementCount VF);
1286
1287 /// Collect Uniform and Scalar values for the given \p VF.
1288 /// The sets depend on CM decision for Load/Store instructions
1289 /// that may be vectorized as interleave, gather-scatter or scalarized.
1290 void collectUniformsAndScalars(ElementCount VF) {
1291 // Do the analysis once.
1292 if (VF.isScalar() || Uniforms.find(VF) != Uniforms.end())
1293 return;
1294 setCostBasedWideningDecision(VF);
1295 collectLoopUniforms(VF);
1296 collectLoopScalars(VF);
1297 }
1298
1299 /// Returns true if the target machine supports masked store operation
1300 /// for the given \p DataType and kind of access to \p Ptr.
1301 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) {
1302 return Legal->isConsecutivePtr(Ptr) &&
1303 TTI.isLegalMaskedStore(DataType, Alignment);
1304 }
1305
1306 /// Returns true if the target machine supports masked load operation
1307 /// for the given \p DataType and kind of access to \p Ptr.
1308 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) {
1309 return Legal->isConsecutivePtr(Ptr) &&
1310 TTI.isLegalMaskedLoad(DataType, Alignment);
1311 }
1312
1313 /// Returns true if the target machine supports masked scatter operation
1314 /// for the given \p DataType.
1315 bool isLegalMaskedScatter(Type *DataType, Align Alignment) {
1316 return TTI.isLegalMaskedScatter(DataType, Alignment);
1317 }
1318
1319 /// Returns true if the target machine supports masked gather operation
1320 /// for the given \p DataType.
1321 bool isLegalMaskedGather(Type *DataType, Align Alignment) {
1322 return TTI.isLegalMaskedGather(DataType, Alignment);
1323 }
1324
1325 /// Returns true if the target machine can represent \p V as a masked gather
1326 /// or scatter operation.
1327 bool isLegalGatherOrScatter(Value *V) {
1328 bool LI = isa<LoadInst>(V);
1329 bool SI = isa<StoreInst>(V);
1330 if (!LI && !SI)
1331 return false;
1332 auto *Ty = getMemInstValueType(V);
1333 Align Align = getLoadStoreAlignment(V);
1334 return (LI && isLegalMaskedGather(Ty, Align)) ||
1335 (SI && isLegalMaskedScatter(Ty, Align));
1336 }
1337
1338 /// Returns true if \p I is an instruction that will be scalarized with
1339 /// predication. Such instructions include conditional stores and
1340 /// instructions that may divide by zero.
1341 /// If a non-zero VF has been calculated, we check if I will be scalarized
1342 /// predication for that VF.
1343 bool isScalarWithPredication(Instruction *I,
1344 ElementCount VF = ElementCount::getFixed(1));
1345
1346 // Returns true if \p I is an instruction that will be predicated either
1347 // through scalar predication or masked load/store or masked gather/scatter.
1348 // Superset of instructions that return true for isScalarWithPredication.
1349 bool isPredicatedInst(Instruction *I) {
1350 if (!blockNeedsPredication(I->getParent()))
1351 return false;
1352 // Loads and stores that need some form of masked operation are predicated
1353 // instructions.
1354 if (isa<LoadInst>(I) || isa<StoreInst>(I))
1355 return Legal->isMaskRequired(I);
1356 return isScalarWithPredication(I);
1357 }
1358
1359 /// Returns true if \p I is a memory instruction with consecutive memory
1360 /// access that can be widened.
1361 bool
1362 memoryInstructionCanBeWidened(Instruction *I,
1363 ElementCount VF = ElementCount::getFixed(1));
1364
1365 /// Returns true if \p I is a memory instruction in an interleaved-group
1366 /// of memory accesses that can be vectorized with wide vector loads/stores
1367 /// and shuffles.
1368 bool
1369 interleavedAccessCanBeWidened(Instruction *I,
1370 ElementCount VF = ElementCount::getFixed(1));
1371
1372 /// Check if \p Instr belongs to any interleaved access group.
1373 bool isAccessInterleaved(Instruction *Instr) {
1374 return InterleaveInfo.isInterleaved(Instr);
1375 }
1376
1377 /// Get the interleaved access group that \p Instr belongs to.
1378 const InterleaveGroup<Instruction> *
1379 getInterleavedAccessGroup(Instruction *Instr) {
1380 return InterleaveInfo.getInterleaveGroup(Instr);
1381 }
1382
1383 /// Returns true if an interleaved group requires a scalar iteration
1384 /// to handle accesses with gaps, and there is nothing preventing us from
1385 /// creating a scalar epilogue.
1386 bool requiresScalarEpilogue() const {
1387 return isScalarEpilogueAllowed() && InterleaveInfo.requiresScalarEpilogue();
1388 }
1389
1390 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1391 /// loop hint annotation.
1392 bool isScalarEpilogueAllowed() const {
1393 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1394 }
1395
1396 /// Returns true if all loop blocks should be masked to fold tail loop.
1397 bool foldTailByMasking() const { return FoldTailByMasking; }
1398
1399 bool blockNeedsPredication(BasicBlock *BB) {
1400 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1401 }
1402
1403 /// A SmallMapVector to store the InLoop reduction op chains, mapping phi
1404 /// nodes to the chain of instructions representing the reductions. Uses a
1405 /// MapVector to ensure deterministic iteration order.
1406 using ReductionChainMap =
1407 SmallMapVector<PHINode *, SmallVector<Instruction *, 4>, 4>;
1408
1409 /// Return the chain of instructions representing an inloop reduction.
1410 const ReductionChainMap &getInLoopReductionChains() const {
1411 return InLoopReductionChains;
1412 }
1413
1414 /// Returns true if the Phi is part of an inloop reduction.
1415 bool isInLoopReduction(PHINode *Phi) const {
1416 return InLoopReductionChains.count(Phi);
1417 }
1418
1419 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1420 /// with factor VF. Return the cost of the instruction, including
1421 /// scalarization overhead if it's needed.
1422 unsigned getVectorIntrinsicCost(CallInst *CI, ElementCount VF);
1423
1424 /// Estimate cost of a call instruction CI if it were vectorized with factor
1425 /// VF. Return the cost of the instruction, including scalarization overhead
1426 /// if it's needed. The flag NeedToScalarize shows if the call needs to be
1427 /// scalarized -
1428 /// i.e. either vector version isn't available, or is too expensive.
1429 unsigned getVectorCallCost(CallInst *CI, ElementCount VF,
1430 bool &NeedToScalarize);
1431
1432 /// Invalidates decisions already taken by the cost model.
1433 void invalidateCostModelingDecisions() {
1434 WideningDecisions.clear();
1435 Uniforms.clear();
1436 Scalars.clear();
1437 }
1438
1439private:
1440 unsigned NumPredStores = 0;
1441
1442 /// \return An upper bound for the vectorization factor, a power-of-2 larger
1443 /// than zero. One is returned if vectorization should best be avoided due
1444 /// to cost.
1445 ElementCount computeFeasibleMaxVF(unsigned ConstTripCount);
1446
1447 /// The vectorization cost is a combination of the cost itself and a boolean
1448 /// indicating whether any of the contributing operations will actually
1449 /// operate on
1450 /// vector values after type legalization in the backend. If this latter value
1451 /// is
1452 /// false, then all operations will be scalarized (i.e. no vectorization has
1453 /// actually taken place).
1454 using VectorizationCostTy = std::pair<unsigned, bool>;
1455
1456 /// Returns the expected execution cost. The unit of the cost does
1457 /// not matter because we use the 'cost' units to compare different
1458 /// vector widths. The cost that is returned is *not* normalized by
1459 /// the factor width.
1460 VectorizationCostTy expectedCost(ElementCount VF);
1461
1462 /// Returns the execution time cost of an instruction for a given vector
1463 /// width. Vector width of one means scalar.
1464 VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF);
1465
1466 /// The cost-computation logic from getInstructionCost which provides
1467 /// the vector type as an output parameter.
1468 unsigned getInstructionCost(Instruction *I, ElementCount VF, Type *&VectorTy);
1469
1470 /// Calculate vectorization cost of memory instruction \p I.
1471 unsigned getMemoryInstructionCost(Instruction *I, ElementCount VF);
1472
1473 /// The cost computation for scalarized memory instruction.
1474 unsigned getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1475
1476 /// The cost computation for interleaving group of memory instructions.
1477 unsigned getInterleaveGroupCost(Instruction *I, ElementCount VF);
1478
1479 /// The cost computation for Gather/Scatter instruction.
1480 unsigned getGatherScatterCost(Instruction *I, ElementCount VF);
1481
1482 /// The cost computation for widening instruction \p I with consecutive
1483 /// memory access.
1484 unsigned getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1485
1486 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1487 /// Load: scalar load + broadcast.
1488 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1489 /// element)
1490 unsigned getUniformMemOpCost(Instruction *I, ElementCount VF);
1491
1492 /// Estimate the overhead of scalarizing an instruction. This is a
1493 /// convenience wrapper for the type-based getScalarizationOverhead API.
1494 unsigned getScalarizationOverhead(Instruction *I, ElementCount VF);
1495
1496 /// Returns whether the instruction is a load or store and will be a emitted
1497 /// as a vector operation.
1498 bool isConsecutiveLoadOrStore(Instruction *I);
1499
1500 /// Returns true if an artificially high cost for emulated masked memrefs
1501 /// should be used.
1502 bool useEmulatedMaskMemRefHack(Instruction *I);
1503
1504 /// Map of scalar integer values to the smallest bitwidth they can be legally
1505 /// represented as. The vector equivalents of these values should be truncated
1506 /// to this type.
1507 MapVector<Instruction *, uint64_t> MinBWs;
1508
1509 /// A type representing the costs for instructions if they were to be
1510 /// scalarized rather than vectorized. The entries are Instruction-Cost
1511 /// pairs.
1512 using ScalarCostsTy = DenseMap<Instruction *, unsigned>;
1513
1514 /// A set containing all BasicBlocks that are known to present after
1515 /// vectorization as a predicated block.
1516 SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;
1517
1518 /// Records whether it is allowed to have the original scalar loop execute at
1519 /// least once. This may be needed as a fallback loop in case runtime
1520 /// aliasing/dependence checks fail, or to handle the tail/remainder
1521 /// iterations when the trip count is unknown or doesn't divide by the VF,
1522 /// or as a peel-loop to handle gaps in interleave-groups.
1523 /// Under optsize and when the trip count is very small we don't allow any
1524 /// iterations to execute in the scalar loop.
1525 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1526
1527 /// All blocks of loop are to be masked to fold tail of scalar iterations.
1528 bool FoldTailByMasking = false;
1529
1530 /// A map holding scalar costs for different vectorization factors. The
1531 /// presence of a cost for an instruction in the mapping indicates that the
1532 /// instruction will be scalarized when vectorizing with the associated
1533 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1534 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
1535
1536 /// Holds the instructions known to be uniform after vectorization.
1537 /// The data is collected per VF.
1538 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1539
1540 /// Holds the instructions known to be scalar after vectorization.
1541 /// The data is collected per VF.
1542 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1543
1544 /// Holds the instructions (address computations) that are forced to be
1545 /// scalarized.
1546 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1547
1548 /// PHINodes of the reductions that should be expanded in-loop along with
1549 /// their associated chains of reduction operations, in program order from top
1550 /// (PHI) to bottom
1551 ReductionChainMap InLoopReductionChains;
1552
1553 /// Returns the expected difference in cost from scalarizing the expression
1554 /// feeding a predicated instruction \p PredInst. The instructions to
1555 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1556 /// non-negative return value implies the expression will be scalarized.
1557 /// Currently, only single-use chains are considered for scalarization.
1558 int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,
1559 ElementCount VF);
1560
1561 /// Collect the instructions that are uniform after vectorization. An
1562 /// instruction is uniform if we represent it with a single scalar value in
1563 /// the vectorized loop corresponding to each vector iteration. Examples of
1564 /// uniform instructions include pointer operands of consecutive or
1565 /// interleaved memory accesses. Note that although uniformity implies an
1566 /// instruction will be scalar, the reverse is not true. In general, a
1567 /// scalarized instruction will be represented by VF scalar values in the
1568 /// vectorized loop, each corresponding to an iteration of the original
1569 /// scalar loop.
1570 void collectLoopUniforms(ElementCount VF);
1571
1572 /// Collect the instructions that are scalar after vectorization. An
1573 /// instruction is scalar if it is known to be uniform or will be scalarized
1574 /// during vectorization. Non-uniform scalarized instructions will be
1575 /// represented by VF values in the vectorized loop, each corresponding to an
1576 /// iteration of the original scalar loop.
1577 void collectLoopScalars(ElementCount VF);
1578
1579 /// Keeps cost model vectorization decision and cost for instructions.
1580 /// Right now it is used for memory instructions only.
1581 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1582 std::pair<InstWidening, unsigned>>;
1583
1584 DecisionList WideningDecisions;
1585
1586 /// Returns true if \p V is expected to be vectorized and it needs to be
1587 /// extracted.
1588 bool needsExtract(Value *V, ElementCount VF) const {
1589 Instruction *I = dyn_cast<Instruction>(V);
1590 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1591 TheLoop->isLoopInvariant(I))
1592 return false;
1593
1594 // Assume we can vectorize V (and hence we need extraction) if the
1595 // scalars are not computed yet. This can happen, because it is called
1596 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1597 // the scalars are collected. That should be a safe assumption in most
1598 // cases, because we check if the operands have vectorizable types
1599 // beforehand in LoopVectorizationLegality.
1600 return Scalars.find(VF) == Scalars.end() ||
1601 !isScalarAfterVectorization(I, VF);
1602 };
1603
1604 /// Returns a range containing only operands needing to be extracted.
1605 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1606 ElementCount VF) {
1607 return SmallVector<Value *, 4>(make_filter_range(
1608 Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
1609 }
1610
1611public:
1612 /// The loop that we evaluate.
1613 Loop *TheLoop;
1614
1615 /// Predicated scalar evolution analysis.
1616 PredicatedScalarEvolution &PSE;
1617
1618 /// Loop Info analysis.
1619 LoopInfo *LI;
1620
1621 /// Vectorization legality.
1622 LoopVectorizationLegality *Legal;
1623
1624 /// Vector target information.
1625 const TargetTransformInfo &TTI;
1626
1627 /// Target Library Info.
1628 const TargetLibraryInfo *TLI;
1629
1630 /// Demanded bits analysis.
1631 DemandedBits *DB;
1632
1633 /// Assumption cache.
1634 AssumptionCache *AC;
1635
1636 /// Interface to emit optimization remarks.
1637 OptimizationRemarkEmitter *ORE;
1638
1639 const Function *TheFunction;
1640
1641 /// Loop Vectorize Hint.
1642 const LoopVectorizeHints *Hints;
1643
1644 /// The interleave access information contains groups of interleaved accesses
1645 /// with the same stride and close to each other.
1646 InterleavedAccessInfo &InterleaveInfo;
1647
1648 /// Values to ignore in the cost model.
1649 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1650
1651 /// Values to ignore in the cost model when VF > 1.
1652 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1653};
1654
1655} // end namespace llvm
1656
1657// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1658// vectorization. The loop needs to be annotated with #pragma omp simd
1659// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1660// vector length information is not provided, vectorization is not considered
1661// explicit. Interleave hints are not allowed either. These limitations will be
1662// relaxed in the future.
1663// Please, note that we are currently forced to abuse the pragma 'clang
1664// vectorize' semantics. This pragma provides *auto-vectorization hints*
1665// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1666// provides *explicit vectorization hints* (LV can bypass legal checks and
1667// assume that vectorization is legal). However, both hints are implemented
1668// using the same metadata (llvm.loop.vectorize, processed by
1669// LoopVectorizeHints). This will be fixed in the future when the native IR
1670// representation for pragma 'omp simd' is introduced.
1671static bool isExplicitVecOuterLoop(Loop *OuterLp,
1672 OptimizationRemarkEmitter *ORE) {
1673 assert(!OuterLp->isInnermost() && "This is not an outer loop")((!OuterLp->isInnermost() && "This is not an outer loop"
) ? static_cast<void> (0) : __assert_fail ("!OuterLp->isInnermost() && \"This is not an outer loop\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1673, __PRETTY_FUNCTION__))
;
1674 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
1675
1676 // Only outer loops with an explicit vectorization hint are supported.
1677 // Unannotated outer loops are ignored.
1678 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1679 return false;
1680
1681 Function *Fn = OuterLp->getHeader()->getParent();
1682 if (!Hints.allowVectorization(Fn, OuterLp,
1683 true /*VectorizeOnlyWhenForced*/)) {
1684 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints prevent outer loop vectorization.\n"
; } } while (false)
;
1685 return false;
1686 }
1687
1688 if (Hints.getInterleave() > 1) {
1689 // TODO: Interleave support is future work.
1690 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n"; } } while (false)
1691 "outer loops.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n"; } } while (false)
;
1692 Hints.emitRemarkWithHints();
1693 return false;
1694 }
1695
1696 return true;
1697}
1698
1699static void collectSupportedLoops(Loop &L, LoopInfo *LI,
1700 OptimizationRemarkEmitter *ORE,
1701 SmallVectorImpl<Loop *> &V) {
1702 // Collect inner loops and outer loops without irreducible control flow. For
1703 // now, only collect outer loops that have explicit vectorization hints. If we
1704 // are stress testing the VPlan H-CFG construction, we collect the outermost
1705 // loop of every loop nest.
1706 if (L.isInnermost() || VPlanBuildStressTest ||
1707 (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
1708 LoopBlocksRPO RPOT(&L);
1709 RPOT.perform(LI);
1710 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
1711 V.push_back(&L);
1712 // TODO: Collect inner loops inside marked outer loops in case
1713 // vectorization fails for the outer loop. Do not invoke
1714 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1715 // already known to be reducible. We can use an inherited attribute for
1716 // that.
1717 return;
1718 }
1719 }
1720 for (Loop *InnerL : L)
1721 collectSupportedLoops(*InnerL, LI, ORE, V);
1722}
1723
1724namespace {
1725
1726/// The LoopVectorize Pass.
1727struct LoopVectorize : public FunctionPass {
1728 /// Pass identification, replacement for typeid
1729 static char ID;
1730
1731 LoopVectorizePass Impl;
1732
1733 explicit LoopVectorize(bool InterleaveOnlyWhenForced = false,
1734 bool VectorizeOnlyWhenForced = false)
1735 : FunctionPass(ID),
1736 Impl({InterleaveOnlyWhenForced, VectorizeOnlyWhenForced}) {
1737 initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
1738 }
1739
1740 bool runOnFunction(Function &F) override {
1741 if (skipFunction(F))
1742 return false;
1743
1744 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
1745 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1746 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1747 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1748 auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
1749 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
1750 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
1751 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
1752 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
1753 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
1754 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
1755 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
1756 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1757
1758 std::function<const LoopAccessInfo &(Loop &)> GetLAA =
1759 [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };
1760
1761 return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,
1762 GetLAA, *ORE, PSI).MadeAnyChange;
1763 }
1764
1765 void getAnalysisUsage(AnalysisUsage &AU) const override {
1766 AU.addRequired<AssumptionCacheTracker>();
1767 AU.addRequired<BlockFrequencyInfoWrapperPass>();
1768 AU.addRequired<DominatorTreeWrapperPass>();
1769 AU.addRequired<LoopInfoWrapperPass>();
1770 AU.addRequired<ScalarEvolutionWrapperPass>();
1771 AU.addRequired<TargetTransformInfoWrapperPass>();
1772 AU.addRequired<AAResultsWrapperPass>();
1773 AU.addRequired<LoopAccessLegacyAnalysis>();
1774 AU.addRequired<DemandedBitsWrapperPass>();
1775 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
1776 AU.addRequired<InjectTLIMappingsLegacy>();
1777
1778 // We currently do not preserve loopinfo/dominator analyses with outer loop
1779 // vectorization. Until this is addressed, mark these analyses as preserved
1780 // only for non-VPlan-native path.
1781 // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
1782 if (!EnableVPlanNativePath) {
1783 AU.addPreserved<LoopInfoWrapperPass>();
1784 AU.addPreserved<DominatorTreeWrapperPass>();
1785 }
1786
1787 AU.addPreserved<BasicAAWrapperPass>();
1788 AU.addPreserved<GlobalsAAWrapperPass>();
1789 AU.addRequired<ProfileSummaryInfoWrapperPass>();
1790 }
1791};
1792
1793} // end anonymous namespace
1794
1795//===----------------------------------------------------------------------===//
1796// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
1797// LoopVectorizationCostModel and LoopVectorizationPlanner.
1798//===----------------------------------------------------------------------===//
1799
1800Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
1801 // We need to place the broadcast of invariant variables outside the loop,
1802 // but only if it's proven safe to do so. Else, broadcast will be inside
1803 // vector loop body.
1804 Instruction *Instr = dyn_cast<Instruction>(V);
1805 bool SafeToHoist = OrigLoop->isLoopInvariant(V) &&
1806 (!Instr ||
1807 DT->dominates(Instr->getParent(), LoopVectorPreHeader));
1808 // Place the code for broadcasting invariant variables in the new preheader.
1809 IRBuilder<>::InsertPointGuard Guard(Builder);
1810 if (SafeToHoist)
1811 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
1812
1813 // Broadcast the scalar into all locations in the vector.
1814 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
1815
1816 return Shuf;
1817}
1818
1819void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
1820 const InductionDescriptor &II, Value *Step, Instruction *EntryVal) {
1821 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1822, __PRETTY_FUNCTION__))
1822 "Expected either an induction phi-node or a truncate of it!")(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1822, __PRETTY_FUNCTION__))
;
1823 Value *Start = II.getStartValue();
1824
1825 // Construct the initial value of the vector IV in the vector loop preheader
1826 auto CurrIP = Builder.saveIP();
1827 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
1828 if (isa<TruncInst>(EntryVal)) {
1829 assert(Start->getType()->isIntegerTy() &&((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1830, __PRETTY_FUNCTION__))
1830 "Truncation requires an integer type")((Start->getType()->isIntegerTy() && "Truncation requires an integer type"
) ? static_cast<void> (0) : __assert_fail ("Start->getType()->isIntegerTy() && \"Truncation requires an integer type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1830, __PRETTY_FUNCTION__))
;
1831 auto *TruncType = cast<IntegerType>(EntryVal->getType());
1832 Step = Builder.CreateTrunc(Step, TruncType);
1833 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
1834 }
1835 Value *SplatStart = Builder.CreateVectorSplat(VF, Start);
1836 Value *SteppedStart =
1837 getStepVector(SplatStart, 0, Step, II.getInductionOpcode());
1838
1839 // We create vector phi nodes for both integer and floating-point induction
1840 // variables. Here, we determine the kind of arithmetic we will perform.
1841 Instruction::BinaryOps AddOp;
1842 Instruction::BinaryOps MulOp;
1843 if (Step->getType()->isIntegerTy()) {
1844 AddOp = Instruction::Add;
1845 MulOp = Instruction::Mul;
1846 } else {
1847 AddOp = II.getInductionOpcode();
1848 MulOp = Instruction::FMul;
1849 }
1850
1851 // Multiply the vectorization factor by the step using integer or
1852 // floating-point arithmetic as appropriate.
1853 Value *ConstVF =
1854 getSignedIntOrFpConstant(Step->getType(), VF.getKnownMinValue());
1855 Value *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, Step, ConstVF));
1856
1857 // Create a vector splat to use in the induction update.
1858 //
1859 // FIXME: If the step is non-constant, we create the vector splat with
1860 // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
1861 // handle a constant vector splat.
1862 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1862, __PRETTY_FUNCTION__))
;
1863 Value *SplatVF = isa<Constant>(Mul)
1864 ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
1865 : Builder.CreateVectorSplat(VF, Mul);
1866 Builder.restoreIP(CurrIP);
1867
1868 // We may need to add the step a number of times, depending on the unroll
1869 // factor. The last of those goes into the PHI.
1870 PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",
1871 &*LoopVectorBody->getFirstInsertionPt());
1872 VecInd->setDebugLoc(EntryVal->getDebugLoc());
1873 Instruction *LastInduction = VecInd;
1874 for (unsigned Part = 0; Part < UF; ++Part) {
1875 VectorLoopValueMap.setVectorValue(EntryVal, Part, LastInduction);
1876
1877 if (isa<TruncInst>(EntryVal))
1878 addMetadata(LastInduction, EntryVal);
1879 recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, Part);
1880
1881 LastInduction = cast<Instruction>(addFastMathFlag(
1882 Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add")));
1883 LastInduction->setDebugLoc(EntryVal->getDebugLoc());
1884 }
1885
1886 // Move the last step to the end of the latch block. This ensures consistent
1887 // placement of all induction updates.
1888 auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
1889 auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());
1890 auto *ICmp = cast<Instruction>(Br->getCondition());
1891 LastInduction->moveBefore(ICmp);
1892 LastInduction->setName("vec.ind.next");
1893
1894 VecInd->addIncoming(SteppedStart, LoopVectorPreHeader);
1895 VecInd->addIncoming(LastInduction, LoopVectorLatch);
1896}
1897
1898bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const {
1899 return Cost->isScalarAfterVectorization(I, VF) ||
1900 Cost->isProfitableToScalarize(I, VF);
1901}
1902
1903bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {
1904 if (shouldScalarizeInstruction(IV))
1905 return true;
1906 auto isScalarInst = [&](User *U) -> bool {
1907 auto *I = cast<Instruction>(U);
1908 return (OrigLoop->contains(I) && shouldScalarizeInstruction(I));
1909 };
1910 return llvm::any_of(IV->users(), isScalarInst);
1911}
1912
1913void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
1914 const InductionDescriptor &ID, const Instruction *EntryVal,
1915 Value *VectorLoopVal, unsigned Part, unsigned Lane) {
1916 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1917, __PRETTY_FUNCTION__))
1917 "Expected either an induction phi-node or a truncate of it!")(((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal
)) && "Expected either an induction phi-node or a truncate of it!"
) ? static_cast<void> (0) : __assert_fail ("(isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) && \"Expected either an induction phi-node or a truncate of it!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1917, __PRETTY_FUNCTION__))
;
1918
1919 // This induction variable is not the phi from the original loop but the
1920 // newly-created IV based on the proof that casted Phi is equal to the
1921 // uncasted Phi in the vectorized loop (under a runtime guard possibly). It
1922 // re-uses the same InductionDescriptor that original IV uses but we don't
1923 // have to do any recording in this case - that is done when original IV is
1924 // processed.
1925 if (isa<TruncInst>(EntryVal))
1926 return;
1927
1928 const SmallVectorImpl<Instruction *> &Casts = ID.getCastInsts();
1929 if (Casts.empty())
1930 return;
1931 // Only the first Cast instruction in the Casts vector is of interest.
1932 // The rest of the Casts (if exist) have no uses outside the
1933 // induction update chain itself.
1934 Instruction *CastInst = *Casts.begin();
1935 if (Lane < UINT_MAX(2147483647 *2U +1U))
1936 VectorLoopValueMap.setScalarValue(CastInst, {Part, Lane}, VectorLoopVal);
1937 else
1938 VectorLoopValueMap.setVectorValue(CastInst, Part, VectorLoopVal);
1939}
1940
1941void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, TruncInst *Trunc) {
1942 assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1943, __PRETTY_FUNCTION__))
1943 "Primary induction variable must have an integer type")(((IV->getType()->isIntegerTy() || IV != OldInduction) &&
"Primary induction variable must have an integer type") ? static_cast
<void> (0) : __assert_fail ("(IV->getType()->isIntegerTy() || IV != OldInduction) && \"Primary induction variable must have an integer type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1943, __PRETTY_FUNCTION__))
;
1944
1945 auto II = Legal->getInductionVars().find(IV);
1946 assert(II != Legal->getInductionVars().end() && "IV is not an induction")((II != Legal->getInductionVars().end() && "IV is not an induction"
) ? static_cast<void> (0) : __assert_fail ("II != Legal->getInductionVars().end() && \"IV is not an induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1946, __PRETTY_FUNCTION__))
;
1947
1948 auto ID = II->second;
1949 assert(IV->getType() == ID.getStartValue()->getType() && "Types must match")((IV->getType() == ID.getStartValue()->getType() &&
"Types must match") ? static_cast<void> (0) : __assert_fail
("IV->getType() == ID.getStartValue()->getType() && \"Types must match\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1949, __PRETTY_FUNCTION__))
;
1950
1951 // The value from the original loop to which we are mapping the new induction
1952 // variable.
1953 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
1954
1955 auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
1956
1957 // Generate code for the induction step. Note that induction steps are
1958 // required to be loop-invariant
1959 auto CreateStepValue = [&](const SCEV *Step) -> Value * {
1960 assert(PSE.getSE()->isLoopInvariant(Step, OrigLoop) &&((PSE.getSE()->isLoopInvariant(Step, OrigLoop) && "Induction step should be loop invariant"
) ? static_cast<void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(Step, OrigLoop) && \"Induction step should be loop invariant\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1961, __PRETTY_FUNCTION__))
1961 "Induction step should be loop invariant")((PSE.getSE()->isLoopInvariant(Step, OrigLoop) && "Induction step should be loop invariant"
) ? static_cast<void> (0) : __assert_fail ("PSE.getSE()->isLoopInvariant(Step, OrigLoop) && \"Induction step should be loop invariant\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1961, __PRETTY_FUNCTION__))
;
1962 if (PSE.getSE()->isSCEVable(IV->getType())) {
1963 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
1964 return Exp.expandCodeFor(Step, Step->getType(),
1965 LoopVectorPreHeader->getTerminator());
1966 }
1967 return cast<SCEVUnknown>(Step)->getValue();
1968 };
1969
1970 // The scalar value to broadcast. This is derived from the canonical
1971 // induction variable. If a truncation type is given, truncate the canonical
1972 // induction variable and step. Otherwise, derive these values from the
1973 // induction descriptor.
1974 auto CreateScalarIV = [&](Value *&Step) -> Value * {
1975 Value *ScalarIV = Induction;
1976 if (IV != OldInduction) {
1977 ScalarIV = IV->getType()->isIntegerTy()
1978 ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
1979 : Builder.CreateCast(Instruction::SIToFP, Induction,
1980 IV->getType());
1981 ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID);
1982 ScalarIV->setName("offset.idx");
1983 }
1984 if (Trunc) {
1985 auto *TruncType = cast<IntegerType>(Trunc->getType());
1986 assert(Step->getType()->isIntegerTy() &&((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1987, __PRETTY_FUNCTION__))
1987 "Truncation requires an integer step")((Step->getType()->isIntegerTy() && "Truncation requires an integer step"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isIntegerTy() && \"Truncation requires an integer step\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1987, __PRETTY_FUNCTION__))
;
1988 ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);
1989 Step = Builder.CreateTrunc(Step, TruncType);
1990 }
1991 return ScalarIV;
1992 };
1993
1994 // Create the vector values from the scalar IV, in the absence of creating a
1995 // vector IV.
1996 auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
1997 Value *Broadcasted = getBroadcastInstrs(ScalarIV);
1998 for (unsigned Part = 0; Part < UF; ++Part) {
1999 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 1999, __PRETTY_FUNCTION__))
;
2000 Value *EntryPart =
2001 getStepVector(Broadcasted, VF.getKnownMinValue() * Part, Step,
2002 ID.getInductionOpcode());
2003 VectorLoopValueMap.setVectorValue(EntryVal, Part, EntryPart);
2004 if (Trunc)
2005 addMetadata(EntryPart, Trunc);
2006 recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, Part);
2007 }
2008 };
2009
2010 // Now do the actual transformations, and start with creating the step value.
2011 Value *Step = CreateStepValue(ID.getStep());
2012 if (VF.isZero() || VF.isScalar()) {
2013 Value *ScalarIV = CreateScalarIV(Step);
2014 CreateSplatIV(ScalarIV, Step);
2015 return;
2016 }
2017
2018 // Determine if we want a scalar version of the induction variable. This is
2019 // true if the induction variable itself is not widened, or if it has at
2020 // least one user in the loop that is not widened.
2021 auto NeedsScalarIV = needsScalarInduction(EntryVal);
2022 if (!NeedsScalarIV) {
2023 createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
2024 return;
2025 }
2026
2027 // Try to create a new independent vector induction variable. If we can't
2028 // create the phi node, we will splat the scalar induction variable in each
2029 // loop iteration.
2030 if (!shouldScalarizeInstruction(EntryVal)) {
2031 createVectorIntOrFpInductionPHI(ID, Step, EntryVal);
2032 Value *ScalarIV = CreateScalarIV(Step);
2033 // Create scalar steps that can be used by instructions we will later
2034 // scalarize. Note that the addition of the scalar steps will not increase
2035 // the number of instructions in the loop in the common case prior to
2036 // InstCombine. We will be trading one vector extract for each scalar step.
2037 buildScalarSteps(ScalarIV, Step, EntryVal, ID);
2038 return;
2039 }
2040
2041 // All IV users are scalar instructions, so only emit a scalar IV, not a
2042 // vectorised IV. Except when we tail-fold, then the splat IV feeds the
2043 // predicate used by the masked loads/stores.
2044 Value *ScalarIV = CreateScalarIV(Step);
2045 if (!Cost->isScalarEpilogueAllowed())
2046 CreateSplatIV(ScalarIV, Step);
2047 buildScalarSteps(ScalarIV, Step, EntryVal, ID);
2048}
2049
2050Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
2051 Instruction::BinaryOps BinOp) {
2052 // Create and check the types.
2053 auto *ValVTy = cast<FixedVectorType>(Val->getType());
2054 int VLen = ValVTy->getNumElements();
2055
2056 Type *STy = Val->getType()->getScalarType();
2057 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
"Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2058, __PRETTY_FUNCTION__))
2058 "Induction Step must be an integer or FP")(((STy->isIntegerTy() || STy->isFloatingPointTy()) &&
"Induction Step must be an integer or FP") ? static_cast<
void> (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2058, __PRETTY_FUNCTION__))
;
2059 assert(Step->getType() == STy && "Step has wrong type")((Step->getType() == STy && "Step has wrong type")
? static_cast<void> (0) : __assert_fail ("Step->getType() == STy && \"Step has wrong type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2059, __PRETTY_FUNCTION__))
;
2060
2061 SmallVector<Constant *, 8> Indices;
2062
2063 if (STy->isIntegerTy()) {
2064 // Create a vector of consecutive numbers from zero to VF.
2065 for (int i = 0; i < VLen; ++i)
2066 Indices.push_back(ConstantInt::get(STy, StartIdx + i));
2067
2068 // Add the consecutive indices to the vector value.
2069 Constant *Cv = ConstantVector::get(Indices);
2070 assert(Cv->getType() == Val->getType() && "Invalid consecutive vec")((Cv->getType() == Val->getType() && "Invalid consecutive vec"
) ? static_cast<void> (0) : __assert_fail ("Cv->getType() == Val->getType() && \"Invalid consecutive vec\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2070, __PRETTY_FUNCTION__))
;
2071 Step = Builder.CreateVectorSplat(VLen, Step);
2072 assert(Step->getType() == Val->getType() && "Invalid step vec")((Step->getType() == Val->getType() && "Invalid step vec"
) ? static_cast<void> (0) : __assert_fail ("Step->getType() == Val->getType() && \"Invalid step vec\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2072, __PRETTY_FUNCTION__))
;
2073 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
2074 // which can be found from the original scalar operations.
2075 Step = Builder.CreateMul(Cv, Step);
2076 return Builder.CreateAdd(Val, Step, "induction");
2077 }
2078
2079 // Floating point induction.
2080 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
"Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2081, __PRETTY_FUNCTION__))
2081 "Binary Opcode should be specified for FP induction")(((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
"Binary Opcode should be specified for FP induction") ? static_cast
<void> (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2081, __PRETTY_FUNCTION__))
;
2082 // Create a vector of consecutive numbers from zero to VF.
2083 for (int i = 0; i < VLen; ++i)
2084 Indices.push_back(ConstantFP::get(STy, (double)(StartIdx + i)));
2085
2086 // Add the consecutive indices to the vector value.
2087 Constant *Cv = ConstantVector::get(Indices);
2088
2089 Step = Builder.CreateVectorSplat(VLen, Step);
2090
2091 // Floating point operations had to be 'fast' to enable the induction.
2092 FastMathFlags Flags;
2093 Flags.setFast();
2094
2095 Value *MulOp = Builder.CreateFMul(Cv, Step);
2096 if (isa<Instruction>(MulOp))
2097 // Have to check, MulOp may be a constant
2098 cast<Instruction>(MulOp)->setFastMathFlags(Flags);
2099
2100 Value *BOp = Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
2101 if (isa<Instruction>(BOp))
2102 cast<Instruction>(BOp)->setFastMathFlags(Flags);
2103 return BOp;
2104}
2105
2106void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
2107 Instruction *EntryVal,
2108 const InductionDescriptor &ID) {
2109 // We shouldn't have to build scalar steps if we aren't vectorizing.
2110 assert(VF.isVector() && "VF should be greater than one")((VF.isVector() && "VF should be greater than one") ?
static_cast<void> (0) : __assert_fail ("VF.isVector() && \"VF should be greater than one\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2110, __PRETTY_FUNCTION__))
;
2111 assert(!VF.isScalable() &&((!VF.isScalable() && "the code below assumes a fixed number of elements at compile time"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"the code below assumes a fixed number of elements at compile time\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2112, __PRETTY_FUNCTION__))
2112 "the code below assumes a fixed number of elements at compile time")((!VF.isScalable() && "the code below assumes a fixed number of elements at compile time"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"the code below assumes a fixed number of elements at compile time\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2112, __PRETTY_FUNCTION__))
;
2113 // Get the value type and ensure it and the step have the same integer type.
2114 Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
2115 assert(ScalarIVTy == Step->getType() &&((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2116, __PRETTY_FUNCTION__))
2116 "Val and Step should have the same type")((ScalarIVTy == Step->getType() && "Val and Step should have the same type"
) ? static_cast<void> (0) : __assert_fail ("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2116, __PRETTY_FUNCTION__))
;
2117
2118 // We build scalar steps for both integer and floating-point induction
2119 // variables. Here, we determine the kind of arithmetic we will perform.
2120 Instruction::BinaryOps AddOp;
2121 Instruction::BinaryOps MulOp;
2122 if (ScalarIVTy->isIntegerTy()) {
2123 AddOp = Instruction::Add;
2124 MulOp = Instruction::Mul;
2125 } else {
2126 AddOp = ID.getInductionOpcode();
2127 MulOp = Instruction::FMul;
2128 }
2129
2130 // Determine the number of scalars we need to generate for each unroll
2131 // iteration. If EntryVal is uniform, we only need to generate the first
2132 // lane. Otherwise, we generate all VF values.
2133 unsigned Lanes =
2134 Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF)
2135 ? 1
2136 : VF.getKnownMinValue();
2137 // Compute the scalar steps and save the results in VectorLoopValueMap.
2138 for (unsigned Part = 0; Part < UF; ++Part) {
2139 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2140 auto *StartIdx = getSignedIntOrFpConstant(
2141 ScalarIVTy, VF.getKnownMinValue() * Part + Lane);
2142 auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step));
2143 auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul));
2144 VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add);
2145 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, Part, Lane);
2146 }
2147 }
2148}
2149
2150Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) {
2151 assert(V != Induction && "The new induction variable should not be used.")((V != Induction && "The new induction variable should not be used."
) ? static_cast<void> (0) : __assert_fail ("V != Induction && \"The new induction variable should not be used.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2151, __PRETTY_FUNCTION__))
;
2152 assert(!V->getType()->isVectorTy() && "Can't widen a vector")((!V->getType()->isVectorTy() && "Can't widen a vector"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVectorTy() && \"Can't widen a vector\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2152, __PRETTY_FUNCTION__))
;
2153 assert(!V->getType()->isVoidTy() && "Type does not produce a value")((!V->getType()->isVoidTy() && "Type does not produce a value"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVoidTy() && \"Type does not produce a value\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2153, __PRETTY_FUNCTION__))
;
2154
2155 // If we have a stride that is replaced by one, do it here. Defer this for
2156 // the VPlan-native path until we start running Legal checks in that path.
2157 if (!EnableVPlanNativePath && Legal->hasStride(V))
2158 V = ConstantInt::get(V->getType(), 1);
2159
2160 // If we have a vector mapped to this value, return it.
2161 if (VectorLoopValueMap.hasVectorValue(V, Part))
2162 return VectorLoopValueMap.getVectorValue(V, Part);
2163
2164 // If the value has not been vectorized, check if it has been scalarized
2165 // instead. If it has been scalarized, and we actually need the value in
2166 // vector form, we will construct the vector values on demand.
2167 if (VectorLoopValueMap.hasAnyScalarValue(V)) {
2168 Value *ScalarValue = VectorLoopValueMap.getScalarValue(V, {Part, 0});
2169
2170 // If we've scalarized a value, that value should be an instruction.
2171 auto *I = cast<Instruction>(V);
2172
2173 // If we aren't vectorizing, we can just copy the scalar map values over to
2174 // the vector map.
2175 if (VF.isScalar()) {
2176 VectorLoopValueMap.setVectorValue(V, Part, ScalarValue);
2177 return ScalarValue;
2178 }
2179
2180 // Get the last scalar instruction we generated for V and Part. If the value
2181 // is known to be uniform after vectorization, this corresponds to lane zero
2182 // of the Part unroll iteration. Otherwise, the last instruction is the one
2183 // we created for the last vector lane of the Part unroll iteration.
2184 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2184, __PRETTY_FUNCTION__))
;
2185 unsigned LastLane = Cost->isUniformAfterVectorization(I, VF)
2186 ? 0
2187 : VF.getKnownMinValue() - 1;
2188 auto *LastInst = cast<Instruction>(
2189 VectorLoopValueMap.getScalarValue(V, {Part, LastLane}));
2190
2191 // Set the insert point after the last scalarized instruction. This ensures
2192 // the insertelement sequence will directly follow the scalar definitions.
2193 auto OldIP = Builder.saveIP();
2194 auto NewIP = std::next(BasicBlock::iterator(LastInst));
2195 Builder.SetInsertPoint(&*NewIP);
2196
2197 // However, if we are vectorizing, we need to construct the vector values.
2198 // If the value is known to be uniform after vectorization, we can just
2199 // broadcast the scalar value corresponding to lane zero for each unroll
2200 // iteration. Otherwise, we construct the vector values using insertelement
2201 // instructions. Since the resulting vectors are stored in
2202 // VectorLoopValueMap, we will only generate the insertelements once.
2203 Value *VectorValue = nullptr;
2204 if (Cost->isUniformAfterVectorization(I, VF)) {
2205 VectorValue = getBroadcastInstrs(ScalarValue);
2206 VectorLoopValueMap.setVectorValue(V, Part, VectorValue);
2207 } else {
2208 // Initialize packing with insertelements to start from undef.
2209 assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2209, __PRETTY_FUNCTION__))
;
2210 Value *Undef = UndefValue::get(VectorType::get(V->getType(), VF));
2211 VectorLoopValueMap.setVectorValue(V, Part, Undef);
2212 for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
2213 packScalarIntoVectorValue(V, {Part, Lane});
2214 VectorValue = VectorLoopValueMap.getVectorValue(V, Part);
2215 }
2216 Builder.restoreIP(OldIP);
2217 return VectorValue;
2218 }
2219
2220 // If this scalar is unknown, assume that it is a constant or that it is
2221 // loop invariant. Broadcast V and save the value for future uses.
2222 Value *B = getBroadcastInstrs(V);
2223 VectorLoopValueMap.setVectorValue(V, Part, B);
2224 return B;
2225}
2226
2227Value *
2228InnerLoopVectorizer::getOrCreateScalarValue(Value *V,
2229 const VPIteration &Instance) {
2230 // If the value is not an instruction contained in the loop, it should
2231 // already be scalar.
2232 if (OrigLoop->isLoopInvariant(V))
2233 return V;
2234
2235 assert(Instance.Lane > 0((Instance.Lane > 0 ? !Cost->isUniformAfterVectorization
(cast<Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Instance.Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2237, __PRETTY_FUNCTION__))
2236 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF)((Instance.Lane > 0 ? !Cost->isUniformAfterVectorization
(cast<Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Instance.Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2237, __PRETTY_FUNCTION__))
2237 : true && "Uniform values only have lane zero")((Instance.Lane > 0 ? !Cost->isUniformAfterVectorization
(cast<Instruction>(V), VF) : true && "Uniform values only have lane zero"
) ? static_cast<void> (0) : __assert_fail ("Instance.Lane > 0 ? !Cost->isUniformAfterVectorization(cast<Instruction>(V), VF) : true && \"Uniform values only have lane zero\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2237, __PRETTY_FUNCTION__))
;
2238
2239 // If the value from the original loop has not been vectorized, it is
2240 // represented by UF x VF scalar values in the new loop. Return the requested
2241 // scalar value.
2242 if (VectorLoopValueMap.hasScalarValue(V, Instance))
2243 return VectorLoopValueMap.getScalarValue(V, Instance);
2244
2245 // If the value has not been scalarized, get its entry in VectorLoopValueMap
2246 // for the given unroll part. If this entry is not a vector type (i.e., the
2247 // vectorization factor is one), there is no need to generate an
2248 // extractelement instruction.
2249 auto *U = getOrCreateVectorValue(V, Instance.Part);
2250 if (!U->getType()->isVectorTy()) {
2251 assert(VF.isScalar() && "Value not scalarized has non-vector type")((VF.isScalar() && "Value not scalarized has non-vector type"
) ? static_cast<void> (0) : __assert_fail ("VF.isScalar() && \"Value not scalarized has non-vector type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2251, __PRETTY_FUNCTION__))
;
2252 return U;
2253 }
2254
2255 // Otherwise, the value from the original loop has been vectorized and is
2256 // represented by UF vector values. Extract and return the requested scalar
2257 // value from the appropriate vector lane.
2258 return Builder.CreateExtractElement(U, Builder.getInt32(Instance.Lane));
2259}
2260
2261void InnerLoopVectorizer::packScalarIntoVectorValue(
2262 Value *V, const VPIteration &Instance) {
2263 assert(V != Induction && "The new induction variable should not be used.")((V != Induction && "The new induction variable should not be used."
) ? static_cast<void> (0) : __assert_fail ("V != Induction && \"The new induction variable should not be used.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2263, __PRETTY_FUNCTION__))
;
2264 assert(!V->getType()->isVectorTy() && "Can't pack a vector")((!V->getType()->isVectorTy() && "Can't pack a vector"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVectorTy() && \"Can't pack a vector\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2264, __PRETTY_FUNCTION__))
;
2265 assert(!V->getType()->isVoidTy() && "Type does not produce a value")((!V->getType()->isVoidTy() && "Type does not produce a value"
) ? static_cast<void> (0) : __assert_fail ("!V->getType()->isVoidTy() && \"Type does not produce a value\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2265, __PRETTY_FUNCTION__))
;
2266
2267 Value *ScalarInst = VectorLoopValueMap.getScalarValue(V, Instance);
2268 Value *VectorValue = VectorLoopValueMap.getVectorValue(V, Instance.Part);
2269 VectorValue = Builder.CreateInsertElement(VectorValue, ScalarInst,
2270 Builder.getInt32(Instance.Lane));
2271 VectorLoopValueMap.resetVectorValue(V, Instance.Part, VectorValue);
2272}
2273
2274Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
2275 assert(Vec->getType()->isVectorTy() && "Invalid type")((Vec->getType()->isVectorTy() && "Invalid type"
) ? static_cast<void> (0) : __assert_fail ("Vec->getType()->isVectorTy() && \"Invalid type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2275, __PRETTY_FUNCTION__))
;
2276 assert(!VF.isScalable() && "Cannot reverse scalable vectors")((!VF.isScalable() && "Cannot reverse scalable vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"Cannot reverse scalable vectors\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2276, __PRETTY_FUNCTION__))
;
2277 SmallVector<int, 8> ShuffleMask;
2278 for (unsigned i = 0; i < VF.getKnownMinValue(); ++i)
2279 ShuffleMask.push_back(VF.getKnownMinValue() - i - 1);
2280
2281 return Builder.CreateShuffleVector(Vec, ShuffleMask, "reverse");
2282}
2283
2284// Return whether we allow using masked interleave-groups (for dealing with
2285// strided loads/stores that reside in predicated blocks, or for dealing
2286// with gaps).
2287static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
2288 // If an override option has been passed in for interleaved accesses, use it.
2289 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2290 return EnableMaskedInterleavedMemAccesses;
2291
2292 return TTI.enableMaskedInterleavedAccessVectorization();
2293}
2294
2295// Try to vectorize the interleave group that \p Instr belongs to.
2296//
2297// E.g. Translate following interleaved load group (factor = 3):
2298// for (i = 0; i < N; i+=3) {
2299// R = Pic[i]; // Member of index 0
2300// G = Pic[i+1]; // Member of index 1
2301// B = Pic[i+2]; // Member of index 2
2302// ... // do something to R, G, B
2303// }
2304// To:
2305// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
2306// %R.vec = shuffle %wide.vec, undef, <0, 3, 6, 9> ; R elements
2307// %G.vec = shuffle %wide.vec, undef, <1, 4, 7, 10> ; G elements
2308// %B.vec = shuffle %wide.vec, undef, <2, 5, 8, 11> ; B elements
2309//
2310// Or translate following interleaved store group (factor = 3):
2311// for (i = 0; i < N; i+=3) {
2312// ... do something to R, G, B
2313// Pic[i] = R; // Member of index 0
2314// Pic[i+1] = G; // Member of index 1
2315// Pic[i+2] = B; // Member of index 2
2316// }
2317// To:
2318// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
2319// %B_U.vec = shuffle %B.vec, undef, <0, 1, 2, 3, u, u, u, u>
2320// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
2321// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
2322// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
2323void InnerLoopVectorizer::vectorizeInterleaveGroup(
2324 const InterleaveGroup<Instruction> *Group, VPTransformState &State,
2325 VPValue *Addr, VPValue *BlockInMask) {
2326 Instruction *Instr = Group->getInsertPos();
2327 const DataLayout &DL = Instr->getModule()->getDataLayout();
2328
2329 // Prepare for the vector type of the interleaved load/store.
2330 Type *ScalarTy = getMemInstValueType(Instr);
2331 unsigned InterleaveFactor = Group->getFactor();
2332 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2332, __PRETTY_FUNCTION__))
;
2333 auto *VecTy = VectorType::get(ScalarTy, VF * InterleaveFactor);
2334
2335 // Prepare for the new pointers.
2336 SmallVector<Value *, 2> AddrParts;
2337 unsigned Index = Group->getIndex(Instr);
2338
2339 // TODO: extend the masked interleaved-group support to reversed access.
2340 assert((!BlockInMask || !Group->isReverse()) &&(((!BlockInMask || !Group->isReverse()) && "Reversed masked interleave-group not supported."
) ? static_cast<void> (0) : __assert_fail ("(!BlockInMask || !Group->isReverse()) && \"Reversed masked interleave-group not supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2341, __PRETTY_FUNCTION__))
2341 "Reversed masked interleave-group not supported.")(((!BlockInMask || !Group->isReverse()) && "Reversed masked interleave-group not supported."
) ? static_cast<void> (0) : __assert_fail ("(!BlockInMask || !Group->isReverse()) && \"Reversed masked interleave-group not supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2341, __PRETTY_FUNCTION__))
;
2342
2343 // If the group is reverse, adjust the index to refer to the last vector lane
2344 // instead of the first. We adjust the index from the first vector lane,
2345 // rather than directly getting the pointer for lane VF - 1, because the
2346 // pointer operand of the interleaved access is supposed to be uniform. For
2347 // uniform instructions, we're only required to generate a value for the
2348 // first vector lane in each unroll iteration.
2349 assert(!VF.isScalable() &&((!VF.isScalable() && "scalable vector reverse operation is not implemented"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vector reverse operation is not implemented\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2350, __PRETTY_FUNCTION__))
2350 "scalable vector reverse operation is not implemented")((!VF.isScalable() && "scalable vector reverse operation is not implemented"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vector reverse operation is not implemented\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2350, __PRETTY_FUNCTION__))
;
2351 if (Group->isReverse())
2352 Index += (VF.getKnownMinValue() - 1) * Group->getFactor();
2353
2354 for (unsigned Part = 0; Part < UF; Part++) {
2355 Value *AddrPart = State.get(Addr, {Part, 0});
2356 setDebugLocFromInst(Builder, AddrPart);
2357
2358 // Notice current instruction could be any index. Need to adjust the address
2359 // to the member of index 0.
2360 //
2361 // E.g. a = A[i+1]; // Member of index 1 (Current instruction)
2362 // b = A[i]; // Member of index 0
2363 // Current pointer is pointed to A[i+1], adjust it to A[i].
2364 //
2365 // E.g. A[i+1] = a; // Member of index 1
2366 // A[i] = b; // Member of index 0
2367 // A[i+2] = c; // Member of index 2 (Current instruction)
2368 // Current pointer is pointed to A[i+2], adjust it to A[i].
2369
2370 bool InBounds = false;
2371 if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
2372 InBounds = gep->isInBounds();
2373 AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Builder.getInt32(-Index));
2374 cast<GetElementPtrInst>(AddrPart)->setIsInBounds(InBounds);
2375
2376 // Cast to the vector pointer type.
2377 unsigned AddressSpace = AddrPart->getType()->getPointerAddressSpace();
2378 Type *PtrTy = VecTy->getPointerTo(AddressSpace);
2379 AddrParts.push_back(Builder.CreateBitCast(AddrPart, PtrTy));
2380 }
2381
2382 setDebugLocFromInst(Builder, Instr);
2383 Value *UndefVec = UndefValue::get(VecTy);
2384
2385 Value *MaskForGaps = nullptr;
2386 if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) {
2387 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2387, __PRETTY_FUNCTION__))
;
2388 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
2389 assert(MaskForGaps && "Mask for Gaps is required but it is null")((MaskForGaps && "Mask for Gaps is required but it is null"
) ? static_cast<void> (0) : __assert_fail ("MaskForGaps && \"Mask for Gaps is required but it is null\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2389, __PRETTY_FUNCTION__))
;
2390 }
2391
2392 // Vectorize the interleaved load group.
2393 if (isa<LoadInst>(Instr)) {
2394 // For each unroll part, create a wide load for the group.
2395 SmallVector<Value *, 2> NewLoads;
2396 for (unsigned Part = 0; Part < UF; Part++) {
2397 Instruction *NewLoad;
2398 if (BlockInMask || MaskForGaps) {
2399 assert(useMaskedInterleavedAccesses(*TTI) &&((useMaskedInterleavedAccesses(*TTI) && "masked interleaved groups are not allowed."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2400, __PRETTY_FUNCTION__))
2400 "masked interleaved groups are not allowed.")((useMaskedInterleavedAccesses(*TTI) && "masked interleaved groups are not allowed."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2400, __PRETTY_FUNCTION__))
;
2401 Value *GroupMask = MaskForGaps;
2402 if (BlockInMask) {
2403 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2404 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2404, __PRETTY_FUNCTION__))
;
2405 Value *ShuffledMask = Builder.CreateShuffleVector(
2406 BlockInMaskPart,
2407 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2408 "interleaved.mask");
2409 GroupMask = MaskForGaps
2410 ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
2411 MaskForGaps)
2412 : ShuffledMask;
2413 }
2414 NewLoad =
2415 Builder.CreateMaskedLoad(AddrParts[Part], Group->getAlign(),
2416 GroupMask, UndefVec, "wide.masked.vec");
2417 }
2418 else
2419 NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part],
2420 Group->getAlign(), "wide.vec");
2421 Group->addMetadata(NewLoad);
2422 NewLoads.push_back(NewLoad);
2423 }
2424
2425 // For each member in the group, shuffle out the appropriate data from the
2426 // wide loads.
2427 for (unsigned I = 0; I < InterleaveFactor; ++I) {
2428 Instruction *Member = Group->getMember(I);
2429
2430 // Skip the gaps in the group.
2431 if (!Member)
2432 continue;
2433
2434 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2434, __PRETTY_FUNCTION__))
;
2435 auto StrideMask =
2436 createStrideMask(I, InterleaveFactor, VF.getKnownMinValue());
2437 for (unsigned Part = 0; Part < UF; Part++) {
2438 Value *StridedVec = Builder.CreateShuffleVector(
2439 NewLoads[Part], StrideMask, "strided.vec");
2440
2441 // If this member has different type, cast the result type.
2442 if (Member->getType() != ScalarTy) {
2443 assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2443, __PRETTY_FUNCTION__))
;
2444 VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
2445 StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
2446 }
2447
2448 if (Group->isReverse())
2449 StridedVec = reverseVector(StridedVec);
2450
2451 VectorLoopValueMap.setVectorValue(Member, Part, StridedVec);
2452 }
2453 }
2454 return;
2455 }
2456
2457 // The sub vector type for current instruction.
2458 assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2458, __PRETTY_FUNCTION__))
;
2459 auto *SubVT = VectorType::get(ScalarTy, VF);
2460
2461 // Vectorize the interleaved store group.
2462 for (unsigned Part = 0; Part < UF; Part++) {
2463 // Collect the stored vector from each member.
2464 SmallVector<Value *, 4> StoredVecs;
2465 for (unsigned i = 0; i < InterleaveFactor; i++) {
2466 // Interleaved store group doesn't allow a gap, so each index has a member
2467 Instruction *Member = Group->getMember(i);
2468 assert(Member && "Fail to get a member from an interleaved store group")((Member && "Fail to get a member from an interleaved store group"
) ? static_cast<void> (0) : __assert_fail ("Member && \"Fail to get a member from an interleaved store group\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2468, __PRETTY_FUNCTION__))
;
2469
2470 Value *StoredVec = getOrCreateVectorValue(
2471 cast<StoreInst>(Member)->getValueOperand(), Part);
2472 if (Group->isReverse())
2473 StoredVec = reverseVector(StoredVec);
2474
2475 // If this member has different type, cast it to a unified type.
2476
2477 if (StoredVec->getType() != SubVT)
2478 StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);
2479
2480 StoredVecs.push_back(StoredVec);
2481 }
2482
2483 // Concatenate all vectors into a wide vector.
2484 Value *WideVec = concatenateVectors(Builder, StoredVecs);
2485
2486 // Interleave the elements in the wide vector.
2487 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2487, __PRETTY_FUNCTION__))
;
2488 Value *IVec = Builder.CreateShuffleVector(
2489 WideVec, createInterleaveMask(VF.getKnownMinValue(), InterleaveFactor),
2490 "interleaved.vec");
2491
2492 Instruction *NewStoreInstr;
2493 if (BlockInMask) {
2494 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2495 Value *ShuffledMask = Builder.CreateShuffleVector(
2496 BlockInMaskPart,
2497 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2498 "interleaved.mask");
2499 NewStoreInstr = Builder.CreateMaskedStore(
2500 IVec, AddrParts[Part], Group->getAlign(), ShuffledMask);
2501 }
2502 else
2503 NewStoreInstr =
2504 Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign());
2505
2506 Group->addMetadata(NewStoreInstr);
2507 }
2508}
2509
2510void InnerLoopVectorizer::vectorizeMemoryInstruction(
2511 Instruction *Instr, VPTransformState &State, VPValue *Def, VPValue *Addr,
2512 VPValue *StoredValue, VPValue *BlockInMask) {
2513 // Attempt to issue a wide load.
2514 LoadInst *LI = dyn_cast<LoadInst>(Instr);
2515 StoreInst *SI = dyn_cast<StoreInst>(Instr);
2516
2517 assert((LI || SI) && "Invalid Load/Store instruction")(((LI || SI) && "Invalid Load/Store instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid Load/Store instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2517, __PRETTY_FUNCTION__))
;
2518 assert((!SI || StoredValue) && "No stored value provided for widened store")(((!SI || StoredValue) && "No stored value provided for widened store"
) ? static_cast<void> (0) : __assert_fail ("(!SI || StoredValue) && \"No stored value provided for widened store\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2518, __PRETTY_FUNCTION__))
;
2519 assert((!LI || !StoredValue) && "Stored value provided for widened load")(((!LI || !StoredValue) && "Stored value provided for widened load"
) ? static_cast<void> (0) : __assert_fail ("(!LI || !StoredValue) && \"Stored value provided for widened load\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2519, __PRETTY_FUNCTION__))
;
2520
2521 LoopVectorizationCostModel::InstWidening Decision =
2522 Cost->getWideningDecision(Instr, VF);
2523 assert((Decision == LoopVectorizationCostModel::CM_Widen ||(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
== LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2526, __PRETTY_FUNCTION__))
2524 Decision == LoopVectorizationCostModel::CM_Widen_Reverse ||(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
== LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2526, __PRETTY_FUNCTION__))
2525 Decision == LoopVectorizationCostModel::CM_GatherScatter) &&(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
== LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2526, __PRETTY_FUNCTION__))
2526 "CM decision is not to widen the memory instruction")(((Decision == LoopVectorizationCostModel::CM_Widen || Decision
== LoopVectorizationCostModel::CM_Widen_Reverse || Decision ==
LoopVectorizationCostModel::CM_GatherScatter) && "CM decision is not to widen the memory instruction"
) ? static_cast<void> (0) : __assert_fail ("(Decision == LoopVectorizationCostModel::CM_Widen || Decision == LoopVectorizationCostModel::CM_Widen_Reverse || Decision == LoopVectorizationCostModel::CM_GatherScatter) && \"CM decision is not to widen the memory instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2526, __PRETTY_FUNCTION__))
;
2527
2528 Type *ScalarDataTy = getMemInstValueType(Instr);
2529
2530 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2530, __PRETTY_FUNCTION__))
;
2531 auto *DataTy = VectorType::get(ScalarDataTy, VF);
2532 const Align Alignment = getLoadStoreAlignment(Instr);
2533
2534 // Determine if the pointer operand of the access is either consecutive or
2535 // reverse consecutive.
2536 bool Reverse = (Decision == LoopVectorizationCostModel::CM_Widen_Reverse);
2537 bool ConsecutiveStride =
2538 Reverse || (Decision == LoopVectorizationCostModel::CM_Widen);
2539 bool CreateGatherScatter =
2540 (Decision == LoopVectorizationCostModel::CM_GatherScatter);
2541
2542 // Either Ptr feeds a vector load/store, or a vector GEP should feed a vector
2543 // gather/scatter. Otherwise Decision should have been to Scalarize.
2544 assert((ConsecutiveStride || CreateGatherScatter) &&(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2545, __PRETTY_FUNCTION__))
2545 "The instruction should be scalarized")(((ConsecutiveStride || CreateGatherScatter) && "The instruction should be scalarized"
) ? static_cast<void> (0) : __assert_fail ("(ConsecutiveStride || CreateGatherScatter) && \"The instruction should be scalarized\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2545, __PRETTY_FUNCTION__))
;
2546 (void)ConsecutiveStride;
2547
2548 VectorParts BlockInMaskParts(UF);
2549 bool isMaskRequired = BlockInMask;
2550 if (isMaskRequired)
2551 for (unsigned Part = 0; Part < UF; ++Part)
2552 BlockInMaskParts[Part] = State.get(BlockInMask, Part);
2553
2554 const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
2555 // Calculate the pointer for the specific unroll-part.
2556 GetElementPtrInst *PartPtr = nullptr;
2557
2558 bool InBounds = false;
2559 if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts()))
2560 InBounds = gep->isInBounds();
2561
2562 if (Reverse) {
2563 // If the address is consecutive but reversed, then the
2564 // wide store needs to start at the last vector element.
2565 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
2566 ScalarDataTy, Ptr, Builder.getInt32(-Part * VF.getKnownMinValue())));
2567 PartPtr->setIsInBounds(InBounds);
2568 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
2569 ScalarDataTy, PartPtr, Builder.getInt32(1 - VF.getKnownMinValue())));
2570 PartPtr->setIsInBounds(InBounds);
2571 if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
2572 BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
2573 } else {
2574 PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
2575 ScalarDataTy, Ptr, Builder.getInt32(Part * VF.getKnownMinValue())));
2576 PartPtr->setIsInBounds(InBounds);
2577 }
2578
2579 unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace();
2580 return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));
2581 };
2582
2583 // Handle Stores:
2584 if (SI) {
2585 setDebugLocFromInst(Builder, SI);
2586
2587 for (unsigned Part = 0; Part < UF; ++Part) {
2588 Instruction *NewSI = nullptr;
2589 Value *StoredVal = State.get(StoredValue, Part);
2590 if (CreateGatherScatter) {
2591 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
2592 Value *VectorGep = State.get(Addr, Part);
2593 NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
2594 MaskPart);
2595 } else {
2596 if (Reverse) {
2597 // If we store to reverse consecutive memory locations, then we need
2598 // to reverse the order of elements in the stored value.
2599 StoredVal = reverseVector(StoredVal);
2600 // We don't want to update the value in the map as it might be used in
2601 // another expression. So don't call resetVectorValue(StoredVal).
2602 }
2603 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, {0, 0}));
2604 if (isMaskRequired)
2605 NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
2606 BlockInMaskParts[Part]);
2607 else
2608 NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
2609 }
2610 addMetadata(NewSI, SI);
2611 }
2612 return;
2613 }
2614
2615 // Handle loads.
2616 assert(LI && "Must have a load instruction")((LI && "Must have a load instruction") ? static_cast
<void> (0) : __assert_fail ("LI && \"Must have a load instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2616, __PRETTY_FUNCTION__))
;
2617 setDebugLocFromInst(Builder, LI);
2618 for (unsigned Part = 0; Part < UF; ++Part) {
2619 Value *NewLI;
2620 if (CreateGatherScatter) {
2621 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
2622 Value *VectorGep = State.get(Addr, Part);
2623 NewLI = Builder.CreateMaskedGather(VectorGep, Alignment, MaskPart,
2624 nullptr, "wide.masked.gather");
2625 addMetadata(NewLI, LI);
2626 } else {
2627 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, {0, 0}));
2628 if (isMaskRequired)
2629 NewLI = Builder.CreateMaskedLoad(
2630 VecPtr, Alignment, BlockInMaskParts[Part], UndefValue::get(DataTy),
2631 "wide.masked.load");
2632 else
2633 NewLI =
2634 Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
2635
2636 // Add metadata to the load, but setVectorValue to the reverse shuffle.
2637 addMetadata(NewLI, LI);
2638 if (Reverse)
2639 NewLI = reverseVector(NewLI);
2640 }
2641
2642 State.set(Def, Instr, NewLI, Part);
2643 }
2644}
2645
2646void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPUser &User,
2647 const VPIteration &Instance,
2648 bool IfPredicateInstr,
2649 VPTransformState &State) {
2650 assert(!Instr->getType()->isAggregateType() && "Can't handle vectors")((!Instr->getType()->isAggregateType() && "Can't handle vectors"
) ? static_cast<void> (0) : __assert_fail ("!Instr->getType()->isAggregateType() && \"Can't handle vectors\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2650, __PRETTY_FUNCTION__))
;
2651
2652 setDebugLocFromInst(Builder, Instr);
2653
2654 // Does this instruction return a value ?
2655 bool IsVoidRetTy = Instr->getType()->isVoidTy();
2656
2657 Instruction *Cloned = Instr->clone();
2658 if (!IsVoidRetTy)
2659 Cloned->setName(Instr->getName() + ".cloned");
2660
2661 // Replace the operands of the cloned instructions with their scalar
2662 // equivalents in the new loop.
2663 for (unsigned op = 0, e = User.getNumOperands(); op != e; ++op) {
2664 auto *Operand = dyn_cast<Instruction>(Instr->getOperand(op));
2665 auto InputInstance = Instance;
2666 if (!Operand || !OrigLoop->contains(Operand) ||
2667 (Cost->isUniformAfterVectorization(Operand, State.VF)))
2668 InputInstance.Lane = 0;
2669 auto *NewOp = State.get(User.getOperand(op), InputInstance);
2670 Cloned->setOperand(op, NewOp);
2671 }
2672 addNewMetadata(Cloned, Instr);
2673
2674 // Place the cloned scalar in the new loop.
2675 Builder.Insert(Cloned);
2676
2677 // Add the cloned scalar to the scalar map entry.
2678 VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned);
2679
2680 // If we just cloned a new assumption, add it the assumption cache.
2681 if (auto *II = dyn_cast<IntrinsicInst>(Cloned))
2682 if (II->getIntrinsicID() == Intrinsic::assume)
2683 AC->registerAssumption(II);
2684
2685 // End if-block.
2686 if (IfPredicateInstr)
2687 PredicatedInstructions.push_back(Cloned);
2688}
2689
2690PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,
2691 Value *End, Value *Step,
2692 Instruction *DL) {
2693 BasicBlock *Header = L->getHeader();
2694 BasicBlock *Latch = L->getLoopLatch();
2695 // As we're just creating this loop, it's possible no latch exists
2696 // yet. If so, use the header as this will be a single block loop.
2697 if (!Latch)
2698 Latch = Header;
2699
2700 IRBuilder<> Builder(&*Header->getFirstInsertionPt());
2701 Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);
2702 setDebugLocFromInst(Builder, OldInst);
2703 auto *Induction = Builder.CreatePHI(Start->getType(), 2, "index");
2704
2705 Builder.SetInsertPoint(Latch->getTerminator());
2706 setDebugLocFromInst(Builder, OldInst);
2707
2708 // Create i+1 and fill the PHINode.
2709 Value *Next = Builder.CreateAdd(Induction, Step, "index.next");
2710 Induction->addIncoming(Start, L->getLoopPreheader());
2711 Induction->addIncoming(Next, Latch);
2712 // Create the compare.
2713 Value *ICmp = Builder.CreateICmpEQ(Next, End);
2714 Builder.CreateCondBr(ICmp, L->getExitBlock(), Header);
2715
2716 // Now we have two terminators. Remove the old one from the block.
2717 Latch->getTerminator()->eraseFromParent();
2718
2719 return Induction;
2720}
2721
2722Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
2723 if (TripCount)
2724 return TripCount;
2725
2726 assert(L && "Create Trip Count for null loop.")((L && "Create Trip Count for null loop.") ? static_cast
<void> (0) : __assert_fail ("L && \"Create Trip Count for null loop.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2726, __PRETTY_FUNCTION__))
;
2727 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
2728 // Find the loop boundaries.
2729 ScalarEvolution *SE = PSE.getSE();
2730 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
2731 assert(BackedgeTakenCount != SE->getCouldNotCompute() &&((BackedgeTakenCount != SE->getCouldNotCompute() &&
"Invalid loop count") ? static_cast<void> (0) : __assert_fail
("BackedgeTakenCount != SE->getCouldNotCompute() && \"Invalid loop count\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2732, __PRETTY_FUNCTION__))
2732 "Invalid loop count")((BackedgeTakenCount != SE->getCouldNotCompute() &&
"Invalid loop count") ? static_cast<void> (0) : __assert_fail
("BackedgeTakenCount != SE->getCouldNotCompute() && \"Invalid loop count\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2732, __PRETTY_FUNCTION__))
;
2733
2734 Type *IdxTy = Legal->getWidestInductionType();
2735 assert(IdxTy && "No type for induction")((IdxTy && "No type for induction") ? static_cast<
void> (0) : __assert_fail ("IdxTy && \"No type for induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2735, __PRETTY_FUNCTION__))
;
2736
2737 // The exit count might have the type of i64 while the phi is i32. This can
2738 // happen if we have an induction variable that is sign extended before the
2739 // compare. The only way that we get a backedge taken count is that the
2740 // induction variable was signed and as such will not overflow. In such a case
2741 // truncation is legal.
2742 if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) >
2743 IdxTy->getPrimitiveSizeInBits())
2744 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
2745 BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
2746
2747 // Get the total trip count from the count by adding 1.
2748 const SCEV *ExitCount = SE->getAddExpr(
2749 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
2750
2751 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
2752
2753 // Expand the trip count and place the new instructions in the preheader.
2754 // Notice that the pre-header does not change, only the loop body.
2755 SCEVExpander Exp(*SE, DL, "induction");
2756
2757 // Count holds the overall loop count (N).
2758 TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
2759 L->getLoopPreheader()->getTerminator());
2760
2761 if (TripCount->getType()->isPointerTy())
2762 TripCount =
2763 CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
2764 L->getLoopPreheader()->getTerminator());
2765
2766 return TripCount;
2767}
2768
2769Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
2770 if (VectorTripCount)
2771 return VectorTripCount;
2772
2773 Value *TC = getOrCreateTripCount(L);
2774 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
2775
2776 Type *Ty = TC->getType();
2777 // This is where we can make the step a runtime constant.
2778 assert(!VF.isScalable() && "scalable vectorization is not supported yet")((!VF.isScalable() && "scalable vectorization is not supported yet"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectorization is not supported yet\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2778, __PRETTY_FUNCTION__))
;
2779 Constant *Step = ConstantInt::get(Ty, VF.getKnownMinValue() * UF);
2780
2781 // If the tail is to be folded by masking, round the number of iterations N
2782 // up to a multiple of Step instead of rounding down. This is done by first
2783 // adding Step-1 and then rounding down. Note that it's ok if this addition
2784 // overflows: the vector induction variable will eventually wrap to zero given
2785 // that it starts at zero and its Step is a power of two; the loop will then
2786 // exit, with the last early-exit vector comparison also producing all-true.
2787 if (Cost->foldTailByMasking()) {
2788 assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&((isPowerOf2_32(VF.getKnownMinValue() * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue() * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2789, __PRETTY_FUNCTION__))
2789 "VF*UF must be a power of 2 when folding tail by masking")((isPowerOf2_32(VF.getKnownMinValue() * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue() * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2789, __PRETTY_FUNCTION__))
;
2790 TC = Builder.CreateAdd(
2791 TC, ConstantInt::get(Ty, VF.getKnownMinValue() * UF - 1), "n.rnd.up");
2792 }
2793
2794 // Now we need to generate the expression for the part of the loop that the
2795 // vectorized body will execute. This is equal to N - (N % Step) if scalar
2796 // iterations are not required for correctness, or N - Step, otherwise. Step
2797 // is equal to the vectorization factor (number of SIMD elements) times the
2798 // unroll factor (number of SIMD instructions).
2799 Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
2800
2801 // If there is a non-reversed interleaved group that may speculatively access
2802 // memory out-of-bounds, we need to ensure that there will be at least one
2803 // iteration of the scalar epilogue loop. Thus, if the step evenly divides
2804 // the trip count, we set the remainder to be equal to the step. If the step
2805 // does not evenly divide the trip count, no adjustment is necessary since
2806 // there will already be scalar iterations. Note that the minimum iterations
2807 // check ensures that N >= Step.
2808 if (VF.isVector() && Cost->requiresScalarEpilogue()) {
2809 auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
2810 R = Builder.CreateSelect(IsZero, Step, R);
2811 }
2812
2813 VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
2814
2815 return VectorTripCount;
2816}
2817
2818Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
2819 const DataLayout &DL) {
2820 // Verify that V is a vector type with same number of elements as DstVTy.
2821 auto *DstFVTy = cast<FixedVectorType>(DstVTy);
2822 unsigned VF = DstFVTy->getNumElements();
2823 auto *SrcVecTy = cast<FixedVectorType>(V->getType());
2824 assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match")(((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match"
) ? static_cast<void> (0) : __assert_fail ("(VF == SrcVecTy->getNumElements()) && \"Vector dimensions do not match\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2824, __PRETTY_FUNCTION__))
;
2825 Type *SrcElemTy = SrcVecTy->getElementType();
2826 Type *DstElemTy = DstFVTy->getElementType();
2827 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&(((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy
)) && "Vector elements must have same size") ? static_cast
<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2828, __PRETTY_FUNCTION__))
2828 "Vector elements must have same size")(((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy
)) && "Vector elements must have same size") ? static_cast
<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2828, __PRETTY_FUNCTION__))
;
2829
2830 // Do a direct cast if element types are castable.
2831 if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
2832 return Builder.CreateBitOrPointerCast(V, DstFVTy);
2833 }
2834 // V cannot be directly casted to desired vector type.
2835 // May happen when V is a floating point vector but DstVTy is a vector of
2836 // pointers or vice-versa. Handle this using a two-step bitcast using an
2837 // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
2838 assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&(((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()
) && "Only one type should be a pointer type") ? static_cast
<void> (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2839, __PRETTY_FUNCTION__))
2839 "Only one type should be a pointer type")(((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()
) && "Only one type should be a pointer type") ? static_cast
<void> (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2839, __PRETTY_FUNCTION__))
;
2840 assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&(((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy
()) && "Only one type should be a floating point type"
) ? static_cast<void> (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2841, __PRETTY_FUNCTION__))
2841 "Only one type should be a floating point type")(((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy
()) && "Only one type should be a floating point type"
) ? static_cast<void> (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2841, __PRETTY_FUNCTION__))
;
2842 Type *IntTy =
2843 IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
2844 auto *VecIntTy = FixedVectorType::get(IntTy, VF);
2845 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
2846 return Builder.CreateBitOrPointerCast(CastVal, DstFVTy);
2847}
2848
2849void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
2850 BasicBlock *Bypass) {
2851 Value *Count = getOrCreateTripCount(L);
2852 // Reuse existing vector loop preheader for TC checks.
2853 // Note that new preheader block is generated for vector loop.
2854 BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
2855 IRBuilder<> Builder(TCCheckBlock->getTerminator());
2856
2857 // Generate code to check if the loop's trip count is less than VF * UF, or
2858 // equal to it in case a scalar epilogue is required; this implies that the
2859 // vector trip count is zero. This check also covers the case where adding one
2860 // to the backedge-taken count overflowed leading to an incorrect trip count
2861 // of zero. In this case we will also jump to the scalar loop.
2862 auto P = Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE
2863 : ICmpInst::ICMP_ULT;
2864
2865 // If tail is to be folded, vector loop takes care of all iterations.
2866 Value *CheckMinIters = Builder.getFalse();
2867 if (!Cost->foldTailByMasking()) {
2868 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2868, __PRETTY_FUNCTION__))
;
2869 CheckMinIters = Builder.CreateICmp(
2870 P, Count,
2871 ConstantInt::get(Count->getType(), VF.getKnownMinValue() * UF),
2872 "min.iters.check");
2873 }
2874 // Create new preheader for vector loop.
2875 LoopVectorPreHeader =
2876 SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), DT, LI, nullptr,
2877 "vector.ph");
2878
2879 assert(DT->properlyDominates(DT->getNode(TCCheckBlock),((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2881, __PRETTY_FUNCTION__))
2880 DT->getNode(Bypass)->getIDom()) &&((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2881, __PRETTY_FUNCTION__))
2881 "TC check is expected to dominate Bypass")((DT->properlyDominates(DT->getNode(TCCheckBlock), DT->
getNode(Bypass)->getIDom()) && "TC check is expected to dominate Bypass"
) ? static_cast<void> (0) : __assert_fail ("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2881, __PRETTY_FUNCTION__))
;
2882
2883 // Update dominator for Bypass & LoopExit.
2884 DT->changeImmediateDominator(Bypass, TCCheckBlock);
2885 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);
2886
2887 ReplaceInstWithInst(
2888 TCCheckBlock->getTerminator(),
2889 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
2890 LoopBypassBlocks.push_back(TCCheckBlock);
2891}
2892
2893void InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {
2894 // Reuse existing vector loop preheader for SCEV checks.
2895 // Note that new preheader block is generated for vector loop.
2896 BasicBlock *const SCEVCheckBlock = LoopVectorPreHeader;
2897
2898 // Generate the code to check that the SCEV assumptions that we made.
2899 // We want the new basic block to start at the first instruction in a
2900 // sequence of instructions that form a check.
2901 SCEVExpander Exp(*PSE.getSE(), Bypass->getModule()->getDataLayout(),
2902 "scev.check");
2903 Value *SCEVCheck = Exp.expandCodeForPredicate(
2904 &PSE.getUnionPredicate(), SCEVCheckBlock->getTerminator());
2905
2906 if (auto *C = dyn_cast<ConstantInt>(SCEVCheck))
2907 if (C->isZero())
2908 return;
2909
2910 assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
&& Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2913, __PRETTY_FUNCTION__))
2911 (OptForSizeBasedOnProfile &&((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
&& Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2913, __PRETTY_FUNCTION__))
2912 Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
&& Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2913, __PRETTY_FUNCTION__))
2913 "Cannot SCEV check stride or overflow when optimizing for size")((!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile
&& Cost->Hints->getForce() != LoopVectorizeHints
::FK_Enabled)) && "Cannot SCEV check stride or overflow when optimizing for size"
) ? static_cast<void> (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2913, __PRETTY_FUNCTION__))
;
2914
2915 SCEVCheckBlock->setName("vector.scevcheck");
2916 // Create new preheader for vector loop.
2917 LoopVectorPreHeader =
2918 SplitBlock(SCEVCheckBlock, SCEVCheckBlock->getTerminator(), DT, LI,
2919 nullptr, "vector.ph");
2920
2921 // Update dominator only if this is first RT check.
2922 if (LoopBypassBlocks.empty()) {
2923 DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
2924 DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
2925 }
2926
2927 ReplaceInstWithInst(
2928 SCEVCheckBlock->getTerminator(),
2929 BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheck));
2930 LoopBypassBlocks.push_back(SCEVCheckBlock);
2931 AddedSafetyChecks = true;
2932}
2933
2934void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) {
2935 // VPlan-native path does not do any analysis for runtime checks currently.
2936 if (EnableVPlanNativePath)
2937 return;
2938
2939 // Reuse existing vector loop preheader for runtime memory checks.
2940 // Note that new preheader block is generated for vector loop.
2941 BasicBlock *const MemCheckBlock = L->getLoopPreheader();
2942
2943 // Generate the code that checks in runtime if arrays overlap. We put the
2944 // checks into a separate block to make the more common case of few elements
2945 // faster.
2946 auto *LAI = Legal->getLAI();
2947 const auto &RtPtrChecking = *LAI->getRuntimePointerChecking();
2948 if (!RtPtrChecking.Need)
2949 return;
2950
2951 if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
2952 assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&((Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled
&& "Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.") ? static_cast<void> (0) : __assert_fail
("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2954, __PRETTY_FUNCTION__))
2953 "Cannot emit memory checks when optimizing for size, unless forced "((Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled
&& "Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.") ? static_cast<void> (0) : __assert_fail
("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2954, __PRETTY_FUNCTION__))
2954 "to vectorize.")((Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled
&& "Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.") ? static_cast<void> (0) : __assert_fail
("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2954, __PRETTY_FUNCTION__))
;
2955 ORE->emit([&]() {
2956 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationCodeSize",
2957 L->getStartLoc(), L->getHeader())
2958 << "Code-size may be reduced by not forcing "
2959 "vectorization, or by source-code modifications "
2960 "eliminating the need for runtime checks "
2961 "(e.g., adding 'restrict').";
2962 });
2963 }
2964
2965 MemCheckBlock->setName("vector.memcheck");
2966 // Create new preheader for vector loop.
2967 LoopVectorPreHeader =
2968 SplitBlock(MemCheckBlock, MemCheckBlock->getTerminator(), DT, LI, nullptr,
2969 "vector.ph");
2970
2971 auto *CondBranch = cast<BranchInst>(
2972 Builder.CreateCondBr(Builder.getTrue(), Bypass, LoopVectorPreHeader));
2973 ReplaceInstWithInst(MemCheckBlock->getTerminator(), CondBranch);
2974 LoopBypassBlocks.push_back(MemCheckBlock);
2975 AddedSafetyChecks = true;
2976
2977 // Update dominator only if this is first RT check.
2978 if (LoopBypassBlocks.empty()) {
2979 DT->changeImmediateDominator(Bypass, MemCheckBlock);
2980 DT->changeImmediateDominator(LoopExitBlock, MemCheckBlock);
2981 }
2982
2983 Instruction *FirstCheckInst;
2984 Instruction *MemRuntimeCheck;
2985 std::tie(FirstCheckInst, MemRuntimeCheck) =
2986 addRuntimeChecks(MemCheckBlock->getTerminator(), OrigLoop,
2987 RtPtrChecking.getChecks(), RtPtrChecking.getSE());
2988 assert(MemRuntimeCheck && "no RT checks generated although RtPtrChecking "((MemRuntimeCheck && "no RT checks generated although RtPtrChecking "
"claimed checks are required") ? static_cast<void> (0)
: __assert_fail ("MemRuntimeCheck && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2989, __PRETTY_FUNCTION__))
2989 "claimed checks are required")((MemRuntimeCheck && "no RT checks generated although RtPtrChecking "
"claimed checks are required") ? static_cast<void> (0)
: __assert_fail ("MemRuntimeCheck && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2989, __PRETTY_FUNCTION__))
;
2990 CondBranch->setCondition(MemRuntimeCheck);
2991
2992 // We currently don't use LoopVersioning for the actual loop cloning but we
2993 // still use it to add the noalias metadata.
2994 LVer = std::make_unique<LoopVersioning>(
2995 *Legal->getLAI(),
2996 Legal->getLAI()->getRuntimePointerChecking()->getChecks(), OrigLoop, LI,
2997 DT, PSE.getSE());
2998 LVer->prepareNoAliasMetadata();
2999}
3000
3001Value *InnerLoopVectorizer::emitTransformedIndex(
3002 IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL,
3003 const InductionDescriptor &ID) const {
3004
3005 SCEVExpander Exp(*SE, DL, "induction");
3006 auto Step = ID.getStep();
3007 auto StartValue = ID.getStartValue();
3008 assert(Index->getType() == Step->getType() &&((Index->getType() == Step->getType() && "Index type does not match StepValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == Step->getType() && \"Index type does not match StepValue type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3009, __PRETTY_FUNCTION__))
3009 "Index type does not match StepValue type")((Index->getType() == Step->getType() && "Index type does not match StepValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == Step->getType() && \"Index type does not match StepValue type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3009, __PRETTY_FUNCTION__))
;
3010
3011 // Note: the IR at this point is broken. We cannot use SE to create any new
3012 // SCEV and then expand it, hoping that SCEV's simplification will give us
3013 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
3014 // lead to various SCEV crashes. So all we can do is to use builder and rely
3015 // on InstCombine for future simplifications. Here we handle some trivial
3016 // cases only.
3017 auto CreateAdd = [&B](Value *X, Value *Y) {
3018 assert(X->getType() == Y->getType() && "Types don't match!")((X->getType() == Y->getType() && "Types don't match!"
) ? static_cast<void> (0) : __assert_fail ("X->getType() == Y->getType() && \"Types don't match!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3018, __PRETTY_FUNCTION__))
;
3019 if (auto *CX = dyn_cast<ConstantInt>(X))
3020 if (CX->isZero())
3021 return Y;
3022 if (auto *CY = dyn_cast<ConstantInt>(Y))
3023 if (CY->isZero())
3024 return X;
3025 return B.CreateAdd(X, Y);
3026 };
3027
3028 auto CreateMul = [&B](Value *X, Value *Y) {
3029 assert(X->getType() == Y->getType() && "Types don't match!")((X->getType() == Y->getType() && "Types don't match!"
) ? static_cast<void> (0) : __assert_fail ("X->getType() == Y->getType() && \"Types don't match!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3029, __PRETTY_FUNCTION__))
;
3030 if (auto *CX = dyn_cast<ConstantInt>(X))
3031 if (CX->isOne())
3032 return Y;
3033 if (auto *CY = dyn_cast<ConstantInt>(Y))
3034 if (CY->isOne())
3035 return X;
3036 return B.CreateMul(X, Y);
3037 };
3038
3039 // Get a suitable insert point for SCEV expansion. For blocks in the vector
3040 // loop, choose the end of the vector loop header (=LoopVectorBody), because
3041 // the DomTree is not kept up-to-date for additional blocks generated in the
3042 // vector loop. By using the header as insertion point, we guarantee that the
3043 // expanded instructions dominate all their uses.
3044 auto GetInsertPoint = [this, &B]() {
3045 BasicBlock *InsertBB = B.GetInsertPoint()->getParent();
3046 if (InsertBB != LoopVectorBody &&
3047 LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB))
3048 return LoopVectorBody->getTerminator();
3049 return &*B.GetInsertPoint();
3050 };
3051 switch (ID.getKind()) {
3052 case InductionDescriptor::IK_IntInduction: {
3053 assert(Index->getType() == StartValue->getType() &&((Index->getType() == StartValue->getType() && "Index type does not match StartValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3054, __PRETTY_FUNCTION__))
3054 "Index type does not match StartValue type")((Index->getType() == StartValue->getType() && "Index type does not match StartValue type"
) ? static_cast<void> (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3054, __PRETTY_FUNCTION__))
;
3055 if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne())
3056 return B.CreateSub(StartValue, Index);
3057 auto *Offset = CreateMul(
3058 Index, Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint()));
3059 return CreateAdd(StartValue, Offset);
3060 }
3061 case InductionDescriptor::IK_PtrInduction: {
3062 assert(isa<SCEVConstant>(Step) &&((isa<SCEVConstant>(Step) && "Expected constant step for pointer induction"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(Step) && \"Expected constant step for pointer induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3063, __PRETTY_FUNCTION__))
3063 "Expected constant step for pointer induction")((isa<SCEVConstant>(Step) && "Expected constant step for pointer induction"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(Step) && \"Expected constant step for pointer induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3063, __PRETTY_FUNCTION__))
;
3064 return B.CreateGEP(
3065 StartValue->getType()->getPointerElementType(), StartValue,
3066 CreateMul(Index,
3067 Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint())));
3068 }
3069 case InductionDescriptor::IK_FpInduction: {
3070 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value")((Step->getType()->isFloatingPointTy() && "Expected FP Step value"
) ? static_cast<void> (0) : __assert_fail ("Step->getType()->isFloatingPointTy() && \"Expected FP Step value\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3070, __PRETTY_FUNCTION__))
;
3071 auto InductionBinOp = ID.getInductionBinOp();
3072 assert(InductionBinOp &&((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3075, __PRETTY_FUNCTION__))
3073 (InductionBinOp->getOpcode() == Instruction::FAdd ||((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3075, __PRETTY_FUNCTION__))
3074 InductionBinOp->getOpcode() == Instruction::FSub) &&((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3075, __PRETTY_FUNCTION__))
3075 "Original bin op should be defined for FP induction")((InductionBinOp && (InductionBinOp->getOpcode() ==
Instruction::FAdd || InductionBinOp->getOpcode() == Instruction
::FSub) && "Original bin op should be defined for FP induction"
) ? static_cast<void> (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3075, __PRETTY_FUNCTION__))
;
3076
3077 Value *StepValue = cast<SCEVUnknown>(Step)->getValue();
3078
3079 // Floating point operations had to be 'fast' to enable the induction.
3080 FastMathFlags Flags;
3081 Flags.setFast();
3082
3083 Value *MulExp = B.CreateFMul(StepValue, Index);
3084 if (isa<Instruction>(MulExp))
3085 // We have to check, the MulExp may be a constant.
3086 cast<Instruction>(MulExp)->setFastMathFlags(Flags);
3087
3088 Value *BOp = B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
3089 "induction");
3090 if (isa<Instruction>(BOp))
3091 cast<Instruction>(BOp)->setFastMathFlags(Flags);
3092
3093 return BOp;
3094 }
3095 case InductionDescriptor::IK_NoInduction:
3096 return nullptr;
3097 }
3098 llvm_unreachable("invalid enum")::llvm::llvm_unreachable_internal("invalid enum", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3098)
;
3099}
3100
3101Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
3102 LoopScalarBody = OrigLoop->getHeader();
3103 LoopVectorPreHeader = OrigLoop->getLoopPreheader();
3104 LoopExitBlock = OrigLoop->getExitBlock();
3105 assert(LoopExitBlock && "Must have an exit block")((LoopExitBlock && "Must have an exit block") ? static_cast
<void> (0) : __assert_fail ("LoopExitBlock && \"Must have an exit block\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3105, __PRETTY_FUNCTION__))
;
3106 assert(LoopVectorPreHeader && "Invalid loop structure")((LoopVectorPreHeader && "Invalid loop structure") ? static_cast
<void> (0) : __assert_fail ("LoopVectorPreHeader && \"Invalid loop structure\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3106, __PRETTY_FUNCTION__))
;
3107
3108 LoopMiddleBlock =
3109 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3110 LI, nullptr, Twine(Prefix) + "middle.block");
3111 LoopScalarPreHeader =
3112 SplitBlock(LoopMiddleBlock, LoopMiddleBlock->getTerminator(), DT, LI,
3113 nullptr, Twine(Prefix) + "scalar.ph");
3114 // We intentionally don't let SplitBlock to update LoopInfo since
3115 // LoopVectorBody should belong to another loop than LoopVectorPreHeader.
3116 // LoopVectorBody is explicitly added to the correct place few lines later.
3117 LoopVectorBody =
3118 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3119 nullptr, nullptr, Twine(Prefix) + "vector.body");
3120
3121 // Update dominator for loop exit.
3122 DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);
3123
3124 // Create and register the new vector loop.
3125 Loop *Lp = LI->AllocateLoop();
3126 Loop *ParentLoop = OrigLoop->getParentLoop();
3127
3128 // Insert the new loop into the loop nest and register the new basic blocks
3129 // before calling any utilities such as SCEV that require valid LoopInfo.
3130 if (ParentLoop) {
3131 ParentLoop->addChildLoop(Lp);
3132 } else {
3133 LI->addTopLevelLoop(Lp);
3134 }
3135 Lp->addBasicBlockToLoop(LoopVectorBody, *LI);
3136 return Lp;
3137}
3138
3139void InnerLoopVectorizer::createInductionResumeValues(Loop *L,
3140 Value *VectorTripCount) {
3141 assert(VectorTripCount && L && "Expected valid arguments")((VectorTripCount && L && "Expected valid arguments"
) ? static_cast<void> (0) : __assert_fail ("VectorTripCount && L && \"Expected valid arguments\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3141, __PRETTY_FUNCTION__))
;
3142 // We are going to resume the execution of the scalar loop.
3143 // Go over all of the induction variables that we found and fix the
3144 // PHIs that are left in the scalar version of the loop.
3145 // The starting values of PHI nodes depend on the counter of the last
3146 // iteration in the vectorized loop.
3147 // If we come from a bypass edge then we need to start from the original
3148 // start value.
3149 for (auto &InductionEntry : Legal->getInductionVars()) {
3150 PHINode *OrigPhi = InductionEntry.first;
3151 InductionDescriptor II = InductionEntry.second;
3152
3153 // Create phi nodes to merge from the backedge-taken check block.
3154 PHINode *BCResumeVal =
3155 PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
3156 LoopScalarPreHeader->getTerminator());
3157 // Copy original phi DL over to the new one.
3158 BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
3159 Value *&EndValue = IVEndValues[OrigPhi];
3160 if (OrigPhi == OldInduction) {
3161 // We know what the end value is.
3162 EndValue = VectorTripCount;
3163 } else {
3164 IRBuilder<> B(L->getLoopPreheader()->getTerminator());
3165 Type *StepType = II.getStep()->getType();
3166 Instruction::CastOps CastOp =
3167 CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
3168 Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd");
3169 const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout();
3170 EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
3171 EndValue->setName("ind.end");
3172 }
3173
3174 // The new PHI merges the original incoming value, in case of a bypass,
3175 // or the value at the end of the vectorized loop.
3176 BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
3177
3178 // Fix the scalar body counter (PHI node).
3179 // The old induction's phi node in the scalar body needs the truncated
3180 // value.
3181 for (BasicBlock *BB : LoopBypassBlocks)
3182 BCResumeVal->addIncoming(II.getStartValue(), BB);
3183 OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
3184 }
3185}
3186
3187BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L,
3188 MDNode *OrigLoopID) {
3189 assert(L && "Expected valid loop.")((L && "Expected valid loop.") ? static_cast<void>
(0) : __assert_fail ("L && \"Expected valid loop.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3189, __PRETTY_FUNCTION__))
;
3190
3191 // The trip counts should be cached by now.
3192 Value *Count = getOrCreateTripCount(L);
3193 Value *VectorTripCount = getOrCreateVectorTripCount(L);
3194
3195 // We need the OrigLoop (scalar loop part) latch terminator to help
3196 // produce correct debug info for the middle block BB instructions.
3197 // The legality check stage guarantees that the loop will have a single
3198 // latch.
3199 assert(isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator()) &&((isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator
()) && "Scalar loop latch terminator isn't a branch")
? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator()) && \"Scalar loop latch terminator isn't a branch\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3200, __PRETTY_FUNCTION__))
3200 "Scalar loop latch terminator isn't a branch")((isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator
()) && "Scalar loop latch terminator isn't a branch")
? static_cast<void> (0) : __assert_fail ("isa<BranchInst>(OrigLoop->getLoopLatch()->getTerminator()) && \"Scalar loop latch terminator isn't a branch\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3200, __PRETTY_FUNCTION__))
;
3201 BranchInst *ScalarLatchBr =
3202 cast<BranchInst>(OrigLoop->getLoopLatch()->getTerminator());
3203
3204 // Add a check in the middle block to see if we have completed
3205 // all of the iterations in the first vector loop.
3206 // If (N - N%VF) == N, then we *don't* need to run the remainder.
3207 // If tail is to be folded, we know we don't need to run the remainder.
3208 Value *CmpN = Builder.getTrue();
3209 if (!Cost->foldTailByMasking()) {
3210 CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, Count,
3211 VectorTripCount, "cmp.n",
3212 LoopMiddleBlock->getTerminator());
3213
3214 // Here we use the same DebugLoc as the scalar loop latch branch instead
3215 // of the corresponding compare because they may have ended up with
3216 // different line numbers and we want to avoid awkward line stepping while
3217 // debugging. Eg. if the compare has got a line number inside the loop.
3218 cast<Instruction>(CmpN)->setDebugLoc(ScalarLatchBr->getDebugLoc());
3219 }
3220
3221 BranchInst *BrInst =
3222 BranchInst::Create(LoopExitBlock, LoopScalarPreHeader, CmpN);
3223 BrInst->setDebugLoc(ScalarLatchBr->getDebugLoc());
3224 ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst);
3225
3226 // Get ready to start creating new instructions into the vectorized body.
3227 assert(LoopVectorPreHeader == L->getLoopPreheader() &&((LoopVectorPreHeader == L->getLoopPreheader() && "Inconsistent vector loop preheader"
) ? static_cast<void> (0) : __assert_fail ("LoopVectorPreHeader == L->getLoopPreheader() && \"Inconsistent vector loop preheader\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3228, __PRETTY_FUNCTION__))
3228 "Inconsistent vector loop preheader")((LoopVectorPreHeader == L->getLoopPreheader() && "Inconsistent vector loop preheader"
) ? static_cast<void> (0) : __assert_fail ("LoopVectorPreHeader == L->getLoopPreheader() && \"Inconsistent vector loop preheader\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3228, __PRETTY_FUNCTION__))
;
3229 Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());
3230
3231 Optional<MDNode *> VectorizedLoopID =
3232 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
3233 LLVMLoopVectorizeFollowupVectorized});
3234 if (VectorizedLoopID.hasValue()) {
3235 L->setLoopID(VectorizedLoopID.getValue());
3236
3237 // Do not setAlreadyVectorized if loop attributes have been defined
3238 // explicitly.
3239 return LoopVectorPreHeader;
3240 }
3241
3242 // Keep all loop hints from the original loop on the vector loop (we'll
3243 // replace the vectorizer-specific hints below).
3244 if (MDNode *LID = OrigLoop->getLoopID())
3245 L->setLoopID(LID);
3246
3247 LoopVectorizeHints Hints(L, true, *ORE);
3248 Hints.setAlreadyVectorized();
3249
3250#ifdef EXPENSIVE_CHECKS
3251 assert(DT->verify(DominatorTree::VerificationLevel::Fast))((DT->verify(DominatorTree::VerificationLevel::Fast)) ? static_cast
<void> (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast)"
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3251, __PRETTY_FUNCTION__))
;
3252 LI->verify(*DT);
3253#endif
3254
3255 return LoopVectorPreHeader;
3256}
3257
3258BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
3259 /*
3260 In this function we generate a new loop. The new loop will contain
3261 the vectorized instructions while the old loop will continue to run the
3262 scalar remainder.
3263
3264 [ ] <-- loop iteration number check.
3265 / |
3266 / v
3267 | [ ] <-- vector loop bypass (may consist of multiple blocks).
3268 | / |
3269 | / v
3270 || [ ] <-- vector pre header.
3271 |/ |
3272 | v
3273 | [ ] \
3274 | [ ]_| <-- vector loop.
3275 | |
3276 | v
3277 | -[ ] <--- middle-block.
3278 | / |
3279 | / v
3280 -|- >[ ] <--- new preheader.
3281 | |
3282 | v
3283 | [ ] \
3284 | [ ]_| <-- old scalar loop to handle remainder.
3285 \ |
3286 \ v
3287 >[ ] <-- exit block.
3288 ...
3289 */
3290
3291 // Get the metadata of the original loop before it gets modified.
3292 MDNode *OrigLoopID = OrigLoop->getLoopID();
3293
3294 // Create an empty vector loop, and prepare basic blocks for the runtime
3295 // checks.
3296 Loop *Lp = createVectorLoopSkeleton("");
3297
3298 // Now, compare the new count to zero. If it is zero skip the vector loop and
3299 // jump to the scalar loop. This check also covers the case where the
3300 // backedge-taken count is uint##_max: adding one to it will overflow leading
3301 // to an incorrect trip count of zero. In this (rare) case we will also jump
3302 // to the scalar loop.
3303 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader);
3304
3305 // Generate the code to check any assumptions that we've made for SCEV
3306 // expressions.
3307 emitSCEVChecks(Lp, LoopScalarPreHeader);
3308
3309 // Generate the code that checks in runtime if arrays overlap. We put the
3310 // checks into a separate block to make the more common case of few elements
3311 // faster.
3312 emitMemRuntimeChecks(Lp, LoopScalarPreHeader);
3313
3314 // Some loops have a single integer induction variable, while other loops
3315 // don't. One example is c++ iterators that often have multiple pointer
3316 // induction variables. In the code below we also support a case where we
3317 // don't have a single induction variable.
3318 //
3319 // We try to obtain an induction variable from the original loop as hard
3320 // as possible. However if we don't find one that:
3321 // - is an integer
3322 // - counts from zero, stepping by one
3323 // - is the size of the widest induction variable type
3324 // then we create a new one.
3325 OldInduction = Legal->getPrimaryInduction();
3326 Type *IdxTy = Legal->getWidestInductionType();
3327 Value *StartIdx = ConstantInt::get(IdxTy, 0);
3328 // The loop step is equal to the vectorization factor (num of SIMD elements)
3329 // times the unroll factor (num of SIMD instructions).
3330 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3330, __PRETTY_FUNCTION__))
;
3331 Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
3332 Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
3333 Induction =
3334 createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
3335 getDebugLocFromInstOrOperands(OldInduction));
3336
3337 // Emit phis for the new starting index of the scalar loop.
3338 createInductionResumeValues(Lp, CountRoundDown);
3339
3340 return completeLoopSkeleton(Lp, OrigLoopID);
3341}
3342
3343// Fix up external users of the induction variable. At this point, we are
3344// in LCSSA form, with all external PHIs that use the IV having one input value,
3345// coming from the remainder loop. We need those PHIs to also have a correct
3346// value for the IV when arriving directly from the middle block.
3347void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
3348 const InductionDescriptor &II,
3349 Value *CountRoundDown, Value *EndValue,
3350 BasicBlock *MiddleBlock) {
3351 // There are two kinds of external IV usages - those that use the value
3352 // computed in the last iteration (the PHI) and those that use the penultimate
3353 // value (the value that feeds into the phi from the loop latch).
3354 // We allow both, but they, obviously, have different values.
3355
3356 assert(OrigLoop->getExitBlock() && "Expected a single exit block")((OrigLoop->getExitBlock() && "Expected a single exit block"
) ? static_cast<void> (0) : __assert_fail ("OrigLoop->getExitBlock() && \"Expected a single exit block\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3356, __PRETTY_FUNCTION__))
;
3357
3358 DenseMap<Value *, Value *> MissingVals;
3359
3360 // An external user of the last iteration's value should see the value that
3361 // the remainder loop uses to initialize its own IV.
3362 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());
3363 for (User *U : PostInc->users()) {
3364 Instruction *UI = cast<Instruction>(U);
3365 if (!OrigLoop->contains(UI)) {
3366 assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3366, __PRETTY_FUNCTION__))
;
3367 MissingVals[UI] = EndValue;
3368 }
3369 }
3370
3371 // An external user of the penultimate value need to see EndValue - Step.
3372 // The simplest way to get this is to recompute it from the constituent SCEVs,
3373 // that is Start + (Step * (CRD - 1)).
3374 for (User *U : OrigPhi->users()) {
3375 auto *UI = cast<Instruction>(U);
3376 if (!OrigLoop->contains(UI)) {
3377 const DataLayout &DL =
3378 OrigLoop->getHeader()->getModule()->getDataLayout();
3379 assert(isa<PHINode>(UI) && "Expected LCSSA form")((isa<PHINode>(UI) && "Expected LCSSA form") ? static_cast
<void> (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3379, __PRETTY_FUNCTION__))
;
3380
3381 IRBuilder<> B(MiddleBlock->getTerminator());
3382 Value *CountMinusOne = B.CreateSub(
3383 CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));
3384 Value *CMO =
3385 !II.getStep()->getType()->isIntegerTy()
3386 ? B.CreateCast(Instruction::SIToFP, CountMinusOne,
3387 II.getStep()->getType())
3388 : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
3389 CMO->setName("cast.cmo");
3390 Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II);
3391 Escape->setName("ind.escape");
3392 MissingVals[UI] = Escape;
3393 }
3394 }
3395
3396 for (auto &I : MissingVals) {
3397 PHINode *PHI = cast<PHINode>(I.first);
3398 // One corner case we have to handle is two IVs "chasing" each-other,
3399 // that is %IV2 = phi [...], [ %IV1, %latch ]
3400 // In this case, if IV1 has an external use, we need to avoid adding both
3401 // "last value of IV1" and "penultimate value of IV2". So, verify that we
3402 // don't already have an incoming value for the middle block.
3403 if (PHI->getBasicBlockIndex(MiddleBlock) == -1)
3404 PHI->addIncoming(I.second, MiddleBlock);
3405 }
3406}
3407
3408namespace {
3409
3410struct CSEDenseMapInfo {
3411 static bool canHandle(const Instruction *I) {
3412 return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
3413 isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
3414 }
3415
3416 static inline Instruction *getEmptyKey() {
3417 return DenseMapInfo<Instruction *>::getEmptyKey();
3418 }
3419
3420 static inline Instruction *getTombstoneKey() {
3421 return DenseMapInfo<Instruction *>::getTombstoneKey();
3422 }
3423
3424 static unsigned getHashValue(const Instruction *I) {
3425 assert(canHandle(I) && "Unknown instruction!")((canHandle(I) && "Unknown instruction!") ? static_cast
<void> (0) : __assert_fail ("canHandle(I) && \"Unknown instruction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3425, __PRETTY_FUNCTION__))
;
3426 return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
3427 I->value_op_end()));
3428 }
3429
3430 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
3431 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
3432 LHS == getTombstoneKey() || RHS == getTombstoneKey())
3433 return LHS == RHS;
3434 return LHS->isIdenticalTo(RHS);
3435 }
3436};
3437
3438} // end anonymous namespace
3439
3440///Perform cse of induction variable instructions.
3441static void cse(BasicBlock *BB) {
3442 // Perform simple cse.
3443 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
3444 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
3445 Instruction *In = &*I++;
3446
3447 if (!CSEDenseMapInfo::canHandle(In))
3448 continue;
3449
3450 // Check if we can replace this instruction with any of the
3451 // visited instructions.
3452 if (Instruction *V = CSEMap.lookup(In)) {
3453 In->replaceAllUsesWith(V);
3454 In->eraseFromParent();
3455 continue;
3456 }
3457
3458 CSEMap[In] = In;
3459 }
3460}
3461
3462unsigned LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
3463 ElementCount VF,
3464 bool &NeedToScalarize) {
3465 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3465, __PRETTY_FUNCTION__))
;
3466 Function *F = CI->getCalledFunction();
3467 Type *ScalarRetTy = CI->getType();
3468 SmallVector<Type *, 4> Tys, ScalarTys;
3469 for (auto &ArgOp : CI->arg_operands())
3470 ScalarTys.push_back(ArgOp->getType());
3471
3472 // Estimate cost of scalarized vector call. The source operands are assumed
3473 // to be vectors, so we need to extract individual elements from there,
3474 // execute VF scalar calls, and then gather the result into the vector return
3475 // value.
3476 unsigned ScalarCallCost = TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys,
3477 TTI::TCK_RecipThroughput);
3478 if (VF.isScalar())
3479 return ScalarCallCost;
3480
3481 // Compute corresponding vector type for return value and arguments.
3482 Type *RetTy = ToVectorTy(ScalarRetTy, VF);
3483 for (Type *ScalarTy : ScalarTys)
3484 Tys.push_back(ToVectorTy(ScalarTy, VF));
3485
3486 // Compute costs of unpacking argument values for the scalar calls and
3487 // packing the return values to a vector.
3488 unsigned ScalarizationCost = getScalarizationOverhead(CI, VF);
3489
3490 unsigned Cost = ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
3491
3492 // If we can't emit a vector call for this function, then the currently found
3493 // cost is the cost we need to return.
3494 NeedToScalarize = true;
3495 VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
3496 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3497
3498 if (!TLI || CI->isNoBuiltin() || !VecFunc)
3499 return Cost;
3500
3501 // If the corresponding vector cost is cheaper, return its cost.
3502 unsigned VectorCallCost = TTI.getCallInstrCost(nullptr, RetTy, Tys,
3503 TTI::TCK_RecipThroughput);
3504 if (VectorCallCost < Cost) {
3505 NeedToScalarize = false;
3506 return VectorCallCost;
3507 }
3508 return Cost;
3509}
3510
3511unsigned LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
3512 ElementCount VF) {
3513 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3514 assert(ID && "Expected intrinsic call!")((ID && "Expected intrinsic call!") ? static_cast<
void> (0) : __assert_fail ("ID && \"Expected intrinsic call!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3514, __PRETTY_FUNCTION__))
;
3515
3516 IntrinsicCostAttributes CostAttrs(ID, *CI, VF);
3517 return TTI.getIntrinsicInstrCost(CostAttrs,
3518 TargetTransformInfo::TCK_RecipThroughput);
3519}
3520
3521static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
3522 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3523 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3524 return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
3525}
3526
3527static Type *largestIntegerVectorType(Type *T1, Type *T2) {
3528 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3529 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3530 return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
3531}
3532
3533void InnerLoopVectorizer::truncateToMinimalBitwidths() {
3534 // For every instruction `I` in MinBWs, truncate the operands, create a
3535 // truncated version of `I` and reextend its result. InstCombine runs
3536 // later and will remove any ext/trunc pairs.
3537 SmallPtrSet<Value *, 4> Erased;
3538 for (const auto &KV : Cost->getMinimalBitwidths()) {
3539 // If the value wasn't vectorized, we must maintain the original scalar
3540 // type. The absence of the value from VectorLoopValueMap indicates that it
3541 // wasn't vectorized.
3542 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))
3543 continue;
3544 for (unsigned Part = 0; Part < UF; ++Part) {
3545 Value *I = getOrCreateVectorValue(KV.first, Part);
3546 if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I))
3547 continue;
3548 Type *OriginalTy = I->getType();
3549 Type *ScalarTruncatedTy =
3550 IntegerType::get(OriginalTy->getContext(), KV.second);
3551 auto *TruncatedTy = FixedVectorType::get(
3552 ScalarTruncatedTy,
3553 cast<FixedVectorType>(OriginalTy)->getNumElements());
3554 if (TruncatedTy == OriginalTy)
3555 continue;
3556
3557 IRBuilder<> B(cast<Instruction>(I));
3558 auto ShrinkOperand = [&](Value *V) -> Value * {
3559 if (auto *ZI = dyn_cast<ZExtInst>(V))
3560 if (ZI->getSrcTy() == TruncatedTy)
3561 return ZI->getOperand(0);
3562 return B.CreateZExtOrTrunc(V, TruncatedTy);
3563 };
3564
3565 // The actual instruction modification depends on the instruction type,
3566 // unfortunately.
3567 Value *NewI = nullptr;
3568 if (auto *BO = dyn_cast<BinaryOperator>(I)) {
3569 NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),
3570 ShrinkOperand(BO->getOperand(1)));
3571
3572 // Any wrapping introduced by shrinking this operation shouldn't be
3573 // considered undefined behavior. So, we can't unconditionally copy
3574 // arithmetic wrapping flags to NewI.
3575 cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);
3576 } else if (auto *CI = dyn_cast<ICmpInst>(I)) {
3577 NewI =
3578 B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),
3579 ShrinkOperand(CI->getOperand(1)));
3580 } else if (auto *SI = dyn_cast<SelectInst>(I)) {
3581 NewI = B.CreateSelect(SI->getCondition(),
3582 ShrinkOperand(SI->getTrueValue()),
3583 ShrinkOperand(SI->getFalseValue()));
3584 } else if (auto *CI = dyn_cast<CastInst>(I)) {
3585 switch (CI->getOpcode()) {
3586 default:
3587 llvm_unreachable("Unhandled cast!")::llvm::llvm_unreachable_internal("Unhandled cast!", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3587)
;
3588 case Instruction::Trunc:
3589 NewI = ShrinkOperand(CI->getOperand(0));
3590 break;
3591 case Instruction::SExt:
3592 NewI = B.CreateSExtOrTrunc(
3593 CI->getOperand(0),
3594 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3595 break;
3596 case Instruction::ZExt:
3597 NewI = B.CreateZExtOrTrunc(
3598 CI->getOperand(0),
3599 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3600 break;
3601 }
3602 } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {
3603 auto Elements0 = cast<FixedVectorType>(SI->getOperand(0)->getType())
3604 ->getNumElements();
3605 auto *O0 = B.CreateZExtOrTrunc(
3606 SI->getOperand(0),
3607 FixedVectorType::get(ScalarTruncatedTy, Elements0));
3608 auto Elements1 = cast<FixedVectorType>(SI->getOperand(1)->getType())
3609 ->getNumElements();
3610 auto *O1 = B.CreateZExtOrTrunc(
3611 SI->getOperand(1),
3612 FixedVectorType::get(ScalarTruncatedTy, Elements1));
3613
3614 NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
3615 } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
3616 // Don't do anything with the operands, just extend the result.
3617 continue;
3618 } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
3619 auto Elements = cast<FixedVectorType>(IE->getOperand(0)->getType())
3620 ->getNumElements();
3621 auto *O0 = B.CreateZExtOrTrunc(
3622 IE->getOperand(0),
3623 FixedVectorType::get(ScalarTruncatedTy, Elements));
3624 auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);
3625 NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));
3626 } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
3627 auto Elements = cast<FixedVectorType>(EE->getOperand(0)->getType())
3628 ->getNumElements();
3629 auto *O0 = B.CreateZExtOrTrunc(
3630 EE->getOperand(0),
3631 FixedVectorType::get(ScalarTruncatedTy, Elements));
3632 NewI = B.CreateExtractElement(O0, EE->getOperand(2));
3633 } else {
3634 // If we don't know what to do, be conservative and don't do anything.
3635 continue;
3636 }
3637
3638 // Lastly, extend the result.
3639 NewI->takeName(cast<Instruction>(I));
3640 Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
3641 I->replaceAllUsesWith(Res);
3642 cast<Instruction>(I)->eraseFromParent();
3643 Erased.insert(I);
3644 VectorLoopValueMap.resetVectorValue(KV.first, Part, Res);
3645 }
3646 }
3647
3648 // We'll have created a bunch of ZExts that are now parentless. Clean up.
3649 for (const auto &KV : Cost->getMinimalBitwidths()) {
3650 // If the value wasn't vectorized, we must maintain the original scalar
3651 // type. The absence of the value from VectorLoopValueMap indicates that it
3652 // wasn't vectorized.
3653 if (!VectorLoopValueMap.hasAnyVectorValue(KV.first))
3654 continue;
3655 for (unsigned Part = 0; Part < UF; ++Part) {
3656 Value *I = getOrCreateVectorValue(KV.first, Part);
3657 ZExtInst *Inst = dyn_cast<ZExtInst>(I);
3658 if (Inst && Inst->use_empty()) {
3659 Value *NewI = Inst->getOperand(0);
3660 Inst->eraseFromParent();
3661 VectorLoopValueMap.resetVectorValue(KV.first, Part, NewI);
3662 }
3663 }
3664 }
3665}
3666
3667void InnerLoopVectorizer::fixVectorizedLoop() {
3668 // Insert truncates and extends for any truncated instructions as hints to
3669 // InstCombine.
3670 if (VF.isVector())
3671 truncateToMinimalBitwidths();
3672
3673 // Fix widened non-induction PHIs by setting up the PHI operands.
3674 if (OrigPHIsToFix.size()) {
3675 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3676, __PRETTY_FUNCTION__))
3676 "Unexpected non-induction PHIs for fixup in non VPlan-native path")((EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3676, __PRETTY_FUNCTION__))
;
3677 fixNonInductionPHIs();
3678 }
3679
3680 // At this point every instruction in the original loop is widened to a
3681 // vector form. Now we need to fix the recurrences in the loop. These PHI
3682 // nodes are currently empty because we did not want to introduce cycles.
3683 // This is the second stage of vectorizing recurrences.
3684 fixCrossIterationPHIs();
3685
3686 // Forget the original basic block.
3687 PSE.getSE()->forgetLoop(OrigLoop);
3688
3689 // Fix-up external users of the induction variables.
3690 for (auto &Entry : Legal->getInductionVars())
3691 fixupIVUsers(Entry.first, Entry.second,
3692 getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)),
3693 IVEndValues[Entry.first], LoopMiddleBlock);
3694
3695 fixLCSSAPHIs();
3696 for (Instruction *PI : PredicatedInstructions)
3697 sinkScalarOperands(&*PI);
3698
3699 // Remove redundant induction instructions.
3700 cse(LoopVectorBody);
3701
3702 // Set/update profile weights for the vector and remainder loops as original
3703 // loop iterations are now distributed among them. Note that original loop
3704 // represented by LoopScalarBody becomes remainder loop after vectorization.
3705 //
3706 // For cases like foldTailByMasking() and requiresScalarEpiloque() we may
3707 // end up getting slightly roughened result but that should be OK since
3708 // profile is not inherently precise anyway. Note also possible bypass of
3709 // vector code caused by legality checks is ignored, assigning all the weight
3710 // to the vector loop, optimistically.
3711 assert(!VF.isScalable() &&((!VF.isScalable() && "cannot use scalable ElementCount to determine unroll factor"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"cannot use scalable ElementCount to determine unroll factor\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3712, __PRETTY_FUNCTION__))
3712 "cannot use scalable ElementCount to determine unroll factor")((!VF.isScalable() && "cannot use scalable ElementCount to determine unroll factor"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"cannot use scalable ElementCount to determine unroll factor\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3712, __PRETTY_FUNCTION__))
;
3713 setProfileInfoAfterUnrolling(
3714 LI->getLoopFor(LoopScalarBody), LI->getLoopFor(LoopVectorBody),
3715 LI->getLoopFor(LoopScalarBody), VF.getKnownMinValue() * UF);
3716}
3717
3718void InnerLoopVectorizer::fixCrossIterationPHIs() {
3719 // In order to support recurrences we need to be able to vectorize Phi nodes.
3720 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
3721 // stage #2: We now need to fix the recurrences by adding incoming edges to
3722 // the currently empty PHI nodes. At this point every instruction in the
3723 // original loop is widened to a vector form so we can use them to construct
3724 // the incoming edges.
3725 for (PHINode &Phi : OrigLoop->getHeader()->phis()) {
3726 // Handle first-order recurrences and reductions that need to be fixed.
3727 if (Legal->isFirstOrderRecurrence(&Phi))
3728 fixFirstOrderRecurrence(&Phi);
3729 else if (Legal->isReductionVariable(&Phi))
3730 fixReduction(&Phi);
3731 }
3732}
3733
3734void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi) {
3735 // This is the second phase of vectorizing first-order recurrences. An
3736 // overview of the transformation is described below. Suppose we have the
3737 // following loop.
3738 //
3739 // for (int i = 0; i < n; ++i)
3740 // b[i] = a[i] - a[i - 1];
3741 //
3742 // There is a first-order recurrence on "a". For this loop, the shorthand
3743 // scalar IR looks like:
3744 //
3745 // scalar.ph:
3746 // s_init = a[-1]
3747 // br scalar.body
3748 //
3749 // scalar.body:
3750 // i = phi [0, scalar.ph], [i+1, scalar.body]
3751 // s1 = phi [s_init, scalar.ph], [s2, scalar.body]
3752 // s2 = a[i]
3753 // b[i] = s2 - s1
3754 // br cond, scalar.body, ...
3755 //
3756 // In this example, s1 is a recurrence because it's value depends on the
3757 // previous iteration. In the first phase of vectorization, we created a
3758 // temporary value for s1. We now complete the vectorization and produce the
3759 // shorthand vector IR shown below (for VF = 4, UF = 1).
3760 //
3761 // vector.ph:
3762 // v_init = vector(..., ..., ..., a[-1])
3763 // br vector.body
3764 //
3765 // vector.body
3766 // i = phi [0, vector.ph], [i+4, vector.body]
3767 // v1 = phi [v_init, vector.ph], [v2, vector.body]
3768 // v2 = a[i, i+1, i+2, i+3];
3769 // v3 = vector(v1(3), v2(0, 1, 2))
3770 // b[i, i+1, i+2, i+3] = v2 - v3
3771 // br cond, vector.body, middle.block
3772 //
3773 // middle.block:
3774 // x = v2(3)
3775 // br scalar.ph
3776 //
3777 // scalar.ph:
3778 // s_init = phi [x, middle.block], [a[-1], otherwise]
3779 // br scalar.body
3780 //
3781 // After execution completes the vector loop, we extract the next value of
3782 // the recurrence (x) to use as the initial value in the scalar loop.
3783
3784 // Get the original loop preheader and single loop latch.
3785 auto *Preheader = OrigLoop->getLoopPreheader();
3786 auto *Latch = OrigLoop->getLoopLatch();
3787
3788 // Get the initial and previous values of the scalar recurrence.
3789 auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader);
3790 auto *Previous = Phi->getIncomingValueForBlock(Latch);
3791
3792 // Create a vector from the initial value.
3793 auto *VectorInit = ScalarInit;
3794 if (VF.isVector()) {
3795 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
3796 assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3796, __PRETTY_FUNCTION__))
;
3797 VectorInit = Builder.CreateInsertElement(
3798 UndefValue::get(VectorType::get(VectorInit->getType(), VF)), VectorInit,
3799 Builder.getInt32(VF.getKnownMinValue() - 1), "vector.recur.init");
3800 }
3801
3802 // We constructed a temporary phi node in the first phase of vectorization.
3803 // This phi node will eventually be deleted.
3804 Builder.SetInsertPoint(
3805 cast<Instruction>(VectorLoopValueMap.getVectorValue(Phi, 0)));
3806
3807 // Create a phi node for the new recurrence. The current value will either be
3808 // the initial value inserted into a vector or loop-varying vector value.
3809 auto *VecPhi = Builder.CreatePHI(VectorInit->getType(), 2, "vector.recur");
3810 VecPhi->addIncoming(VectorInit, LoopVectorPreHeader);
3811
3812 // Get the vectorized previous value of the last part UF - 1. It appears last
3813 // among all unrolled iterations, due to the order of their construction.
3814 Value *PreviousLastPart = getOrCreateVectorValue(Previous, UF - 1);
3815
3816 // Find and set the insertion point after the previous value if it is an
3817 // instruction.
3818 BasicBlock::iterator InsertPt;
3819 // Note that the previous value may have been constant-folded so it is not
3820 // guaranteed to be an instruction in the vector loop.
3821 // FIXME: Loop invariant values do not form recurrences. We should deal with
3822 // them earlier.
3823 if (LI->getLoopFor(LoopVectorBody)->isLoopInvariant(PreviousLastPart))
3824 InsertPt = LoopVectorBody->getFirstInsertionPt();
3825 else {
3826 Instruction *PreviousInst = cast<Instruction>(PreviousLastPart);
3827 if (isa<PHINode>(PreviousLastPart))
3828 // If the previous value is a phi node, we should insert after all the phi
3829 // nodes in the block containing the PHI to avoid breaking basic block
3830 // verification. Note that the basic block may be different to
3831 // LoopVectorBody, in case we predicate the loop.
3832 InsertPt = PreviousInst->getParent()->getFirstInsertionPt();
3833 else
3834 InsertPt = ++PreviousInst->getIterator();
3835 }
3836 Builder.SetInsertPoint(&*InsertPt);
3837
3838 // We will construct a vector for the recurrence by combining the values for
3839 // the current and previous iterations. This is the required shuffle mask.
3840 assert(!VF.isScalable())((!VF.isScalable()) ? static_cast<void> (0) : __assert_fail
("!VF.isScalable()", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3840, __PRETTY_FUNCTION__))
;
3841 SmallVector<int, 8> ShuffleMask(VF.getKnownMinValue());
3842 ShuffleMask[0] = VF.getKnownMinValue() - 1;
3843 for (unsigned I = 1; I < VF.getKnownMinValue(); ++I)
3844 ShuffleMask[I] = I + VF.getKnownMinValue() - 1;
3845
3846 // The vector from which to take the initial value for the current iteration
3847 // (actual or unrolled). Initially, this is the vector phi node.
3848 Value *Incoming = VecPhi;
3849
3850 // Shuffle the current and previous vector and update the vector parts.
3851 for (unsigned Part = 0; Part < UF; ++Part) {
3852 Value *PreviousPart = getOrCreateVectorValue(Previous, Part);
3853 Value *PhiPart = VectorLoopValueMap.getVectorValue(Phi, Part);
3854 auto *Shuffle =
3855 VF.isVector()
3856 ? Builder.CreateShuffleVector(Incoming, PreviousPart, ShuffleMask)
3857 : Incoming;
3858 PhiPart->replaceAllUsesWith(Shuffle);
3859 cast<Instruction>(PhiPart)->eraseFromParent();
3860 VectorLoopValueMap.resetVectorValue(Phi, Part, Shuffle);
3861 Incoming = PreviousPart;
3862 }
3863
3864 // Fix the latch value of the new recurrence in the vector loop.
3865 VecPhi->addIncoming(Incoming, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
3866
3867 // Extract the last vector element in the middle block. This will be the
3868 // initial value for the recurrence when jumping to the scalar loop.
3869 auto *ExtractForScalar = Incoming;
3870 if (VF.isVector()) {
3871 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
3872 ExtractForScalar = Builder.CreateExtractElement(
3873 ExtractForScalar, Builder.getInt32(VF.getKnownMinValue() - 1),
3874 "vector.recur.extract");
3875 }
3876 // Extract the second last element in the middle block if the
3877 // Phi is used outside the loop. We need to extract the phi itself
3878 // and not the last element (the phi update in the current iteration). This
3879 // will be the value when jumping to the exit block from the LoopMiddleBlock,
3880 // when the scalar loop is not run at all.
3881 Value *ExtractForPhiUsedOutsideLoop = nullptr;
3882 if (VF.isVector())
3883 ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(
3884 Incoming, Builder.getInt32(VF.getKnownMinValue() - 2),
3885 "vector.recur.extract.for.phi");
3886 // When loop is unrolled without vectorizing, initialize
3887 // ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value of
3888 // `Incoming`. This is analogous to the vectorized case above: extracting the
3889 // second last element when VF > 1.
3890 else if (UF > 1)
3891 ExtractForPhiUsedOutsideLoop = getOrCreateVectorValue(Previous, UF - 2);
3892
3893 // Fix the initial value of the original recurrence in the scalar loop.
3894 Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
3895 auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
3896 for (auto *BB : predecessors(LoopScalarPreHeader)) {
3897 auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
3898 Start->addIncoming(Incoming, BB);
3899 }
3900
3901 Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start);
3902 Phi->setName("scalar.recur");
3903
3904 // Finally, fix users of the recurrence outside the loop. The users will need
3905 // either the last value of the scalar recurrence or the last value of the
3906 // vector recurrence we extracted in the middle block. Since the loop is in
3907 // LCSSA form, we just need to find all the phi nodes for the original scalar
3908 // recurrence in the exit block, and then add an edge for the middle block.
3909 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
3910 if (LCSSAPhi.getIncomingValue(0) == Phi) {
3911 LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
3912 }
3913 }
3914}
3915
3916void InnerLoopVectorizer::fixReduction(PHINode *Phi) {
3917 Constant *Zero = Builder.getInt32(0);
3918
3919 // Get it's reduction variable descriptor.
3920 assert(Legal->isReductionVariable(Phi) &&((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3921, __PRETTY_FUNCTION__))
3921 "Unable to find the reduction variable")((Legal->isReductionVariable(Phi) && "Unable to find the reduction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->isReductionVariable(Phi) && \"Unable to find the reduction variable\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3921, __PRETTY_FUNCTION__))
;
3922 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
3923
3924 RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
3925 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
3926 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
3927 RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind =
3928 RdxDesc.getMinMaxRecurrenceKind();
3929 setDebugLocFromInst(Builder, ReductionStartValue);
3930 bool IsInLoopReductionPhi = Cost->isInLoopReduction(Phi);
3931
3932 // We need to generate a reduction vector from the incoming scalar.
3933 // To do so, we need to generate the 'identity' vector and override
3934 // one of the elements with the incoming scalar reduction. We need
3935 // to do it in the vector-loop preheader.
3936 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
3937
3938 // This is the vector-clone of the value that leaves the loop.
3939 Type *VecTy = getOrCreateVectorValue(LoopExitInst, 0)->getType();
3940
3941 // Find the reduction identity variable. Zero for addition, or, xor,
3942 // one for multiplication, -1 for And.
3943 Value *Identity;
3944 Value *VectorStart;
3945 if (RK == RecurrenceDescriptor::RK_IntegerMinMax ||
3946 RK == RecurrenceDescriptor::RK_FloatMinMax) {
3947 // MinMax reduction have the start value as their identify.
3948 if (VF.isScalar() || IsInLoopReductionPhi) {
3949 VectorStart = Identity = ReductionStartValue;
3950 } else {
3951 VectorStart = Identity =
3952 Builder.CreateVectorSplat(VF, ReductionStartValue, "minmax.ident");
3953 }
3954 } else {
3955 // Handle other reduction kinds:
3956 Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity(
3957 RK, MinMaxKind, VecTy->getScalarType());
3958 if (VF.isScalar() || IsInLoopReductionPhi) {
3959 Identity = Iden;
3960 // This vector is the Identity vector where the first element is the
3961 // incoming scalar reduction.
3962 VectorStart = ReductionStartValue;
3963 } else {
3964 Identity = ConstantVector::getSplat(VF, Iden);
3965
3966 // This vector is the Identity vector where the first element is the
3967 // incoming scalar reduction.
3968 VectorStart =
3969 Builder.CreateInsertElement(Identity, ReductionStartValue, Zero);
3970 }
3971 }
3972
3973 // Wrap flags are in general invalid after vectorization, clear them.
3974 clearReductionWrapFlags(RdxDesc);
3975
3976 // Fix the vector-loop phi.
3977
3978 // Reductions do not have to start at zero. They can start with
3979 // any loop invariant values.
3980 BasicBlock *Latch = OrigLoop->getLoopLatch();
3981 Value *LoopVal = Phi->getIncomingValueForBlock(Latch);
3982
3983 for (unsigned Part = 0; Part < UF; ++Part) {
3984 Value *VecRdxPhi = getOrCreateVectorValue(Phi, Part);
3985 Value *Val = getOrCreateVectorValue(LoopVal, Part);
3986 // Make sure to add the reduction start value only to the
3987 // first unroll part.
3988 Value *StartVal = (Part == 0) ? VectorStart : Identity;
3989 cast<PHINode>(VecRdxPhi)->addIncoming(StartVal, LoopVectorPreHeader);
3990 cast<PHINode>(VecRdxPhi)
3991 ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
3992 }
3993
3994 // Before each round, move the insertion point right between
3995 // the PHIs and the values we are going to write.
3996 // This allows us to write both PHINodes and the extractelement
3997 // instructions.
3998 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
3999
4000 setDebugLocFromInst(Builder, LoopExitInst);
4001
4002 // If tail is folded by masking, the vector value to leave the loop should be
4003 // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
4004 // instead of the former. For an inloop reduction the reduction will already
4005 // be predicated, and does not need to be handled here.
4006 if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) {
4007 for (unsigned Part = 0; Part < UF; ++Part) {
4008 Value *VecLoopExitInst =
4009 VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
4010 Value *Sel = nullptr;
4011 for (User *U : VecLoopExitInst->users()) {
4012 if (isa<SelectInst>(U)) {
4013 assert(!Sel && "Reduction exit feeding two selects")((!Sel && "Reduction exit feeding two selects") ? static_cast
<void> (0) : __assert_fail ("!Sel && \"Reduction exit feeding two selects\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4013, __PRETTY_FUNCTION__))
;
4014 Sel = U;
4015 } else
4016 assert(isa<PHINode>(U) && "Reduction exit must feed Phi's or select")((isa<PHINode>(U) && "Reduction exit must feed Phi's or select"
) ? static_cast<void> (0) : __assert_fail ("isa<PHINode>(U) && \"Reduction exit must feed Phi's or select\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4016, __PRETTY_FUNCTION__))
;
4017 }
4018 assert(Sel && "Reduction exit feeds no select")((Sel && "Reduction exit feeds no select") ? static_cast
<void> (0) : __assert_fail ("Sel && \"Reduction exit feeds no select\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4018, __PRETTY_FUNCTION__))
;
4019 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, Sel);
4020
4021 // If the target can create a predicated operator for the reduction at no
4022 // extra cost in the loop (for example a predicated vadd), it can be
4023 // cheaper for the select to remain in the loop than be sunk out of it,
4024 // and so use the select value for the phi instead of the old
4025 // LoopExitValue.
4026 RecurrenceDescriptor RdxDesc = Legal->getReductionVars()[Phi];
4027 if (PreferPredicatedReductionSelect ||
4028 TTI->preferPredicatedReductionSelect(
4029 RdxDesc.getRecurrenceBinOp(), Phi->getType(),
4030 TargetTransformInfo::ReductionFlags())) {
4031 auto *VecRdxPhi = cast<PHINode>(getOrCreateVectorValue(Phi, Part));
4032 VecRdxPhi->setIncomingValueForBlock(
4033 LI->getLoopFor(LoopVectorBody)->getLoopLatch(), Sel);
4034 }
4035 }
4036 }
4037
4038 // If the vector reduction can be performed in a smaller type, we truncate
4039 // then extend the loop exit value to enable InstCombine to evaluate the
4040 // entire expression in the smaller type.
4041 if (VF.isVector() && Phi->getType() != RdxDesc.getRecurrenceType()) {
4042 assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!")((!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!"
) ? static_cast<void> (0) : __assert_fail ("!IsInLoopReductionPhi && \"Unexpected truncated inloop reduction!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4042, __PRETTY_FUNCTION__))
;
4043 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4043, __PRETTY_FUNCTION__))
;
4044 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
4045 Builder.SetInsertPoint(
4046 LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
4047 VectorParts RdxParts(UF);
4048 for (unsigned Part = 0; Part < UF; ++Part) {
4049 RdxParts[Part] = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
4050 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
4051 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
4052 : Builder.CreateZExt(Trunc, VecTy);
4053 for (Value::user_iterator UI = RdxParts[Part]->user_begin();
4054 UI != RdxParts[Part]->user_end();)
4055 if (*UI != Trunc) {
4056 (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd);
4057 RdxParts[Part] = Extnd;
4058 } else {
4059 ++UI;
4060 }
4061 }
4062 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
4063 for (unsigned Part = 0; Part < UF; ++Part) {
4064 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
4065 VectorLoopValueMap.resetVectorValue(LoopExitInst, Part, RdxParts[Part]);
4066 }
4067 }
4068
4069 // Reduce all of the unrolled parts into a single vector.
4070 Value *ReducedPartRdx = VectorLoopValueMap.getVectorValue(LoopExitInst, 0);
4071 unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK);
4072
4073 // The middle block terminator has already been assigned a DebugLoc here (the
4074 // OrigLoop's single latch terminator). We want the whole middle block to
4075 // appear to execute on this line because: (a) it is all compiler generated,
4076 // (b) these instructions are always executed after evaluating the latch
4077 // conditional branch, and (c) other passes may add new predecessors which
4078 // terminate on this line. This is the easiest way to ensure we don't
4079 // accidentally cause an extra step back into the loop while debugging.
4080 setDebugLocFromInst(Builder, LoopMiddleBlock->getTerminator());
4081 for (unsigned Part = 1; Part < UF; ++Part) {
4082 Value *RdxPart = VectorLoopValueMap.getVectorValue(LoopExitInst, Part);
4083 if (Op != Instruction::ICmp && Op != Instruction::FCmp)
4084 // Floating point operations had to be 'fast' to enable the reduction.
4085 ReducedPartRdx = addFastMathFlag(
4086 Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxPart,
4087 ReducedPartRdx, "bin.rdx"),
4088 RdxDesc.getFastMathFlags());
4089 else
4090 ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
4091 RdxPart);
4092 }
4093
4094 // Create the reduction after the loop. Note that inloop reductions create the
4095 // target reduction in the loop using a Reduction recipe.
4096 if (VF.isVector() && !IsInLoopReductionPhi) {
4097 bool NoNaN = Legal->hasFunNoNaNAttr();
4098 ReducedPartRdx =
4099 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, NoNaN);
4100 // If the reduction can be performed in a smaller type, we need to extend
4101 // the reduction to the wider type before we branch to the original loop.
4102 if (Phi->getType() != RdxDesc.getRecurrenceType())
4103 ReducedPartRdx =
4104 RdxDesc.isSigned()
4105 ? Builder.CreateSExt(ReducedPartRdx, Phi->getType())
4106 : Builder.CreateZExt(ReducedPartRdx, Phi->getType());
4107 }
4108
4109 // Create a phi node that merges control-flow from the backedge-taken check
4110 // block and the middle block.
4111 PHINode *BCBlockPhi = PHINode::Create(Phi->getType(), 2, "bc.merge.rdx",
4112 LoopScalarPreHeader->getTerminator());
4113 for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
4114 BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
4115 BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);
4116
4117 // Now, we need to fix the users of the reduction variable
4118 // inside and outside of the scalar remainder loop.
4119 // We know that the loop is in LCSSA form. We need to update the
4120 // PHI nodes in the exit blocks.
4121 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
4122 // All PHINodes need to have a single entry edge, or two if
4123 // we already fixed them.
4124 assert(LCSSAPhi.getNumIncomingValues() < 3 && "Invalid LCSSA PHI")((LCSSAPhi.getNumIncomingValues() < 3 && "Invalid LCSSA PHI"
) ? static_cast<void> (0) : __assert_fail ("LCSSAPhi.getNumIncomingValues() < 3 && \"Invalid LCSSA PHI\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4124, __PRETTY_FUNCTION__))
;
4125
4126 // We found a reduction value exit-PHI. Update it with the
4127 // incoming bypass edge.
4128 if (LCSSAPhi.getIncomingValue(0) == LoopExitInst)
4129 LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);
4130 } // end of the LCSSA phi scan.
4131
4132 // Fix the scalar loop reduction variable with the incoming reduction sum
4133 // from the vector body and from the backedge value.
4134 int IncomingEdgeBlockIdx =
4135 Phi->getBasicBlockIndex(OrigLoop->getLoopLatch());
4136 assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index")((IncomingEdgeBlockIdx >= 0 && "Invalid block index"
) ? static_cast<void> (0) : __assert_fail ("IncomingEdgeBlockIdx >= 0 && \"Invalid block index\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4136, __PRETTY_FUNCTION__))
;
4137 // Pick the other block.
4138 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
4139 Phi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
4140 Phi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
4141}
4142
4143void InnerLoopVectorizer::clearReductionWrapFlags(
4144 RecurrenceDescriptor &RdxDesc) {
4145 RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind();
4146 if (RK != RecurrenceDescriptor::RK_IntegerAdd &&
4147 RK != RecurrenceDescriptor::RK_IntegerMult)
4148 return;
4149
4150 Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
4151 assert(LoopExitInstr && "null loop exit instruction")((LoopExitInstr && "null loop exit instruction") ? static_cast
<void> (0) : __assert_fail ("LoopExitInstr && \"null loop exit instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4151, __PRETTY_FUNCTION__))
;
4152 SmallVector<Instruction *, 8> Worklist;
4153 SmallPtrSet<Instruction *, 8> Visited;
4154 Worklist.push_back(LoopExitInstr);
4155 Visited.insert(LoopExitInstr);
4156
4157 while (!Worklist.empty()) {
4158 Instruction *Cur = Worklist.pop_back_val();
4159 if (isa<OverflowingBinaryOperator>(Cur))
4160 for (unsigned Part = 0; Part < UF; ++Part) {
4161 Value *V = getOrCreateVectorValue(Cur, Part);
4162 cast<Instruction>(V)->dropPoisonGeneratingFlags();
4163 }
4164
4165 for (User *U : Cur->users()) {
4166 Instruction *UI = cast<Instruction>(U);
4167 if ((Cur != LoopExitInstr || OrigLoop->contains(UI->getParent())) &&
4168 Visited.insert(UI).second)
4169 Worklist.push_back(UI);
4170 }
4171 }
4172}
4173
4174void InnerLoopVectorizer::fixLCSSAPHIs() {
4175 assert(!VF.isScalable() && "the code below assumes fixed width vectors")((!VF.isScalable() && "the code below assumes fixed width vectors"
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"the code below assumes fixed width vectors\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4175, __PRETTY_FUNCTION__))
;
4176 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
4177 if (LCSSAPhi.getNumIncomingValues() == 1) {
4178 auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
4179 // Non-instruction incoming values will have only one value.
4180 unsigned LastLane = 0;
4181 if (isa<Instruction>(IncomingValue))
4182 LastLane = Cost->isUniformAfterVectorization(
4183 cast<Instruction>(IncomingValue), VF)
4184 ? 0
4185 : VF.getKnownMinValue() - 1;
4186 // Can be a loop invariant incoming value or the last scalar value to be
4187 // extracted from the vectorized loop.
4188 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
4189 Value *lastIncomingValue =
4190 getOrCreateScalarValue(IncomingValue, { UF - 1, LastLane });
4191 LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock);
4192 }
4193 }
4194}
4195
4196void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
4197 // The basic block and loop containing the predicated instruction.
4198 auto *PredBB = PredInst->getParent();
4199 auto *VectorLoop = LI->getLoopFor(PredBB);
4200
4201 // Initialize a worklist with the operands of the predicated instruction.
4202 SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());
4203
4204 // Holds instructions that we need to analyze again. An instruction may be
4205 // reanalyzed if we don't yet know if we can sink it or not.
4206 SmallVector<Instruction *, 8> InstsToReanalyze;
4207
4208 // Returns true if a given use occurs in the predicated block. Phi nodes use
4209 // their operands in their corresponding predecessor blocks.
4210 auto isBlockOfUsePredicated = [&](Use &U) -> bool {
4211 auto *I = cast<Instruction>(U.getUser());
4212 BasicBlock *BB = I->getParent();
4213 if (auto *Phi = dyn_cast<PHINode>(I))
4214 BB = Phi->getIncomingBlock(
4215 PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
4216 return BB == PredBB;
4217 };
4218
4219 // Iteratively sink the scalarized operands of the predicated instruction
4220 // into the block we created for it. When an instruction is sunk, it's
4221 // operands are then added to the worklist. The algorithm ends after one pass
4222 // through the worklist doesn't sink a single instruction.
4223 bool Changed;
4224 do {
4225 // Add the instructions that need to be reanalyzed to the worklist, and
4226 // reset the changed indicator.
4227 Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());
4228 InstsToReanalyze.clear();
4229 Changed = false;
4230
4231 while (!Worklist.empty()) {
4232 auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());
4233
4234 // We can't sink an instruction if it is a phi node, is already in the
4235 // predicated block, is not in the loop, or may have side effects.
4236 if (!I || isa<PHINode>(I) || I->getParent() == PredBB ||
4237 !VectorLoop->contains(I) || I->mayHaveSideEffects())
4238 continue;
4239
4240 // It's legal to sink the instruction if all its uses occur in the
4241 // predicated block. Otherwise, there's nothing to do yet, and we may
4242 // need to reanalyze the instruction.
4243 if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) {
4244 InstsToReanalyze.push_back(I);
4245 continue;
4246 }
4247
4248 // Move the instruction to the beginning of the predicated block, and add
4249 // it's operands to the worklist.
4250 I->moveBefore(&*PredBB->getFirstInsertionPt());
4251 Worklist.insert(I->op_begin(), I->op_end());
4252
4253 // The sinking may have enabled other instructions to be sunk, so we will
4254 // need to iterate.
4255 Changed = true;
4256 }
4257 } while (Changed);
4258}
4259
4260void InnerLoopVectorizer::fixNonInductionPHIs() {
4261 for (PHINode *OrigPhi : OrigPHIsToFix) {
4262 PHINode *NewPhi =
4263 cast<PHINode>(VectorLoopValueMap.getVectorValue(OrigPhi, 0));
4264 unsigned NumIncomingValues = OrigPhi->getNumIncomingValues();
4265
4266 SmallVector<BasicBlock *, 2> ScalarBBPredecessors(
4267 predecessors(OrigPhi->getParent()));
4268 SmallVector<BasicBlock *, 2> VectorBBPredecessors(
4269 predecessors(NewPhi->getParent()));
4270 assert(ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&((ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&
"Scalar and Vector BB should have the same number of predecessors"
) ? static_cast<void> (0) : __assert_fail ("ScalarBBPredecessors.size() == VectorBBPredecessors.size() && \"Scalar and Vector BB should have the same number of predecessors\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4271, __PRETTY_FUNCTION__))
4271 "Scalar and Vector BB should have the same number of predecessors")((ScalarBBPredecessors.size() == VectorBBPredecessors.size() &&
"Scalar and Vector BB should have the same number of predecessors"
) ? static_cast<void> (0) : __assert_fail ("ScalarBBPredecessors.size() == VectorBBPredecessors.size() && \"Scalar and Vector BB should have the same number of predecessors\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4271, __PRETTY_FUNCTION__))
;
4272
4273 // The insertion point in Builder may be invalidated by the time we get
4274 // here. Force the Builder insertion point to something valid so that we do
4275 // not run into issues during insertion point restore in
4276 // getOrCreateVectorValue calls below.
4277 Builder.SetInsertPoint(NewPhi);
4278
4279 // The predecessor order is preserved and we can rely on mapping between
4280 // scalar and vector block predecessors.
4281 for (unsigned i = 0; i < NumIncomingValues; ++i) {
4282 BasicBlock *NewPredBB = VectorBBPredecessors[i];
4283
4284 // When looking up the new scalar/vector values to fix up, use incoming
4285 // values from original phi.
4286 Value *ScIncV =
4287 OrigPhi->getIncomingValueForBlock(ScalarBBPredecessors[i]);
4288
4289 // Scalar incoming value may need a broadcast
4290 Value *NewIncV = getOrCreateVectorValue(ScIncV, 0);
4291 NewPhi->addIncoming(NewIncV, NewPredBB);
4292 }
4293 }
4294}
4295
4296void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
4297 VPUser &Operands, unsigned UF,
4298 ElementCount VF, bool IsPtrLoopInvariant,
4299 SmallBitVector &IsIndexLoopInvariant,
4300 VPTransformState &State) {
4301 // Construct a vector GEP by widening the operands of the scalar GEP as
4302 // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
4303 // results in a vector of pointers when at least one operand of the GEP
4304 // is vector-typed. Thus, to keep the representation compact, we only use
4305 // vector-typed operands for loop-varying values.
4306
4307 if (VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
4308 // If we are vectorizing, but the GEP has only loop-invariant operands,
4309 // the GEP we build (by only using vector-typed operands for
4310 // loop-varying values) would be a scalar pointer. Thus, to ensure we
4311 // produce a vector of pointers, we need to either arbitrarily pick an
4312 // operand to broadcast, or broadcast a clone of the original GEP.
4313 // Here, we broadcast a clone of the original.
4314 //
4315 // TODO: If at some point we decide to scalarize instructions having
4316 // loop-invariant operands, this special case will no longer be
4317 // required. We would add the scalarization decision to
4318 // collectLoopScalars() and teach getVectorValue() to broadcast
4319 // the lane-zero scalar value.
4320 auto *Clone = Builder.Insert(GEP->clone());
4321 for (unsigned Part = 0; Part < UF; ++Part) {
4322 Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);
4323 VectorLoopValueMap.setVectorValue(GEP, Part, EntryPart);
4324 addMetadata(EntryPart, GEP);
4325 }
4326 } else {
4327 // If the GEP has at least one loop-varying operand, we are sure to
4328 // produce a vector of pointers. But if we are only unrolling, we want
4329 // to produce a scalar GEP for each unroll part. Thus, the GEP we
4330 // produce with the code below will be scalar (if VF == 1) or vector
4331 // (otherwise). Note that for the unroll-only case, we still maintain
4332 // values in the vector mapping with initVector, as we do for other
4333 // instructions.
4334 for (unsigned Part = 0; Part < UF; ++Part) {
4335 // The pointer operand of the new GEP. If it's loop-invariant, we
4336 // won't broadcast it.
4337 auto *Ptr = IsPtrLoopInvariant ? State.get(Operands.getOperand(0), {0, 0})
4338 : State.get(Operands.getOperand(0), Part);
4339
4340 // Collect all the indices for the new GEP. If any index is
4341 // loop-invariant, we won't broadcast it.
4342 SmallVector<Value *, 4> Indices;
4343 for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) {
4344 VPValue *Operand = Operands.getOperand(I);
4345 if (IsIndexLoopInvariant[I - 1])
4346 Indices.push_back(State.get(Operand, {0, 0}));
4347 else
4348 Indices.push_back(State.get(Operand, Part));
4349 }
4350
4351 // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
4352 // but it should be a vector, otherwise.
4353 auto *NewGEP =
4354 GEP->isInBounds()
4355 ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
4356 Indices)
4357 : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
4358 assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&(((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF.isScalar() || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4359, __PRETTY_FUNCTION__))
4359 "NewGEP is not a pointer vector")(((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
"NewGEP is not a pointer vector") ? static_cast<void> (
0) : __assert_fail ("(VF.isScalar() || NewGEP->getType()->isVectorTy()) && \"NewGEP is not a pointer vector\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4359, __PRETTY_FUNCTION__))
;
4360 VectorLoopValueMap.setVectorValue(GEP, Part, NewGEP);
4361 addMetadata(NewGEP, GEP);
4362 }
4363 }
4364}
4365
4366void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN, unsigned UF,
4367 ElementCount VF) {
4368 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4368, __PRETTY_FUNCTION__))
;
4369 PHINode *P = cast<PHINode>(PN);
4370 if (EnableVPlanNativePath) {
4371 // Currently we enter here in the VPlan-native path for non-induction
4372 // PHIs where all control flow is uniform. We simply widen these PHIs.
4373 // Create a vector phi with no operands - the vector phi operands will be
4374 // set at the end of vector code generation.
4375 Type *VecTy =
4376 (VF.isScalar()) ? PN->getType() : VectorType::get(PN->getType(), VF);
4377 Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
4378 VectorLoopValueMap.setVectorValue(P, 0, VecPhi);
4379 OrigPHIsToFix.push_back(P);
4380
4381 return;
4382 }
4383
4384 assert(PN->getParent() == OrigLoop->getHeader() &&((PN->getParent() == OrigLoop->getHeader() && "Non-header phis should have been handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4385, __PRETTY_FUNCTION__))
4385 "Non-header phis should have been handled elsewhere")((PN->getParent() == OrigLoop->getHeader() && "Non-header phis should have been handled elsewhere"
) ? static_cast<void> (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4385, __PRETTY_FUNCTION__))
;
4386
4387 // In order to support recurrences we need to be able to vectorize Phi nodes.
4388 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
4389 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
4390 // this value when we vectorize all of the instructions that use the PHI.
4391 if (Legal->isReductionVariable(P) || Legal->isFirstOrderRecurrence(P)) {
4392 for (unsigned Part = 0; Part < UF; ++Part) {
4393 // This is phase one of vectorizing PHIs.
4394 bool ScalarPHI =
4395 (VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
4396 Type *VecTy =
4397 ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), VF);
4398 Value *EntryPart = PHINode::Create(
4399 VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
4400 VectorLoopValueMap.setVectorValue(P, Part, EntryPart);
4401 }
4402 return;
4403 }
4404
4405 setDebugLocFromInst(Builder, P);
4406
4407 // This PHINode must be an induction variable.
4408 // Make sure that we know about it.
4409 assert(Legal->getInductionVars().count(P) && "Not an induction variable")((Legal->getInductionVars().count(P) && "Not an induction variable"
) ? static_cast<void> (0) : __assert_fail ("Legal->getInductionVars().count(P) && \"Not an induction variable\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4409, __PRETTY_FUNCTION__))
;
4410
4411 InductionDescriptor II = Legal->getInductionVars().lookup(P);
4412 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
4413
4414 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
4415 // which can be found from the original scalar operations.
4416 switch (II.getKind()) {
4417 case InductionDescriptor::IK_NoInduction:
4418 llvm_unreachable("Unknown induction")::llvm::llvm_unreachable_internal("Unknown induction", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4418)
;
4419 case InductionDescriptor::IK_IntInduction:
4420 case InductionDescriptor::IK_FpInduction:
4421 llvm_unreachable("Integer/fp induction is handled elsewhere.")::llvm::llvm_unreachable_internal("Integer/fp induction is handled elsewhere."
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4421)
;
4422 case InductionDescriptor::IK_PtrInduction: {
4423 // Handle the pointer induction variable case.
4424 assert(P->getType()->isPointerTy() && "Unexpected type.")((P->getType()->isPointerTy() && "Unexpected type."
) ? static_cast<void> (0) : __assert_fail ("P->getType()->isPointerTy() && \"Unexpected type.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4424, __PRETTY_FUNCTION__))
;
4425
4426 if (Cost->isScalarAfterVectorization(P, VF)) {
4427 // This is the normalized GEP that starts counting at zero.
4428 Value *PtrInd =
4429 Builder.CreateSExtOrTrunc(Induction, II.getStep()->getType());
4430 // Determine the number of scalars we need to generate for each unroll
4431 // iteration. If the instruction is uniform, we only need to generate the
4432 // first lane. Otherwise, we generate all VF values.
4433 unsigned Lanes =
4434 Cost->isUniformAfterVectorization(P, VF) ? 1 : VF.getKnownMinValue();
4435 for (unsigned Part = 0; Part < UF; ++Part) {
4436 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
4437 Constant *Idx = ConstantInt::get(PtrInd->getType(),
4438 Lane + Part * VF.getKnownMinValue());
4439 Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
4440 Value *SclrGep =
4441 emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
4442 SclrGep->setName("next.gep");
4443 VectorLoopValueMap.setScalarValue(P, {Part, Lane}, SclrGep);
4444 }
4445 }
4446 return;
4447 }
4448 assert(isa<SCEVConstant>(II.getStep()) &&((isa<SCEVConstant>(II.getStep()) && "Induction step not a SCEV constant!"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(II.getStep()) && \"Induction step not a SCEV constant!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4449, __PRETTY_FUNCTION__))
4449 "Induction step not a SCEV constant!")((isa<SCEVConstant>(II.getStep()) && "Induction step not a SCEV constant!"
) ? static_cast<void> (0) : __assert_fail ("isa<SCEVConstant>(II.getStep()) && \"Induction step not a SCEV constant!\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4449, __PRETTY_FUNCTION__))
;
4450 Type *PhiType = II.getStep()->getType();
4451
4452 // Build a pointer phi
4453 Value *ScalarStartValue = II.getStartValue();
4454 Type *ScStValueType = ScalarStartValue->getType();
4455 PHINode *NewPointerPhi =
4456 PHINode::Create(ScStValueType, 2, "pointer.phi", Induction);
4457 NewPointerPhi->addIncoming(ScalarStartValue, LoopVectorPreHeader);
4458
4459 // A pointer induction, performed by using a gep
4460 BasicBlock *LoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
4461 Instruction *InductionLoc = LoopLatch->getTerminator();
4462 const SCEV *ScalarStep = II.getStep();
4463 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
4464 Value *ScalarStepValue =
4465 Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
4466 Value *InductionGEP = GetElementPtrInst::Create(
4467 ScStValueType->getPointerElementType(), NewPointerPhi,
4468 Builder.CreateMul(
4469 ScalarStepValue,
4470 ConstantInt::get(PhiType, VF.getKnownMinValue() * UF)),
4471 "ptr.ind", InductionLoc);
4472 NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
4473
4474 // Create UF many actual address geps that use the pointer
4475 // phi as base and a vectorized version of the step value
4476 // (<step*0, ..., step*N>) as offset.
4477 for (unsigned Part = 0; Part < UF; ++Part) {
4478 SmallVector<Constant *, 8> Indices;
4479 // Create a vector of consecutive numbers from zero to VF.
4480 for (unsigned i = 0; i < VF.getKnownMinValue(); ++i)
4481 Indices.push_back(
4482 ConstantInt::get(PhiType, i + Part * VF.getKnownMinValue()));
4483 Constant *StartOffset = ConstantVector::get(Indices);
4484
4485 Value *GEP = Builder.CreateGEP(
4486 ScStValueType->getPointerElementType(), NewPointerPhi,
4487 Builder.CreateMul(
4488 StartOffset,
4489 Builder.CreateVectorSplat(VF.getKnownMinValue(), ScalarStepValue),
4490 "vector.gep"));
4491 VectorLoopValueMap.setVectorValue(P, Part, GEP);
4492 }
4493 }
4494 }
4495}
4496
4497/// A helper function for checking whether an integer division-related
4498/// instruction may divide by zero (in which case it must be predicated if
4499/// executed conditionally in the scalar code).
4500/// TODO: It may be worthwhile to generalize and check isKnownNonZero().
4501/// Non-zero divisors that are non compile-time constants will not be
4502/// converted into multiplication, so we will still end up scalarizing
4503/// the division, but can do so w/o predication.
4504static bool mayDivideByZero(Instruction &I) {
4505 assert((I.getOpcode() == Instruction::UDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4509, __PRETTY_FUNCTION__))
4506 I.getOpcode() == Instruction::SDiv ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4509, __PRETTY_FUNCTION__))
4507 I.getOpcode() == Instruction::URem ||(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4509, __PRETTY_FUNCTION__))
4508 I.getOpcode() == Instruction::SRem) &&(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4509, __PRETTY_FUNCTION__))
4509 "Unexpected instruction")(((I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction
::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode()
== Instruction::SRem) && "Unexpected instruction") ?
static_cast<void> (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4509, __PRETTY_FUNCTION__))
;
4510 Value *Divisor = I.getOperand(1);
4511 auto *CInt = dyn_cast<ConstantInt>(Divisor);
4512 return !CInt || CInt->isZero();
4513}
4514
4515void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User,
4516 VPTransformState &State) {
4517 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4517, __PRETTY_FUNCTION__))
;
4518 switch (I.getOpcode()) {
4519 case Instruction::Call:
4520 case Instruction::Br:
4521 case Instruction::PHI:
4522 case Instruction::GetElementPtr:
4523 case Instruction::Select:
4524 llvm_unreachable("This instruction is handled by a different recipe.")::llvm::llvm_unreachable_internal("This instruction is handled by a different recipe."
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4524)
;
4525 case Instruction::UDiv:
4526 case Instruction::SDiv:
4527 case Instruction::SRem:
4528 case Instruction::URem:
4529 case Instruction::Add:
4530 case Instruction::FAdd:
4531 case Instruction::Sub:
4532 case Instruction::FSub:
4533 case Instruction::FNeg:
4534 case Instruction::Mul:
4535 case Instruction::FMul:
4536 case Instruction::FDiv:
4537 case Instruction::FRem:
4538 case Instruction::Shl:
4539 case Instruction::LShr:
4540 case Instruction::AShr:
4541 case Instruction::And:
4542 case Instruction::Or:
4543 case Instruction::Xor: {
4544 // Just widen unops and binops.
4545 setDebugLocFromInst(Builder, &I);
4546
4547 for (unsigned Part = 0; Part < UF; ++Part) {
4548 SmallVector<Value *, 2> Ops;
4549 for (VPValue *VPOp : User.operands())
4550 Ops.push_back(State.get(VPOp, Part));
4551
4552 Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
4553
4554 if (auto *VecOp = dyn_cast<Instruction>(V))
4555 VecOp->copyIRFlags(&I);
4556
4557 // Use this vector value for all users of the original instruction.
4558 VectorLoopValueMap.setVectorValue(&I, Part, V);
4559 addMetadata(V, &I);
4560 }
4561
4562 break;
4563 }
4564 case Instruction::ICmp:
4565 case Instruction::FCmp: {
4566 // Widen compares. Generate vector compares.
4567 bool FCmp = (I.getOpcode() == Instruction::FCmp);
4568 auto *Cmp = cast<CmpInst>(&I);
4569 setDebugLocFromInst(Builder, Cmp);
4570 for (unsigned Part = 0; Part < UF; ++Part) {
4571 Value *A = State.get(User.getOperand(0), Part);
4572 Value *B = State.get(User.getOperand(1), Part);
4573 Value *C = nullptr;
4574 if (FCmp) {
4575 // Propagate fast math flags.
4576 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
4577 Builder.setFastMathFlags(Cmp->getFastMathFlags());
4578 C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
4579 } else {
4580 C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
4581 }
4582 VectorLoopValueMap.setVectorValue(&I, Part, C);
4583 addMetadata(C, &I);
4584 }
4585
4586 break;
4587 }
4588
4589 case Instruction::ZExt:
4590 case Instruction::SExt:
4591 case Instruction::FPToUI:
4592 case Instruction::FPToSI:
4593 case Instruction::FPExt:
4594 case Instruction::PtrToInt:
4595 case Instruction::IntToPtr:
4596 case Instruction::SIToFP:
4597 case Instruction::UIToFP:
4598 case Instruction::Trunc:
4599 case Instruction::FPTrunc:
4600 case Instruction::BitCast: {
4601 auto *CI = cast<CastInst>(&I);
4602 setDebugLocFromInst(Builder, CI);
4603
4604 /// Vectorize casts.
4605 assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4605, __PRETTY_FUNCTION__))
;
4606 Type *DestTy =
4607 (VF.isScalar()) ? CI->getType() : VectorType::get(CI->getType(), VF);
4608
4609 for (unsigned Part = 0; Part < UF; ++Part) {
4610 Value *A = State.get(User.getOperand(0), Part);
4611 Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
4612 VectorLoopValueMap.setVectorValue(&I, Part, Cast);
4613 addMetadata(Cast, &I);
4614 }
4615 break;
4616 }
4617 default:
4618 // This instruction is not vectorized by simple widening.
4619 LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found an unhandled instruction: "
<< I; } } while (false)
;
4620 llvm_unreachable("Unhandled instruction!")::llvm::llvm_unreachable_internal("Unhandled instruction!", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4620)
;
4621 } // end of switch.
4622}
4623
4624void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
4625 VPUser &ArgOperands,
4626 VPTransformState &State) {
4627 assert(!isa<DbgInfoIntrinsic>(I) &&((!isa<DbgInfoIntrinsic>(I) && "DbgInfoIntrinsic should have been dropped during VPlan construction"
) ? static_cast<void> (0) : __assert_fail ("!isa<DbgInfoIntrinsic>(I) && \"DbgInfoIntrinsic should have been dropped during VPlan construction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4628, __PRETTY_FUNCTION__))
4628 "DbgInfoIntrinsic should have been dropped during VPlan construction")((!isa<DbgInfoIntrinsic>(I) && "DbgInfoIntrinsic should have been dropped during VPlan construction"
) ? static_cast<void> (0) : __assert_fail ("!isa<DbgInfoIntrinsic>(I) && \"DbgInfoIntrinsic should have been dropped during VPlan construction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4628, __PRETTY_FUNCTION__))
;
4629 setDebugLocFromInst(Builder, &I);
4630
4631 Module *M = I.getParent()->getParent()->getParent();
4632 auto *CI = cast<CallInst>(&I);
4633
4634 SmallVector<Type *, 4> Tys;
4635 for (Value *ArgOperand : CI->arg_operands())
4636 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
4637
4638 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4639
4640 // The flag shows whether we use Intrinsic or a usual Call for vectorized
4641 // version of the instruction.
4642 // Is it beneficial to perform intrinsic call compared to lib call?
4643 bool NeedToScalarize = false;
4644 unsigned CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
4645 bool UseVectorIntrinsic =
4646 ID && Cost->getVectorIntrinsicCost(CI, VF) <= CallCost;
4647 assert((UseVectorIntrinsic || !NeedToScalarize) &&(((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."
) ? static_cast<void> (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4648, __PRETTY_FUNCTION__))
4648 "Instruction should be scalarized elsewhere.")(((UseVectorIntrinsic || !NeedToScalarize) && "Instruction should be scalarized elsewhere."
) ? static_cast<void> (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4648, __PRETTY_FUNCTION__))
;
4649
4650 for (unsigned Part = 0; Part < UF; ++Part) {
4651 SmallVector<Value *, 4> Args;
4652 for (auto &I : enumerate(ArgOperands.operands())) {
4653 // Some intrinsics have a scalar argument - don't replace it with a
4654 // vector.
4655 Value *Arg;
4656 if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
4657 Arg = State.get(I.value(), Part);
4658 else
4659 Arg = State.get(I.value(), {0, 0});
4660 Args.push_back(Arg);
4661 }
4662
4663 Function *VectorF;
4664 if (UseVectorIntrinsic) {
4665 // Use vector version of the intrinsic.
4666 Type *TysForDecl[] = {CI->getType()};
4667 if (VF.isVector()) {
4668 assert(!VF.isScalable() && "VF is assumed to be non scalable.")((!VF.isScalable() && "VF is assumed to be non scalable."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4668, __PRETTY_FUNCTION__))
;
4669 TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
4670 }
4671 VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
4672 assert(VectorF && "Can't retrieve vector intrinsic.")((VectorF && "Can't retrieve vector intrinsic.") ? static_cast
<void> (0) : __assert_fail ("VectorF && \"Can't retrieve vector intrinsic.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4672, __PRETTY_FUNCTION__))
;
4673 } else {
4674 // Use vector version of the function call.
4675 const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
4676#ifndef NDEBUG
4677 assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&((VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
"Can't create vector function.") ? static_cast<void> (
0) : __assert_fail ("VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && \"Can't create vector function.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4678, __PRETTY_FUNCTION__))
4678 "Can't create vector function.")((VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&
"Can't create vector function.") ? static_cast<void> (
0) : __assert_fail ("VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && \"Can't create vector function.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4678, __PRETTY_FUNCTION__))
;
4679#endif
4680 VectorF = VFDatabase(*CI).getVectorizedFunction(Shape);
4681 }
4682 SmallVector<OperandBundleDef, 1> OpBundles;
4683 CI->getOperandBundlesAsDefs(OpBundles);
4684 CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
4685
4686 if (isa<FPMathOperator>(V))
4687 V->copyFastMathFlags(CI);
4688
4689 State.set(Def, &I, V, Part);
4690 addMetadata(V, &I);
4691 }
4692}
4693
4694void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef,
4695 VPUser &Operands,
4696 bool InvariantCond,
4697 VPTransformState &State) {
4698 setDebugLocFromInst(Builder, &I);
4699
4700 // The condition can be loop invariant but still defined inside the
4701 // loop. This means that we can't just use the original 'cond' value.
4702 // We have to take the 'vectorized' value and pick the first lane.
4703 // Instcombine will make this a no-op.
4704 auto *InvarCond =
4705 InvariantCond ? State.get(Operands.getOperand(0), {0, 0}) : nullptr;
4706
4707 for (unsigned Part = 0; Part < UF; ++Part) {
4708 Value *Cond =
4709 InvarCond ? InvarCond : State.get(Operands.getOperand(0), Part);
4710 Value *Op0 = State.get(Operands.getOperand(1), Part);
4711 Value *Op1 = State.get(Operands.getOperand(2), Part);
4712 Value *Sel = Builder.CreateSelect(Cond, Op0, Op1);
4713 State.set(VPDef, &I, Sel, Part);
4714 addMetadata(Sel, &I);
4715 }
4716}
4717
4718void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
4719 // We should not collect Scalars more than once per VF. Right now, this
4720 // function is called from collectUniformsAndScalars(), which already does
4721 // this check. Collecting Scalars for VF=1 does not make any sense.
4722 assert(VF.isVector() && Scalars.find(VF) == Scalars.end() &&((VF.isVector() && Scalars.find(VF) == Scalars.end() &&
"This function should not be visited twice for the same VF")
? static_cast<void> (0) : __assert_fail ("VF.isVector() && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4723, __PRETTY_FUNCTION__))
4723 "This function should not be visited twice for the same VF")((VF.isVector() && Scalars.find(VF) == Scalars.end() &&
"This function should not be visited twice for the same VF")
? static_cast<void> (0) : __assert_fail ("VF.isVector() && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4723, __PRETTY_FUNCTION__))
;
4724
4725 SmallSetVector<Instruction *, 8> Worklist;
4726
4727 // These sets are used to seed the analysis with pointers used by memory
4728 // accesses that will remain scalar.
4729 SmallSetVector<Instruction *, 8> ScalarPtrs;
4730 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
4731 auto *Latch = TheLoop->getLoopLatch();
4732
4733 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
4734 // The pointer operands of loads and stores will be scalar as long as the
4735 // memory access is not a gather or scatter operation. The value operand of a
4736 // store will remain scalar if the store is scalarized.
4737 auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
4738 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
4739 assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4740, __PRETTY_FUNCTION__))
4740 "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4740, __PRETTY_FUNCTION__))
;
4741 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
4742 if (Ptr == Store->getValueOperand())
4743 return WideningDecision == CM_Scalarize;
4744 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&((Ptr == getLoadStorePointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4745, __PRETTY_FUNCTION__))
4745 "Ptr is neither a value or pointer operand")((Ptr == getLoadStorePointerOperand(MemAccess) && "Ptr is neither a value or pointer operand"
) ? static_cast<void> (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4745, __PRETTY_FUNCTION__))
;
4746 return WideningDecision != CM_GatherScatter;
4747 };
4748
4749 // A helper that returns true if the given value is a bitcast or
4750 // getelementptr instruction contained in the loop.
4751 auto isLoopVaryingBitCastOrGEP = [&](Value *V) {
4752 return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||
4753 isa<GetElementPtrInst>(V)) &&
4754 !TheLoop->isLoopInvariant(V);
4755 };
4756
4757 auto isScalarPtrInduction = [&](Instruction *MemAccess, Value *Ptr) {
4758 if (!isa<PHINode>(Ptr) ||
4759 !Legal->getInductionVars().count(cast<PHINode>(Ptr)))
4760 return false;
4761 auto &Induction = Legal->getInductionVars()[cast<PHINode>(Ptr)];
4762 if (Induction.getKind() != InductionDescriptor::IK_PtrInduction)
4763 return false;
4764 return isScalarUse(MemAccess, Ptr);
4765 };
4766
4767 // A helper that evaluates a memory access's use of a pointer. If the
4768 // pointer is actually the pointer induction of a loop, it is being
4769 // inserted into Worklist. If the use will be a scalar use, and the
4770 // pointer is only used by memory accesses, we place the pointer in
4771 // ScalarPtrs. Otherwise, the pointer is placed in PossibleNonScalarPtrs.
4772 auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
4773 if (isScalarPtrInduction(MemAccess, Ptr)) {
4774 Worklist.insert(cast<Instruction>(Ptr));
4775 Instruction *Update = cast<Instruction>(
4776 cast<PHINode>(Ptr)->getIncomingValueForBlock(Latch));
4777 Worklist.insert(Update);
4778 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Ptrdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
<< *Ptr << "\n"; } } while (false)
4779 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
<< *Ptr << "\n"; } } while (false)
;
4780 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Updatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
<< *Update << "\n"; } } while (false)
4781 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found new scalar instruction: "
<< *Update << "\n"; } } while (false)
;
4782 return;
4783 }
4784 // We only care about bitcast and getelementptr instructions contained in
4785 // the loop.
4786 if (!isLoopVaryingBitCastOrGEP(Ptr))
4787 return;
4788
4789 // If the pointer has already been identified as scalar (e.g., if it was
4790 // also identified as uniform), there's nothing to do.
4791 auto *I = cast<Instruction>(Ptr);
4792 if (Worklist.count(I))
4793 return;
4794
4795 // If the use of the pointer will be a scalar use, and all users of the
4796 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
4797 // place the pointer in PossibleNonScalarPtrs.
4798 if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
4799 return isa<LoadInst>(U) || isa<StoreInst>(U);
4800 }))
4801 ScalarPtrs.insert(I);
4802 else
4803 PossibleNonScalarPtrs.insert(I);
4804 };
4805
4806 // We seed the scalars analysis with three classes of instructions: (1)
4807 // instructions marked uniform-after-vectorization and (2) bitcast,
4808 // getelementptr and (pointer) phi instructions used by memory accesses
4809 // requiring a scalar use.
4810 //
4811 // (1) Add to the worklist all instructions that have been identified as
4812 // uniform-after-vectorization.
4813 Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());
4814
4815 // (2) Add to the worklist all bitcast and getelementptr instructions used by
4816 // memory accesses requiring a scalar use. The pointer operands of loads and
4817 // stores will be scalar as long as the memory accesses is not a gather or
4818 // scatter operation. The value operand of a store will remain scalar if the
4819 // store is scalarized.
4820 for (auto *BB : TheLoop->blocks())
4821 for (auto &I : *BB) {
4822 if (auto *Load = dyn_cast<LoadInst>(&I)) {
4823 evaluatePtrUse(Load, Load->getPointerOperand());
4824 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
4825 evaluatePtrUse(Store, Store->getPointerOperand());
4826 evaluatePtrUse(Store, Store->getValueOperand());
4827 }
4828 }
4829 for (auto *I : ScalarPtrs)
4830 if (!PossibleNonScalarPtrs.count(I)) {
4831 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *I << "\n"; } } while (false)
;
4832 Worklist.insert(I);
4833 }
4834
4835 // Insert the forced scalars.
4836 // FIXME: Currently widenPHIInstruction() often creates a dead vector
4837 // induction variable when the PHI user is scalarized.
4838 auto ForcedScalar = ForcedScalars.find(VF);
4839 if (ForcedScalar != ForcedScalars.end())
4840 for (auto *I : ForcedScalar->second)
4841 Worklist.insert(I);
4842
4843 // Expand the worklist by looking through any bitcasts and getelementptr
4844 // instructions we've already identified as scalar. This is similar to the
4845 // expansion step in collectLoopUniforms(); however, here we're only
4846 // expanding to include additional bitcasts and getelementptr instructions.
4847 unsigned Idx = 0;
4848 while (Idx != Worklist.size()) {
4849 Instruction *Dst = Worklist[Idx++];
4850 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
4851 continue;
4852 auto *Src = cast<Instruction>(Dst->getOperand(0));
4853 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
4854 auto *J = cast<Instruction>(U);
4855 return !TheLoop->contains(J) || Worklist.count(J) ||
4856 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
4857 isScalarUse(J, Src));
4858 })) {
4859 Worklist.insert(Src);
4860 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Src << "\n"; } } while (false)
;
4861 }
4862 }
4863
4864 // An induction variable will remain scalar if all users of the induction
4865 // variable and induction variable update remain scalar.
4866 for (auto &Induction : Legal->getInductionVars()) {
4867 auto *Ind = Induction.first;
4868 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4869
4870 // If tail-folding is applied, the primary induction variable will be used
4871 // to feed a vector compare.
4872 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
4873 continue;
4874
4875 // Determine if all users of the induction variable are scalar after
4876 // vectorization.
4877 auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
4878 auto *I = cast<Instruction>(U);
4879 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I);
4880 });
4881 if (!ScalarInd)
4882 continue;
4883
4884 // Determine if all users of the induction variable update instruction are
4885 // scalar after vectorization.
4886 auto ScalarIndUpdate =
4887 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
4888 auto *I = cast<Instruction>(U);
4889 return I == Ind || !TheLoop->contains(I) || Worklist.count(I);
4890 });
4891 if (!ScalarIndUpdate)
4892 continue;
4893
4894 // The induction variable and its update instruction will remain scalar.
4895 Worklist.insert(Ind);
4896 Worklist.insert(IndUpdate);
4897 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Ind << "\n"; } } while (false)
;
4898 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
4899 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
;
4900 }
4901
4902 Scalars[VF].insert(Worklist.begin(), Worklist.end());
4903}
4904
4905bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
4906 ElementCount VF) {
4907 assert(!VF.isScalable() && "scalable vectors not yet supported.")((!VF.isScalable() && "scalable vectors not yet supported."
) ? static_cast<void> (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4907, __PRETTY_FUNCTION__))
;
4908 if (!blockNeedsPredication(I->getParent()))
4909 return false;
4910 switch(I->getOpcode()) {
4911 default:
4912 break;
4913 case Instruction::Load:
4914 case Instruction::Store: {
4915 if (!Legal->isMaskRequired(I))
4916 return false;
4917 auto *Ptr = getLoadStorePointerOperand(I);
4918 auto *Ty = getMemInstValueType(I);
4919 // We have already decided how to vectorize this instruction, get that
4920 // result.
4921 if (VF.isVector()) {
4922 InstWidening WideningDecision = getWideningDecision(I, VF);
4923 assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4924, __PRETTY_FUNCTION__))
4924 "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4924, __PRETTY_FUNCTION__))
;
4925 return WideningDecision == CM_Scalarize;
4926 }
4927 const Align Alignment = getLoadStoreAlignment(I);
4928 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
4929 isLegalMaskedGather(Ty, Alignment))
4930 : !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
4931 isLegalMaskedScatter(Ty, Alignment));
4932 }
4933 case Instruction::UDiv:
4934 case Instruction::SDiv:
4935 case Instruction::SRem:
4936 case Instruction::URem:
4937 return mayDivideByZero(*I);
4938 }
4939 return false;
4940}
4941
4942bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
4943 Instruction *I, ElementCount VF) {
4944 assert(isAccessInterleaved(I) && "Expecting interleaved access.")((isAccessInterleaved(I) && "Expecting interleaved access."
) ? static_cast<void> (0) : __assert_fail ("isAccessInterleaved(I) && \"Expecting interleaved access.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4944, __PRETTY_FUNCTION__))
;
4945 assert(getWideningDecision(I, VF) == CM_Unknown &&((getWideningDecision(I, VF) == CM_Unknown && "Decision should not be set yet."
) ? static_cast<void> (0) : __assert_fail ("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4946, __PRETTY_FUNCTION__))
4946 "Decision should not be set yet.")((getWideningDecision(I, VF) == CM_Unknown && "Decision should not be set yet."
) ? static_cast<void> (0) : __assert_fail ("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4946, __PRETTY_FUNCTION__))
;
4947 auto *Group = getInterleavedAccessGroup(I);
4948 assert(Group && "Must have a group.")((Group && "Must have a group.") ? static_cast<void
> (0) : __assert_fail ("Group && \"Must have a group.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4948, __PRETTY_FUNCTION__))
;
4949
4950 // If the instruction's allocated size doesn't equal it's type size, it
4951 // requires padding and will be scalarized.
4952 auto &DL = I->getModule()->getDataLayout();
4953 auto *ScalarTy = getMemInstValueType(I);
4954 if (hasIrregularType(ScalarTy, DL, VF))
4955 return false;
4956
4957 // Check if masking is required.
4958 // A Group may need masking for one of two reasons: it resides in a block that
4959 // needs predication, or it was decided to use masking to deal with gaps.
4960 bool PredicatedAccessRequiresMasking =
4961 Legal->blockNeedsPredication(I->getParent()) && Legal->isMaskRequired(I);
4962 bool AccessWithGapsRequiresMasking =
4963 Group->requiresScalarEpilogue() && !isScalarEpilogueAllowed();
4964 if (!PredicatedAccessRequiresMasking && !AccessWithGapsRequiresMasking)
4965 return true;
4966
4967 // If masked interleaving is required, we expect that the user/target had
4968 // enabled it, because otherwise it either wouldn't have been created or
4969 // it should have been invalidated by the CostModel.
4970 assert(useMaskedInterleavedAccesses(TTI) &&((useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4971, __PRETTY_FUNCTION__))
4971 "Masked interleave-groups for predicated accesses are not enabled.")((useMaskedInterleavedAccesses(TTI) && "Masked interleave-groups for predicated accesses are not enabled."
) ? static_cast<void> (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4971, __PRETTY_FUNCTION__))
;
4972
4973 auto *Ty = getMemInstValueType(I);
4974 const Align Alignment = getLoadStoreAlignment(I);
4975 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
4976 : TTI.isLegalMaskedStore(Ty, Alignment);
4977}
4978
4979bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
4980 Instruction *I, ElementCount VF) {
4981 // Get and ensure we have a valid memory instruction.
4982 LoadInst *LI = dyn_cast<LoadInst>(I);
4983 StoreInst *SI = dyn_cast<StoreInst>(I);
4984 assert((LI || SI) && "Invalid memory instruction")(((LI || SI) && "Invalid memory instruction") ? static_cast
<void> (0) : __assert_fail ("(LI || SI) && \"Invalid memory instruction\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4984, __PRETTY_FUNCTION__))
;
4985
4986 auto *Ptr = getLoadStorePointerOperand(I);
4987
4988 // In order to be widened, the pointer should be consecutive, first of all.
4989 if (!Legal->isConsecutivePtr(Ptr))
4990 return false;
4991
4992 // If the instruction is a store located in a predicated block, it will be
4993 // scalarized.
4994 if (isScalarWithPredication(I))
4995 return false;
4996
4997 // If the instruction's allocated size doesn't equal it's type size, it
4998 // requires padding and will be scalarized.
4999 auto &DL = I->getModule()->getDataLayout();
5000 auto *ScalarTy = LI ? LI->getType() : SI->getValueOperand()->getType();
5001 if (hasIrregularType(ScalarTy, DL, VF))
5002 return false;
5003
5004 return true;
5005}
5006
5007void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
5008 // We should not collect Uniforms more than once per VF. Right now,
5009 // this function is called from collectUniformsAndScalars(), which
5010 // already does this check. Collecting Uniforms for VF=1 does not make any
5011 // sense.
5012
5013 assert(VF.isVector() && Uniforms.find(VF) == Uniforms.end() &&((VF.isVector() && Uniforms.find(VF) == Uniforms.end(
) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5014, __PRETTY_FUNCTION__))
5014 "This function should not be visited twice for the same VF")((VF.isVector() && Uniforms.find(VF) == Uniforms.end(
) && "This function should not be visited twice for the same VF"
) ? static_cast<void> (0) : __assert_fail ("VF.isVector() && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5014, __PRETTY_FUNCTION__))
;
5015
5016 // Visit the list of Uniforms. If we'll not find any uniform value, we'll
5017 // not analyze again. Uniforms.count(VF) will return 1.
5018 Uniforms[VF].clear();
5019
5020 // We now know that the loop is vectorizable!
5021 // Collect instructions inside the loop that will remain uniform after
5022 // vectorization.
5023
5024 // Global values, params and instructions outside of current loop are out of
5025 // scope.
5026 auto isOutOfScope = [&](Value *V) -> bool {
5027 Instruction *I = dyn_cast<Instruction>(V);
5028 return (!I || !TheLoop->contains(I));
5029 };
5030
5031 SetVector<Instruction *> Worklist;
5032 BasicBlock *Latch = TheLoop->getLoopLatch();
5033
5034 // Instructions that are scalar with predication must not be considered
5035 // uniform after vectorization, because that would create an erroneous
5036 // replicating region where only a single instance out of VF should be formed.
5037 // TODO: optimize such seldom cases if found important, see PR40816.
5038 auto addToWorklistIfAllowed = [&](Instruction *I) -> void {
5039 if (isOutOfScope(I)) {
5040 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform due to scope: "
<< *I << "\n"; } } while (false)
5041 << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform due to scope: "
<< *I << "\n"; } } while (false)
;
5042 return;
5043 }
5044 if (isScalarWithPredication(I, VF)) {
5045 LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform being ScalarWithPredication: "
<< *I << "\n"; } } while (false)
5046 << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform being ScalarWithPredication: "
<< *I << "\n"; } } while (false)
;
5047 return;
5048 }
5049 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *I << "\n"; } } while (false)
;
5050 Worklist.insert(I);
5051 };
5052
5053 // Start with the conditional branch. If the branch condition is an
5054 // instruction contained in the loop that is only used by the branch, it is
5055 // uniform.
5056 auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));
5057 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
5058 addToWorklistIfAllowed(Cmp);
5059
5060 // Holds consecutive and consecutive-like pointers. Consecutive-like pointers
5061 // are pointers that are treated like consecutive pointers during
5062 // vectorization. The pointer operands of interleaved accesses are an
5063 // example.
5064 SmallSetVector<Value *, 8> ConsecutiveLikePtrs;
5065
5066 // Holds pointer operands of instructions that are possibly non-uniform.
5067 SmallPtrSet<Value *, 8> PossibleNonUniformPtrs;
5068
5069 auto isUniformDecision = [&](Instruction *I, ElementCount VF) {
5070 InstWidening WideningDecision = getWideningDecision(I, VF);
5071 assert(WideningDecision != CM_Unknown &&((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5072, __PRETTY_FUNCTION__))
5072 "Widening decision should be ready at this moment")((WideningDecision != CM_Unknown && "Widening decision should be ready at this moment"
) ? static_cast<void> (0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5072, __PRETTY_FUNCTION__))
;
5073
5074 // The address of a uniform mem op is itself uniform. We exclude stores
5075 // here as there's an assumption in the current code that all uses of
5076 // uniform instructions are uniform and, as noted below, uniform stores are
5077 // still handled via replication (i.e. aren't uniform after vectorization).
5078 if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) {
5079 assert(WideningDecision == CM_Scalarize)((WideningDecision == CM_Scalarize) ? static_cast<void>
(0) : __assert_fail ("WideningDecision == CM_Scalarize", "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5079, __PRETTY_FUNCTION__))
;
5080 return true;
5081 }
5082
5083 return (WideningDecision == CM_Widen ||
5084 WideningDecision == CM_Widen_Reverse ||
5085 WideningDecision == CM_Interleave);
5086 };
5087
5088
5089 // Returns true if Ptr is the pointer operand of a memory access instruction
5090 // I, and I is known to not require scalarization.
5091 auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
5092 return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
5093 };
5094
5095 // Iterate over the instructions in the loop, and collect all
5096 // consecutive-like pointer operands in ConsecutiveLikePtrs. If it's possible
5097 // that a consecutive-like pointer operand will be scalarized, we collect it
5098 // in PossibleNonUniformPtrs instead. We use two sets here because a single
5099 // getelementptr instruction can be used by both vectorized and scalarized
5100 // memory instructions. For example, if a loop loads and stores from the same
5101 // location, but the store is conditional, the store will be scalarized, and
5102 // the getelementptr won't remain uniform.
5103 for (auto *BB : TheLoop->blocks())
5104 for (auto &I : *BB) {
5105 // If there's no pointer operand, there's nothing to do.
5106 auto *Ptr = getLoadStorePointerOperand(&I);
5107 if (!Ptr)
5108 continue;
5109
5110 // For now, avoid walking use lists in other functions.
5111 // TODO: Rewrite this algorithm from uses up.
5112 if (!isa<Instruction>(Ptr) && !isa<Argument>(Ptr))
5113 continue;
5114
5115 // A uniform memory op is itself uniform. We exclude stores here as we
5116 // haven't yet added dedicated logic in the CLONE path and rely on
5117 // REPLICATE + DSE for correctness.
5118 if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))
5119 addToWorklistIfAllowed(&I);
5120
5121 // True if all users of Ptr are memory accesses that have Ptr as their
5122 // pointer operand.
5123 auto UsersAreMemAccesses =
5124 llvm::all_of(Ptr->users(), [&](User *U) -> bool {
5125 return getLoadStorePointerOperand(U) == Ptr;
5126 });
5127
5128 // Ensure the memory instruction will not be scalarized or used by
5129 // gather/scatter, making its pointer operand non-uniform. If the pointer
5130 // operand is used by any instruction other than a memory access, we
5131 // conservatively assume the pointer operand may be non-uniform.
5132 if (!UsersAreMemAccesses || !isUniformDecision(&I, VF))
5133 PossibleNonUniformPtrs.insert(Ptr);
5134
5135 // If the memory instruction will be vectorized and its pointer operand
5136 // is consecutive-like, or interleaving - the pointer operand should
5137 // remain uniform.
5138 else
5139 ConsecutiveLikePtrs.insert(Ptr);
5140 }
5141
5142 // Add to the Worklist all consecutive and consecutive-like pointers that
5143 // aren't also identified as possibly non-uniform.
5144 for (auto *V : ConsecutiveLikePtrs)
5145 if (!PossibleNonUniformPtrs.count(V))
5146 if (auto *I = dyn_cast<Instruction>(V))
5147 addToWorklistIfAllowed(I);
5148
5149 // Expand Worklist in topological order: whenever a new instruction
5150 // is added , its users should be already inside Worklist. It ensures
5151 // a uniform instruction will only be used by uniform instructions.
5152 unsigned idx = 0;
5153 while (idx != Worklist.size()) {
5154 Instruction *I = Worklist[idx++];
5155
5156 for (auto OV : I->operand_values()) {
5157 // isOutOfScope operands cannot be uniform instructions.
5158 if (isOutOfScope(OV))
5159 continue;
5160 // First order recurrence Phi's should typically be considered
5161 // non-uniform.
5162 auto *OP = dyn_cast<PHINode>(OV);
5163 if (OP && Legal->isFirstOrderRecurrence(OP))
5164 continue;
5165 // If all the users of the operand are uniform, then add the
5166 // operand into the uniform worklist.
5167 auto *OI = cast<Instruction>(OV);
5168 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
5169 auto *J = cast<Instruction>(U);
5170 return Worklist.count(J) || isVectorizedMemAccessUse(J, OI);
5171 }))
5172 addToWorklistIfAllowed(OI);
5173 }
5174 }
5175
5176 // For an instruction to be added into Worklist above, all its users inside
5177 // the loop should also be in Worklist. However, this condition cannot be
5178 // true for phi nodes that form a cyclic dependence. We must process phi
5179 // nodes separately. An induction variable will remain uniform if all users
5180 // of the induction variable and induction variable update remain uniform.
5181 // The code below handles both pointer and non-pointer induction variables.
5182 for (auto &Induction : Legal->getInductionVars()) {
5183 auto *Ind = Induction.first;
5184 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
5185
5186 // Determine if all users of the induction variable are uniform after
5187 // vectorization.
5188 auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
5189 auto *I = cast<Instruction>(U);
5190 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
5191 isVectorizedMemAccessUse(I, Ind);
5192 });
5193 if (!UniformInd)
5194 continue;
5195
5196 // Determine if all users of the induction variable update instruction are
5197 // uniform after vectorization.
5198 auto UniformIndUpdate =
5199 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
5200 auto *I = cast<Instruction>(U);
5201 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
5202 isVectorizedMemAccessUse(I, IndUpdate);
5203 });
5204 if (!UniformIndUpdate)
5205 continue;
5206
5207 // The induction variable and its update instruction will remain uniform.
5208 addToWorklistIfAllowed(Ind);
5209 addToWorklistIfAllowed(IndUpdate);
5210 }
5211
5212 Uniforms[VF].insert(Worklist.begin(), Worklist.end());
5213}
5214
5215bool LoopVectorizationCostModel::runtimeChecksRequired() {
5216 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Performing code size checks.\n"
; } } while (false)
;
5217
5218 if (Legal->getRuntimePointerChecking()->Need) {
5219 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
5220 "runtime pointer checks needed. Enable vectorization of this "
5221 "loop with '#pragma clang loop vectorize(enable)' when "
5222 "compiling with -Os/-Oz",
5223 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5224 return true;
5225 }
5226
5227 if (!PSE.getUnionPredicate().getPredicates().empty()) {
5228 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
5229 "runtime SCEV checks needed. Enable vectorization of this "
5230 "loop with '#pragma clang loop vectorize(enable)' when "
5231 "compiling with -Os/-Oz",
5232 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5233 return true;
5234 }
5235
5236 // FIXME: Avoid specializing for stride==1 instead of bailing out.
5237 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
5238 reportVectorizationFailure("Runtime stride check for small trip count",
5239 "runtime stride == 1 checks needed. Enable vectorization of "
5240 "this loop without such check by compiling with -Os/-Oz",
5241 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5242 return true;
5243 }
5244
5245 return false;
5246}
5247
5248Optional<ElementCount>
5249LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
5250 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
5251 // TODO: It may by useful to do since it's still likely to be dynamically
5252 // uniform if the target can skip.
5253 reportVectorizationFailure(
5254 "Not inserting runtime ptr check for divergent target",
5255 "runtime pointer checks needed. Not enabled for divergent target",
5256 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
5257 return None;
5258 }
5259
5260 unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
5261 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found trip count: "
<< TC << '\n'; } } while (false)
;
5262 if (TC == 1) {
5263 reportVectorizationFailure("Single iteration (non) loop",
5264 "loop trip count is one, irrelevant for vectorization",
5265 "SingleIterationLoop", ORE, TheLoop);
5266 return None;
5267 }
5268
5269 switch (ScalarEpilogueStatus) {
5270 case CM_ScalarEpilogueAllowed:
5271 return UserVF ? UserVF : computeFeasibleMaxVF(TC);
5272 case CM_ScalarEpilogueNotNeededUsePredicate:
5273 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
5274 dbgs() << "LV: vector predicate hint/switch found.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
5275 << "LV: Not allowing scalar epilogue, creating predicated "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
5276 << "vector loop.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
;
5277 break;
5278 case CM_ScalarEpilogueNotAllowedLowTripLoop:
5279 // fallthrough as a special case of OptForSize
5280 case CM_ScalarEpilogueNotAllowedOptSize:
5281 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
5282 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false)
5283 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false)
;
5284 else
5285 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to low trip "
<< "count.\n"; } } while (false)
5286 << "count.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to low trip "
<< "count.\n"; } } while (false)
;
5287
5288 // Bail if runtime checks are required, which are not good when optimising
5289 // for size.
5290 if (runtimeChecksRequired())
5291 return None;
5292 break;
5293 }
5294
5295 // Now try the tail folding
5296
5297 // Invalidate interleave groups that require an epilogue if we can't mask
5298 // the interleave-group.
5299 if (!useMaskedInterleavedAccesses(TTI)) {
5300 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&((WideningDecisions.empty() && Uniforms.empty() &&
Scalars.empty() && "No decisions should have been taken at this point"
) ? static_cast<void> (0) : __assert_fail ("WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && \"No decisions should have been taken at this point\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5301, __PRETTY_FUNCTION__))
5301 "No decisions should have been taken at this point")((WideningDecisions.empty() && Uniforms.empty() &&
Scalars.empty() && "No decisions should have been taken at this point"
) ? static_cast<void> (0) : __assert_fail ("WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && \"No decisions should have been taken at this point\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5301, __PRETTY_FUNCTION__))
;
5302 // Note: There is no need to invalidate any cost modeling decisions here, as
5303 // non where taken so far.
5304 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
5305 }
5306
5307 ElementCount MaxVF = UserVF ? UserVF : computeFeasibleMaxVF(TC);
5308 assert(!MaxVF.isScalable() &&((!MaxVF.isScalable() && "Scalable vectors do not yet support tail folding"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"Scalable vectors do not yet support tail folding\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5309, __PRETTY_FUNCTION__))
5309 "Scalable vectors do not yet support tail folding")((!MaxVF.isScalable() && "Scalable vectors do not yet support tail folding"
) ? static_cast<void> (0) : __assert_fail ("!MaxVF.isScalable() && \"Scalable vectors do not yet support tail folding\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5309, __PRETTY_FUNCTION__))
;
5310 assert((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) &&(((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())
) && "MaxVF must be a power of 2") ? static_cast<void
> (0) : __assert_fail ("(UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) && \"MaxVF must be a power of 2\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5311, __PRETTY_FUNCTION__))
5311 "MaxVF must be a power of 2")(((UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())
) && "MaxVF must be a power of 2") ? static_cast<void
> (0) : __assert_fail ("(UserVF.isNonZero() || isPowerOf2_32(MaxVF.getFixedValue())) && \"MaxVF must be a power of 2\""
, "/build/llvm-toolchain-snapshot-12~++20201124111112+7b5254223ac/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 5311, __PRETTY_FUNCTION__))
;
5312 unsigned MaxVFtimesIC =
5313 UserIC ? MaxVF.getFixedValue() * UserIC : MaxVF.getFixedValue();
5314 if (TC > 0 && TC % MaxVFtimesIC == 0) {
5315 // Accept MaxVF if we do not have a tail.
5316 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No tail will remain for any chosen VF.\n"
; } } while (false)
;
5317 return MaxVF;
5318 }
5319
5320 // If we don't know the precise trip count, or if the trip count that we
5321 // found modulo the vectorization factor is not zero, try to fold the tail
5322 // by masking.
5323 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
5324 if (Legal->prepareToFoldTailByMasking()) {
5325 FoldTailByMasking = true;
5326 return MaxVF;
5327 }
5328
5329 // If there was a tail-folding hint/switch, but we can't fold the tail by
5330 // masking, fallback to a vectorization with a scalar epilogue.
5331 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
5332 if (PreferPredicateOverEpilogue == PreferPredicateTy::PredicateOrDontVectorize) {
5333 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't fold tail by masking: don't vectorize\n"
; } } while (false)
;
5334 return None;
5335 }
5336 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n"; } } while (false)
5337 "scalar epilogue instead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n"; } } while (false)
;
5338 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
5339 return MaxVF;
5340 }
5341
5342 if (TC == 0) {
5343 reportVectorizationFailure(
5344 "Unable to calculate the loop count due to complex control flow",
5345 "unable to calculate the loop count due to complex control flow",
5346 "UnknownLoopCountComplexCFG", ORE, TheLoop);
5347 return None;
5348 }
5349
5350 reportVectorizationFailure(
5351 "Cannot optimize for size and vectorize at the same time.",
5352 "cannot optimize for size and vectorize at the same time. "
5353 "Enable vectorization of this loop with '#pragma clang loop "
5354 "vectorize(enable)' when compiling with -Os/-Oz",
5355 "NoTailLoopWithOptForSize", ORE, TheLoop);
5356 return None;
5357}
5358
5359ElementCount
5360LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount) {
5361 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
5362 unsigned SmallestType, WidestType;
5363 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
5364 unsigned WidestRegister = TTI.getRegisterBitWidth(true);
5365
5366 // Get the maximum safe dependence distance in bits computed by LAA.
5367 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
5368 // the memory accesses that is most restrictive (involved in the smallest
5369 // dependence distance).
5370 unsigned MaxSafeRegisterWidth = Legal->getMaxSafeRegisterWidth();
5371
5372 WidestRegister = std::min(WidestRegister, MaxSafeRegisterWidth);
5373
5374 // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
5375 // Note that both WidestRegister and WidestType may not be a powers of 2.
5376 unsigned MaxVectorSize = PowerOf2Floor(WidestRegister / WidestType);
5377
5378 LLVM_DEBU