Bug Summary

File:build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Warning:line 2907, column 23
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name LoopVectorize.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/build-llvm -resource-dir /usr/lib/llvm-15/lib/clang/15.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/llvm/lib/Transforms/Vectorize -I include -I /build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-15/lib/clang/15.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-03-19-125827-35351-1 -x c++ /build/llvm-toolchain-snapshot-15~++20220319111418+752e9cdbb06a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/Proposal/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanHCFGBuilder.h"
61#include "VPlanPredicator.h"
62#include "VPlanTransforms.h"
63#include "llvm/ADT/APInt.h"
64#include "llvm/ADT/ArrayRef.h"
65#include "llvm/ADT/DenseMap.h"
66#include "llvm/ADT/DenseMapInfo.h"
67#include "llvm/ADT/Hashing.h"
68#include "llvm/ADT/MapVector.h"
69#include "llvm/ADT/None.h"
70#include "llvm/ADT/Optional.h"
71#include "llvm/ADT/STLExtras.h"
72#include "llvm/ADT/SmallPtrSet.h"
73#include "llvm/ADT/SmallSet.h"
74#include "llvm/ADT/SmallVector.h"
75#include "llvm/ADT/Statistic.h"
76#include "llvm/ADT/StringRef.h"
77#include "llvm/ADT/Twine.h"
78#include "llvm/ADT/iterator_range.h"
79#include "llvm/Analysis/AssumptionCache.h"
80#include "llvm/Analysis/BasicAliasAnalysis.h"
81#include "llvm/Analysis/BlockFrequencyInfo.h"
82#include "llvm/Analysis/CFG.h"
83#include "llvm/Analysis/CodeMetrics.h"
84#include "llvm/Analysis/DemandedBits.h"
85#include "llvm/Analysis/GlobalsModRef.h"
86#include "llvm/Analysis/LoopAccessAnalysis.h"
87#include "llvm/Analysis/LoopAnalysisManager.h"
88#include "llvm/Analysis/LoopInfo.h"
89#include "llvm/Analysis/LoopIterator.h"
90#include "llvm/Analysis/OptimizationRemarkEmitter.h"
91#include "llvm/Analysis/ProfileSummaryInfo.h"
92#include "llvm/Analysis/ScalarEvolution.h"
93#include "llvm/Analysis/ScalarEvolutionExpressions.h"
94#include "llvm/Analysis/TargetLibraryInfo.h"
95#include "llvm/Analysis/TargetTransformInfo.h"
96#include "llvm/Analysis/VectorUtils.h"
97#include "llvm/IR/Attributes.h"
98#include "llvm/IR/BasicBlock.h"
99#include "llvm/IR/CFG.h"
100#include "llvm/IR/Constant.h"
101#include "llvm/IR/Constants.h"
102#include "llvm/IR/DataLayout.h"
103#include "llvm/IR/DebugInfoMetadata.h"
104#include "llvm/IR/DebugLoc.h"
105#include "llvm/IR/DerivedTypes.h"
106#include "llvm/IR/DiagnosticInfo.h"
107#include "llvm/IR/Dominators.h"
108#include "llvm/IR/Function.h"
109#include "llvm/IR/IRBuilder.h"
110#include "llvm/IR/InstrTypes.h"
111#include "llvm/IR/Instruction.h"
112#include "llvm/IR/Instructions.h"
113#include "llvm/IR/IntrinsicInst.h"
114#include "llvm/IR/Intrinsics.h"
115#include "llvm/IR/Metadata.h"
116#include "llvm/IR/Module.h"
117#include "llvm/IR/Operator.h"
118#include "llvm/IR/PatternMatch.h"
119#include "llvm/IR/Type.h"
120#include "llvm/IR/Use.h"
121#include "llvm/IR/User.h"
122#include "llvm/IR/Value.h"
123#include "llvm/IR/ValueHandle.h"
124#include "llvm/IR/Verifier.h"
125#include "llvm/InitializePasses.h"
126#include "llvm/Pass.h"
127#include "llvm/Support/Casting.h"
128#include "llvm/Support/CommandLine.h"
129#include "llvm/Support/Compiler.h"
130#include "llvm/Support/Debug.h"
131#include "llvm/Support/ErrorHandling.h"
132#include "llvm/Support/InstructionCost.h"
133#include "llvm/Support/MathExtras.h"
134#include "llvm/Support/raw_ostream.h"
135#include "llvm/Transforms/Utils/BasicBlockUtils.h"
136#include "llvm/Transforms/Utils/InjectTLIMappings.h"
137#include "llvm/Transforms/Utils/LoopSimplify.h"
138#include "llvm/Transforms/Utils/LoopUtils.h"
139#include "llvm/Transforms/Utils/LoopVersioning.h"
140#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
141#include "llvm/Transforms/Utils/SizeOpts.h"
142#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
143#include <algorithm>
144#include <cassert>
145#include <cstdint>
146#include <functional>
147#include <iterator>
148#include <limits>
149#include <memory>
150#include <string>
151#include <tuple>
152#include <utility>
153
154using namespace llvm;
155
156#define LV_NAME"loop-vectorize" "loop-vectorize"
157#define DEBUG_TYPE"loop-vectorize" LV_NAME"loop-vectorize"
158
159#ifndef NDEBUG
160const char VerboseDebug[] = DEBUG_TYPE"loop-vectorize" "-verbose";
161#endif
162
163/// @{
164/// Metadata attribute names
165const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all";
166const char LLVMLoopVectorizeFollowupVectorized[] =
167 "llvm.loop.vectorize.followup_vectorized";
168const char LLVMLoopVectorizeFollowupEpilogue[] =
169 "llvm.loop.vectorize.followup_epilogue";
170/// @}
171
172STATISTIC(LoopsVectorized, "Number of loops vectorized")static llvm::Statistic LoopsVectorized = {"loop-vectorize", "LoopsVectorized"
, "Number of loops vectorized"}
;
173STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization")static llvm::Statistic LoopsAnalyzed = {"loop-vectorize", "LoopsAnalyzed"
, "Number of loops analyzed for vectorization"}
;
174STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized")static llvm::Statistic LoopsEpilogueVectorized = {"loop-vectorize"
, "LoopsEpilogueVectorized", "Number of epilogues vectorized"
}
;
175
176static cl::opt<bool> EnableEpilogueVectorization(
177 "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
178 cl::desc("Enable vectorization of epilogue loops."));
179
180static cl::opt<unsigned> EpilogueVectorizationForceVF(
181 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
182 cl::desc("When epilogue vectorization is enabled, and a value greater than "
183 "1 is specified, forces the given VF for all applicable epilogue "
184 "loops."));
185
186static cl::opt<unsigned> EpilogueVectorizationMinVF(
187 "epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden,
188 cl::desc("Only loops with vectorization factor equal to or larger than "
189 "the specified value are considered for epilogue vectorization."));
190
191/// Loops with a known constant trip count below this number are vectorized only
192/// if no scalar iteration overheads are incurred.
193static cl::opt<unsigned> TinyTripCountVectorThreshold(
194 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
195 cl::desc("Loops with a constant trip count that is smaller than this "
196 "value are vectorized only if no scalar iteration overheads "
197 "are incurred."));
198
199static cl::opt<unsigned> PragmaVectorizeMemoryCheckThreshold(
200 "pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
201 cl::desc("The maximum allowed number of runtime memory checks with a "
202 "vectorize(enable) pragma."));
203
204// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
205// that predication is preferred, and this lists all options. I.e., the
206// vectorizer will try to fold the tail-loop (epilogue) into the vector body
207// and predicate the instructions accordingly. If tail-folding fails, there are
208// different fallback strategies depending on these values:
209namespace PreferPredicateTy {
210 enum Option {
211 ScalarEpilogue = 0,
212 PredicateElseScalarEpilogue,
213 PredicateOrDontVectorize
214 };
215} // namespace PreferPredicateTy
216
217static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
218 "prefer-predicate-over-epilogue",
219 cl::init(PreferPredicateTy::ScalarEpilogue),
220 cl::Hidden,
221 cl::desc("Tail-folding and predication preferences over creating a scalar "
222 "epilogue loop."),
223 cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
224 "scalar-epilogue",llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
225 "Don't tail-predicate loops, create scalar epilogue")llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
,
226 clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
227 "predicate-else-scalar-epilogue",llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
228 "prefer tail-folding, create scalar epilogue if tail "llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
229 "folding fails.")llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
,
230 clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
231 "predicate-dont-vectorize",llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
232 "prefers tail-folding, don't attempt vectorization if "llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
233 "tail-folding fails.")llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
));
234
235static cl::opt<bool> MaximizeBandwidth(
236 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
237 cl::desc("Maximize bandwidth when selecting vectorization factor which "
238 "will be determined by the smallest type in loop."));
239
240static cl::opt<bool> EnableInterleavedMemAccesses(
241 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
242 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
243
244/// An interleave-group may need masking if it resides in a block that needs
245/// predication, or in order to mask away gaps.
246static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
247 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
248 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
249
250static cl::opt<unsigned> TinyTripCountInterleaveThreshold(
251 "tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden,
252 cl::desc("We don't interleave loops with a estimated constant trip count "
253 "below this number"));
254
255static cl::opt<unsigned> ForceTargetNumScalarRegs(
256 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
257 cl::desc("A flag that overrides the target's number of scalar registers."));
258
259static cl::opt<unsigned> ForceTargetNumVectorRegs(
260 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
261 cl::desc("A flag that overrides the target's number of vector registers."));
262
263static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
264 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
265 cl::desc("A flag that overrides the target's max interleave factor for "
266 "scalar loops."));
267
268static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
269 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
270 cl::desc("A flag that overrides the target's max interleave factor for "
271 "vectorized loops."));
272
273static cl::opt<unsigned> ForceTargetInstructionCost(
274 "force-target-instruction-cost", cl::init(0), cl::Hidden,
275 cl::desc("A flag that overrides the target's expected cost for "
276 "an instruction to a single constant value. Mostly "
277 "useful for getting consistent testing."));
278
279static cl::opt<bool> ForceTargetSupportsScalableVectors(
280 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
281 cl::desc(
282 "Pretend that scalable vectors are supported, even if the target does "
283 "not support them. This flag should only be used for testing."));
284
285static cl::opt<unsigned> SmallLoopCost(
286 "small-loop-cost", cl::init(20), cl::Hidden,
287 cl::desc(
288 "The cost of a loop that is considered 'small' by the interleaver."));
289
290static cl::opt<bool> LoopVectorizeWithBlockFrequency(
291 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
292 cl::desc("Enable the use of the block frequency analysis to access PGO "
293 "heuristics minimizing code growth in cold regions and being more "
294 "aggressive in hot regions."));
295
296// Runtime interleave loops for load/store throughput.
297static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
298 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
299 cl::desc(
300 "Enable runtime interleaving until load/store ports are saturated"));
301
302/// Interleave small loops with scalar reductions.
303static cl::opt<bool> InterleaveSmallLoopScalarReduction(
304 "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden,
305 cl::desc("Enable interleaving for loops with small iteration counts that "
306 "contain scalar reductions to expose ILP."));
307
308/// The number of stores in a loop that are allowed to need predication.
309static cl::opt<unsigned> NumberOfStoresToPredicate(
310 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
311 cl::desc("Max number of stores to be predicated behind an if."));
312
313static cl::opt<bool> EnableIndVarRegisterHeur(
314 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
315 cl::desc("Count the induction variable only once when interleaving"));
316
317static cl::opt<bool> EnableCondStoresVectorization(
318 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
319 cl::desc("Enable if predication of stores during vectorization."));
320
321static cl::opt<unsigned> MaxNestedScalarReductionIC(
322 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
323 cl::desc("The maximum interleave count to use when interleaving a scalar "
324 "reduction in a nested loop."));
325
326static cl::opt<bool>
327 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
328 cl::Hidden,
329 cl::desc("Prefer in-loop vector reductions, "
330 "overriding the targets preference."));
331
332static cl::opt<bool> ForceOrderedReductions(
333 "force-ordered-reductions", cl::init(false), cl::Hidden,
334 cl::desc("Enable the vectorisation of loops with in-order (strict) "
335 "FP reductions"));
336
337static cl::opt<bool> PreferPredicatedReductionSelect(
338 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
339 cl::desc(
340 "Prefer predicating a reduction operation over an after loop select."));
341
342cl::opt<bool> EnableVPlanNativePath(
343 "enable-vplan-native-path", cl::init(false), cl::Hidden,
344 cl::desc("Enable VPlan-native vectorization path with "
345 "support for outer loop vectorization."));
346
347// FIXME: Remove this switch once we have divergence analysis. Currently we
348// assume divergent non-backedge branches when this switch is true.
349cl::opt<bool> EnableVPlanPredication(
350 "enable-vplan-predication", cl::init(false), cl::Hidden,
351 cl::desc("Enable VPlan-native vectorization path predicator with "
352 "support for outer loop vectorization."));
353
354// This flag enables the stress testing of the VPlan H-CFG construction in the
355// VPlan-native vectorization path. It must be used in conjuction with
356// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
357// verification of the H-CFGs built.
358static cl::opt<bool> VPlanBuildStressTest(
359 "vplan-build-stress-test", cl::init(false), cl::Hidden,
360 cl::desc(
361 "Build VPlan for every supported loop nest in the function and bail "
362 "out right after the build (stress test the VPlan H-CFG construction "
363 "in the VPlan-native vectorization path)."));
364
365cl::opt<bool> llvm::EnableLoopInterleaving(
366 "interleave-loops", cl::init(true), cl::Hidden,
367 cl::desc("Enable loop interleaving in Loop vectorization passes"));
368cl::opt<bool> llvm::EnableLoopVectorization(
369 "vectorize-loops", cl::init(true), cl::Hidden,
370 cl::desc("Run the Loop vectorization passes"));
371
372cl::opt<bool> PrintVPlansInDotFormat(
373 "vplan-print-in-dot-format", cl::init(false), cl::Hidden,
374 cl::desc("Use dot format instead of plain text when dumping VPlans"));
375
376/// A helper function that returns true if the given type is irregular. The
377/// type is irregular if its allocated size doesn't equal the store size of an
378/// element of the corresponding vector type.
379static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
380 // Determine if an array of N elements of type Ty is "bitcast compatible"
381 // with a <N x Ty> vector.
382 // This is only true if there is no padding between the array elements.
383 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
384}
385
386/// A helper function that returns the reciprocal of the block probability of
387/// predicated blocks. If we return X, we are assuming the predicated block
388/// will execute once for every X iterations of the loop header.
389///
390/// TODO: We should use actual block probability here, if available. Currently,
391/// we always assume predicated blocks have a 50% chance of executing.
392static unsigned getReciprocalPredBlockProb() { return 2; }
393
394/// A helper function that returns an integer or floating-point constant with
395/// value C.
396static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
397 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
398 : ConstantFP::get(Ty, C);
399}
400
401/// Returns "best known" trip count for the specified loop \p L as defined by
402/// the following procedure:
403/// 1) Returns exact trip count if it is known.
404/// 2) Returns expected trip count according to profile data if any.
405/// 3) Returns upper bound estimate if it is known.
406/// 4) Returns None if all of the above failed.
407static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) {
408 // Check if exact trip count is known.
409 if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L))
410 return ExpectedTC;
411
412 // Check if there is an expected trip count available from profile data.
413 if (LoopVectorizeWithBlockFrequency)
414 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
415 return EstimatedTC;
416
417 // Check if upper bound estimate is known.
418 if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
419 return ExpectedTC;
420
421 return None;
422}
423
424// Forward declare GeneratedRTChecks.
425class GeneratedRTChecks;
426
427namespace llvm {
428
429AnalysisKey ShouldRunExtraVectorPasses::Key;
430
431/// InnerLoopVectorizer vectorizes loops which contain only one basic
432/// block to a specified vectorization factor (VF).
433/// This class performs the widening of scalars into vectors, or multiple
434/// scalars. This class also implements the following features:
435/// * It inserts an epilogue loop for handling loops that don't have iteration
436/// counts that are known to be a multiple of the vectorization factor.
437/// * It handles the code generation for reduction variables.
438/// * Scalarization (implementation using scalars) of un-vectorizable
439/// instructions.
440/// InnerLoopVectorizer does not perform any vectorization-legality
441/// checks, and relies on the caller to check for the different legality
442/// aspects. The InnerLoopVectorizer relies on the
443/// LoopVectorizationLegality class to provide information about the induction
444/// and reduction variables that were found to a given vectorization factor.
445class InnerLoopVectorizer {
446public:
447 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
448 LoopInfo *LI, DominatorTree *DT,
449 const TargetLibraryInfo *TLI,
450 const TargetTransformInfo *TTI, AssumptionCache *AC,
451 OptimizationRemarkEmitter *ORE, ElementCount VecWidth,
452 unsigned UnrollFactor, LoopVectorizationLegality *LVL,
453 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
454 ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks)
455 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
456 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
457 Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI),
458 PSI(PSI), RTChecks(RTChecks) {
459 // Query this against the original loop and save it here because the profile
460 // of the original loop header may change as the transformation happens.
461 OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
462 OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
463 }
464
465 virtual ~InnerLoopVectorizer() = default;
466
467 /// Create a new empty loop that will contain vectorized instructions later
468 /// on, while the old loop will be used as the scalar remainder. Control flow
469 /// is generated around the vectorized (and scalar epilogue) loops consisting
470 /// of various checks and bypasses. Return the pre-header block of the new
471 /// loop and the start value for the canonical induction, if it is != 0. The
472 /// latter is the case when vectorizing the epilogue loop. In the case of
473 /// epilogue vectorization, this function is overriden to handle the more
474 /// complex control flow around the loops.
475 virtual std::pair<BasicBlock *, Value *> createVectorizedLoopSkeleton();
476
477 /// Widen a single call instruction within the innermost loop.
478 void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands,
479 VPTransformState &State);
480
481 /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
482 void fixVectorizedLoop(VPTransformState &State);
483
484 // Return true if any runtime check is added.
485 bool areSafetyChecksAdded() { return AddedSafetyChecks; }
486
487 /// A type for vectorized values in the new loop. Each value from the
488 /// original loop, when vectorized, is represented by UF vector values in the
489 /// new unrolled loop, where UF is the unroll factor.
490 using VectorParts = SmallVector<Value *, 2>;
491
492 /// Vectorize a single first-order recurrence or pointer induction PHINode in
493 /// a block. This method handles the induction variable canonicalization. It
494 /// supports both VF = 1 for unrolled loops and arbitrary length vectors.
495 void widenPHIInstruction(Instruction *PN, VPWidenPHIRecipe *PhiR,
496 VPTransformState &State);
497
498 /// A helper function to scalarize a single Instruction in the innermost loop.
499 /// Generates a sequence of scalar instances for each lane between \p MinLane
500 /// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
501 /// inclusive. Uses the VPValue operands from \p RepRecipe instead of \p
502 /// Instr's operands.
503 void scalarizeInstruction(Instruction *Instr, VPReplicateRecipe *RepRecipe,
504 const VPIteration &Instance, bool IfPredicateInstr,
505 VPTransformState &State);
506
507 /// Construct the vector value of a scalarized value \p V one lane at a time.
508 void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance,
509 VPTransformState &State);
510
511 /// Try to vectorize interleaved access group \p Group with the base address
512 /// given in \p Addr, optionally masking the vector operations if \p
513 /// BlockInMask is non-null. Use \p State to translate given VPValues to IR
514 /// values in the vectorized loop.
515 void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
516 ArrayRef<VPValue *> VPDefs,
517 VPTransformState &State, VPValue *Addr,
518 ArrayRef<VPValue *> StoredValues,
519 VPValue *BlockInMask = nullptr);
520
521 /// Set the debug location in the builder \p Ptr using the debug location in
522 /// \p V. If \p Ptr is None then it uses the class member's Builder.
523 void setDebugLocFromInst(const Value *V,
524 Optional<IRBuilderBase *> CustomBuilder = None);
525
526 /// Fix the non-induction PHIs in the OrigPHIsToFix vector.
527 void fixNonInductionPHIs(VPTransformState &State);
528
529 /// Returns true if the reordering of FP operations is not allowed, but we are
530 /// able to vectorize with strict in-order reductions for the given RdxDesc.
531 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc);
532
533 /// Create a broadcast instruction. This method generates a broadcast
534 /// instruction (shuffle) for loop invariant values and for the induction
535 /// value. If this is the induction variable then we extend it to N, N+1, ...
536 /// this is needed because each iteration in the loop corresponds to a SIMD
537 /// element.
538 virtual Value *getBroadcastInstrs(Value *V);
539
540 /// Add metadata from one instruction to another.
541 ///
542 /// This includes both the original MDs from \p From and additional ones (\see
543 /// addNewMetadata). Use this for *newly created* instructions in the vector
544 /// loop.
545 void addMetadata(Instruction *To, Instruction *From);
546
547 /// Similar to the previous function but it adds the metadata to a
548 /// vector of instructions.
549 void addMetadata(ArrayRef<Value *> To, Instruction *From);
550
551 // Returns the resume value (bc.merge.rdx) for a reduction as
552 // generated by fixReduction.
553 PHINode *getReductionResumeValue(const RecurrenceDescriptor &RdxDesc);
554
555protected:
556 friend class LoopVectorizationPlanner;
557
558 /// A small list of PHINodes.
559 using PhiVector = SmallVector<PHINode *, 4>;
560
561 /// A type for scalarized values in the new loop. Each value from the
562 /// original loop, when scalarized, is represented by UF x VF scalar values
563 /// in the new unrolled loop, where UF is the unroll factor and VF is the
564 /// vectorization factor.
565 using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
566
567 /// Set up the values of the IVs correctly when exiting the vector loop.
568 void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
569 Value *CountRoundDown, Value *EndValue,
570 BasicBlock *MiddleBlock, BasicBlock *VectorHeader);
571
572 /// Introduce a conditional branch (on true, condition to be set later) at the
573 /// end of the header=latch connecting it to itself (across the backedge) and
574 /// to the exit block of \p L.
575 void createHeaderBranch(Loop *L);
576
577 /// Handle all cross-iteration phis in the header.
578 void fixCrossIterationPHIs(VPTransformState &State);
579
580 /// Create the exit value of first order recurrences in the middle block and
581 /// update their users.
582 void fixFirstOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR,
583 VPTransformState &State);
584
585 /// Create code for the loop exit value of the reduction.
586 void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
587
588 /// Clear NSW/NUW flags from reduction instructions if necessary.
589 void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
590 VPTransformState &State);
591
592 /// Fixup the LCSSA phi nodes in the unique exit block. This simply
593 /// means we need to add the appropriate incoming value from the middle
594 /// block as exiting edges from the scalar epilogue loop (if present) are
595 /// already in place, and we exit the vector loop exclusively to the middle
596 /// block.
597 void fixLCSSAPHIs(VPTransformState &State);
598
599 /// Iteratively sink the scalarized operands of a predicated instruction into
600 /// the block that was created for it.
601 void sinkScalarOperands(Instruction *PredInst);
602
603 /// Shrinks vector element sizes to the smallest bitwidth they can be legally
604 /// represented as.
605 void truncateToMinimalBitwidths(VPTransformState &State);
606
607 /// Returns (and creates if needed) the original loop trip count.
608 Value *getOrCreateTripCount(Loop *NewLoop);
609
610 /// Returns (and creates if needed) the trip count of the widened loop.
611 Value *getOrCreateVectorTripCount(Loop *NewLoop);
612
613 /// Returns a bitcasted value to the requested vector type.
614 /// Also handles bitcasts of vector<float> <-> vector<pointer> types.
615 Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
616 const DataLayout &DL);
617
618 /// Emit a bypass check to see if the vector trip count is zero, including if
619 /// it overflows.
620 void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);
621
622 /// Emit a bypass check to see if all of the SCEV assumptions we've
623 /// had to make are correct. Returns the block containing the checks or
624 /// nullptr if no checks have been added.
625 BasicBlock *emitSCEVChecks(BasicBlock *Bypass);
626
627 /// Emit bypass checks to check any memory assumptions we may have made.
628 /// Returns the block containing the checks or nullptr if no checks have been
629 /// added.
630 BasicBlock *emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);
631
632 /// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
633 /// vector loop preheader, middle block and scalar preheader. Also
634 /// allocate a loop object for the new vector loop and return it.
635 Loop *createVectorLoopSkeleton(StringRef Prefix);
636
637 /// Create new phi nodes for the induction variables to resume iteration count
638 /// in the scalar epilogue, from where the vectorized loop left off.
639 /// In cases where the loop skeleton is more complicated (eg. epilogue
640 /// vectorization) and the resume values can come from an additional bypass
641 /// block, the \p AdditionalBypass pair provides information about the bypass
642 /// block and the end value on the edge from bypass to this loop.
643 void createInductionResumeValues(
644 Loop *L,
645 std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
646
647 /// Complete the loop skeleton by adding debug MDs, creating appropriate
648 /// conditional branches in the middle block, preparing the builder and
649 /// running the verifier. Take in the vector loop \p L as argument, and return
650 /// the preheader of the completed vector loop.
651 BasicBlock *completeLoopSkeleton(Loop *L, MDNode *OrigLoopID);
652
653 /// Add additional metadata to \p To that was not present on \p Orig.
654 ///
655 /// Currently this is used to add the noalias annotations based on the
656 /// inserted memchecks. Use this for instructions that are *cloned* into the
657 /// vector loop.
658 void addNewMetadata(Instruction *To, const Instruction *Orig);
659
660 /// Collect poison-generating recipes that may generate a poison value that is
661 /// used after vectorization, even when their operands are not poison. Those
662 /// recipes meet the following conditions:
663 /// * Contribute to the address computation of a recipe generating a widen
664 /// memory load/store (VPWidenMemoryInstructionRecipe or
665 /// VPInterleaveRecipe).
666 /// * Such a widen memory load/store has at least one underlying Instruction
667 /// that is in a basic block that needs predication and after vectorization
668 /// the generated instruction won't be predicated.
669 void collectPoisonGeneratingRecipes(VPTransformState &State);
670
671 /// Allow subclasses to override and print debug traces before/after vplan
672 /// execution, when trace information is requested.
673 virtual void printDebugTracesAtStart(){};
674 virtual void printDebugTracesAtEnd(){};
675
676 /// The original loop.
677 Loop *OrigLoop;
678
679 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
680 /// dynamic knowledge to simplify SCEV expressions and converts them to a
681 /// more usable form.
682 PredicatedScalarEvolution &PSE;
683
684 /// Loop Info.
685 LoopInfo *LI;
686
687 /// Dominator Tree.
688 DominatorTree *DT;
689
690 /// Alias Analysis.
691 AAResults *AA;
692
693 /// Target Library Info.
694 const TargetLibraryInfo *TLI;
695
696 /// Target Transform Info.
697 const TargetTransformInfo *TTI;
698
699 /// Assumption Cache.
700 AssumptionCache *AC;
701
702 /// Interface to emit optimization remarks.
703 OptimizationRemarkEmitter *ORE;
704
705 /// LoopVersioning. It's only set up (non-null) if memchecks were
706 /// used.
707 ///
708 /// This is currently only used to add no-alias metadata based on the
709 /// memchecks. The actually versioning is performed manually.
710 std::unique_ptr<LoopVersioning> LVer;
711
712 /// The vectorization SIMD factor to use. Each vector will have this many
713 /// vector elements.
714 ElementCount VF;
715
716 /// The vectorization unroll factor to use. Each scalar is vectorized to this
717 /// many different vector instructions.
718 unsigned UF;
719
720 /// The builder that we use
721 IRBuilder<> Builder;
722
723 // --- Vectorization state ---
724
725 /// The vector-loop preheader.
726 BasicBlock *LoopVectorPreHeader;
727
728 /// The scalar-loop preheader.
729 BasicBlock *LoopScalarPreHeader;
730
731 /// Middle Block between the vector and the scalar.
732 BasicBlock *LoopMiddleBlock;
733
734 /// The unique ExitBlock of the scalar loop if one exists. Note that
735 /// there can be multiple exiting edges reaching this block.
736 BasicBlock *LoopExitBlock;
737
738 /// The scalar loop body.
739 BasicBlock *LoopScalarBody;
740
741 /// A list of all bypass blocks. The first block is the entry of the loop.
742 SmallVector<BasicBlock *, 4> LoopBypassBlocks;
743
744 /// Store instructions that were predicated.
745 SmallVector<Instruction *, 4> PredicatedInstructions;
746
747 /// Trip count of the original loop.
748 Value *TripCount = nullptr;
749
750 /// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
751 Value *VectorTripCount = nullptr;
752
753 /// The legality analysis.
754 LoopVectorizationLegality *Legal;
755
756 /// The profitablity analysis.
757 LoopVectorizationCostModel *Cost;
758
759 // Record whether runtime checks are added.
760 bool AddedSafetyChecks = false;
761
762 // Holds the end values for each induction variable. We save the end values
763 // so we can later fix-up the external users of the induction variables.
764 DenseMap<PHINode *, Value *> IVEndValues;
765
766 // Vector of original scalar PHIs whose corresponding widened PHIs need to be
767 // fixed up at the end of vector code generation.
768 SmallVector<PHINode *, 8> OrigPHIsToFix;
769
770 /// BFI and PSI are used to check for profile guided size optimizations.
771 BlockFrequencyInfo *BFI;
772 ProfileSummaryInfo *PSI;
773
774 // Whether this loop should be optimized for size based on profile guided size
775 // optimizatios.
776 bool OptForSizeBasedOnProfile;
777
778 /// Structure to hold information about generated runtime checks, responsible
779 /// for cleaning the checks, if vectorization turns out unprofitable.
780 GeneratedRTChecks &RTChecks;
781
782 // Holds the resume values for reductions in the loops, used to set the
783 // correct start value of reduction PHIs when vectorizing the epilogue.
784 SmallMapVector<const RecurrenceDescriptor *, PHINode *, 4>
785 ReductionResumeValues;
786};
787
788class InnerLoopUnroller : public InnerLoopVectorizer {
789public:
790 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
791 LoopInfo *LI, DominatorTree *DT,
792 const TargetLibraryInfo *TLI,
793 const TargetTransformInfo *TTI, AssumptionCache *AC,
794 OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
795 LoopVectorizationLegality *LVL,
796 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
797 ProfileSummaryInfo *PSI, GeneratedRTChecks &Check)
798 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
799 ElementCount::getFixed(1), UnrollFactor, LVL, CM,
800 BFI, PSI, Check) {}
801
802private:
803 Value *getBroadcastInstrs(Value *V) override;
804};
805
806/// Encapsulate information regarding vectorization of a loop and its epilogue.
807/// This information is meant to be updated and used across two stages of
808/// epilogue vectorization.
809struct EpilogueLoopVectorizationInfo {
810 ElementCount MainLoopVF = ElementCount::getFixed(0);
811 unsigned MainLoopUF = 0;
812 ElementCount EpilogueVF = ElementCount::getFixed(0);
813 unsigned EpilogueUF = 0;
814 BasicBlock *MainLoopIterationCountCheck = nullptr;
815 BasicBlock *EpilogueIterationCountCheck = nullptr;
816 BasicBlock *SCEVSafetyCheck = nullptr;
817 BasicBlock *MemSafetyCheck = nullptr;
818 Value *TripCount = nullptr;
819 Value *VectorTripCount = nullptr;
820
821 EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
822 ElementCount EVF, unsigned EUF)
823 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF) {
824 assert(EUF == 1 &&(static_cast <bool> (EUF == 1 && "A high UF for the epilogue loop is likely not beneficial."
) ? void (0) : __assert_fail ("EUF == 1 && \"A high UF for the epilogue loop is likely not beneficial.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 825, __extension__
__PRETTY_FUNCTION__))
825 "A high UF for the epilogue loop is likely not beneficial.")(static_cast <bool> (EUF == 1 && "A high UF for the epilogue loop is likely not beneficial."
) ? void (0) : __assert_fail ("EUF == 1 && \"A high UF for the epilogue loop is likely not beneficial.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 825, __extension__
__PRETTY_FUNCTION__))
;
826 }
827};
828
829/// An extension of the inner loop vectorizer that creates a skeleton for a
830/// vectorized loop that has its epilogue (residual) also vectorized.
831/// The idea is to run the vplan on a given loop twice, firstly to setup the
832/// skeleton and vectorize the main loop, and secondly to complete the skeleton
833/// from the first step and vectorize the epilogue. This is achieved by
834/// deriving two concrete strategy classes from this base class and invoking
835/// them in succession from the loop vectorizer planner.
836class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
837public:
838 InnerLoopAndEpilogueVectorizer(
839 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
840 DominatorTree *DT, const TargetLibraryInfo *TLI,
841 const TargetTransformInfo *TTI, AssumptionCache *AC,
842 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
843 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
844 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
845 GeneratedRTChecks &Checks)
846 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
847 EPI.MainLoopVF, EPI.MainLoopUF, LVL, CM, BFI, PSI,
848 Checks),
849 EPI(EPI) {}
850
851 // Override this function to handle the more complex control flow around the
852 // three loops.
853 std::pair<BasicBlock *, Value *>
854 createVectorizedLoopSkeleton() final override {
855 return createEpilogueVectorizedLoopSkeleton();
856 }
857
858 /// The interface for creating a vectorized skeleton using one of two
859 /// different strategies, each corresponding to one execution of the vplan
860 /// as described above.
861 virtual std::pair<BasicBlock *, Value *>
862 createEpilogueVectorizedLoopSkeleton() = 0;
863
864 /// Holds and updates state information required to vectorize the main loop
865 /// and its epilogue in two separate passes. This setup helps us avoid
866 /// regenerating and recomputing runtime safety checks. It also helps us to
867 /// shorten the iteration-count-check path length for the cases where the
868 /// iteration count of the loop is so small that the main vector loop is
869 /// completely skipped.
870 EpilogueLoopVectorizationInfo &EPI;
871};
872
873/// A specialized derived class of inner loop vectorizer that performs
874/// vectorization of *main* loops in the process of vectorizing loops and their
875/// epilogues.
876class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
877public:
878 EpilogueVectorizerMainLoop(
879 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
880 DominatorTree *DT, const TargetLibraryInfo *TLI,
881 const TargetTransformInfo *TTI, AssumptionCache *AC,
882 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
883 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
884 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
885 GeneratedRTChecks &Check)
886 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
887 EPI, LVL, CM, BFI, PSI, Check) {}
888 /// Implements the interface for creating a vectorized skeleton using the
889 /// *main loop* strategy (ie the first pass of vplan execution).
890 std::pair<BasicBlock *, Value *>
891 createEpilogueVectorizedLoopSkeleton() final override;
892
893protected:
894 /// Emits an iteration count bypass check once for the main loop (when \p
895 /// ForEpilogue is false) and once for the epilogue loop (when \p
896 /// ForEpilogue is true).
897 BasicBlock *emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass,
898 bool ForEpilogue);
899 void printDebugTracesAtStart() override;
900 void printDebugTracesAtEnd() override;
901};
902
903// A specialized derived class of inner loop vectorizer that performs
904// vectorization of *epilogue* loops in the process of vectorizing loops and
905// their epilogues.
906class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
907public:
908 EpilogueVectorizerEpilogueLoop(
909 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
910 DominatorTree *DT, const TargetLibraryInfo *TLI,
911 const TargetTransformInfo *TTI, AssumptionCache *AC,
912 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
913 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
914 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
915 GeneratedRTChecks &Checks)
916 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
917 EPI, LVL, CM, BFI, PSI, Checks) {}
918 /// Implements the interface for creating a vectorized skeleton using the
919 /// *epilogue loop* strategy (ie the second pass of vplan execution).
920 std::pair<BasicBlock *, Value *>
921 createEpilogueVectorizedLoopSkeleton() final override;
922
923protected:
924 /// Emits an iteration count bypass check after the main vector loop has
925 /// finished to see if there are any iterations left to execute by either
926 /// the vector epilogue or the scalar epilogue.
927 BasicBlock *emitMinimumVectorEpilogueIterCountCheck(
928 BasicBlock *Bypass,
929 BasicBlock *Insert);
930 void printDebugTracesAtStart() override;
931 void printDebugTracesAtEnd() override;
932};
933} // end namespace llvm
934
935/// Look for a meaningful debug location on the instruction or it's
936/// operands.
937static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
938 if (!I)
939 return I;
940
941 DebugLoc Empty;
942 if (I->getDebugLoc() != Empty)
943 return I;
944
945 for (Use &Op : I->operands()) {
946 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
947 if (OpInst->getDebugLoc() != Empty)
948 return OpInst;
949 }
950
951 return I;
952}
953
954void InnerLoopVectorizer::setDebugLocFromInst(
955 const Value *V, Optional<IRBuilderBase *> CustomBuilder) {
956 IRBuilderBase *B = (CustomBuilder == None) ? &Builder : *CustomBuilder;
957 if (const Instruction *Inst = dyn_cast_or_null<Instruction>(V)) {
958 const DILocation *DIL = Inst->getDebugLoc();
959
960 // When a FSDiscriminator is enabled, we don't need to add the multiply
961 // factors to the discriminators.
962 if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
963 !isa<DbgInfoIntrinsic>(Inst) && !EnableFSDiscriminator) {
964 // FIXME: For scalable vectors, assume vscale=1.
965 auto NewDIL =
966 DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue());
967 if (NewDIL)
968 B->SetCurrentDebugLocation(NewDIL.getValue());
969 else
970 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
971 << "Failed to create new discriminator: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
972 << DIL->getFilename() << " Line: " << DIL->getLine())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "Failed to create new discriminator: "
<< DIL->getFilename() << " Line: " << DIL
->getLine(); } } while (false)
;
973 } else
974 B->SetCurrentDebugLocation(DIL);
975 } else
976 B->SetCurrentDebugLocation(DebugLoc());
977}
978
979/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
980/// is passed, the message relates to that particular instruction.
981#ifndef NDEBUG
982static void debugVectorizationMessage(const StringRef Prefix,
983 const StringRef DebugMsg,
984 Instruction *I) {
985 dbgs() << "LV: " << Prefix << DebugMsg;
986 if (I != nullptr)
987 dbgs() << " " << *I;
988 else
989 dbgs() << '.';
990 dbgs() << '\n';
991}
992#endif
993
994/// Create an analysis remark that explains why vectorization failed
995///
996/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
997/// RemarkName is the identifier for the remark. If \p I is passed it is an
998/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
999/// the location of the remark. \return the remark object that can be
1000/// streamed to.
1001static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
1002 StringRef RemarkName, Loop *TheLoop, Instruction *I) {
1003 Value *CodeRegion = TheLoop->getHeader();
1004 DebugLoc DL = TheLoop->getStartLoc();
1005
1006 if (I) {
1007 CodeRegion = I->getParent();
1008 // If there is no debug location attached to the instruction, revert back to
1009 // using the loop's.
1010 if (I->getDebugLoc())
1011 DL = I->getDebugLoc();
1012 }
1013
1014 return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
1015}
1016
1017namespace llvm {
1018
1019/// Return a value for Step multiplied by VF.
1020Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
1021 int64_t Step) {
1022 assert(Ty->isIntegerTy() && "Expected an integer step")(static_cast <bool> (Ty->isIntegerTy() && "Expected an integer step"
) ? void (0) : __assert_fail ("Ty->isIntegerTy() && \"Expected an integer step\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1022, __extension__
__PRETTY_FUNCTION__))
;
1023 Constant *StepVal = ConstantInt::get(Ty, Step * VF.getKnownMinValue());
1024 return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal;
1025}
1026
1027/// Return the runtime value for VF.
1028Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
1029 Constant *EC = ConstantInt::get(Ty, VF.getKnownMinValue());
1030 return VF.isScalable() ? B.CreateVScale(EC) : EC;
1031}
1032
1033static Value *getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy,
1034 ElementCount VF) {
1035 assert(FTy->isFloatingPointTy() && "Expected floating point type!")(static_cast <bool> (FTy->isFloatingPointTy() &&
"Expected floating point type!") ? void (0) : __assert_fail (
"FTy->isFloatingPointTy() && \"Expected floating point type!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1035, __extension__
__PRETTY_FUNCTION__))
;
1036 Type *IntTy = IntegerType::get(FTy->getContext(), FTy->getScalarSizeInBits());
1037 Value *RuntimeVF = getRuntimeVF(B, IntTy, VF);
1038 return B.CreateUIToFP(RuntimeVF, FTy);
1039}
1040
1041void reportVectorizationFailure(const StringRef DebugMsg,
1042 const StringRef OREMsg, const StringRef ORETag,
1043 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
1044 Instruction *I) {
1045 LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { debugVectorizationMessage("Not vectorizing: "
, DebugMsg, I); } } while (false)
;
1046 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
1047 ORE->emit(
1048 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
1049 << "loop not vectorized: " << OREMsg);
1050}
1051
1052void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
1053 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
1054 Instruction *I) {
1055 LLVM_DEBUG(debugVectorizationMessage("", Msg, I))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { debugVectorizationMessage("", Msg, I); }
} while (false)
;
1056 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
1057 ORE->emit(
1058 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
1059 << Msg);
1060}
1061
1062} // end namespace llvm
1063
1064#ifndef NDEBUG
1065/// \return string containing a file name and a line # for the given loop.
1066static std::string getDebugLocString(const Loop *L) {
1067 std::string Result;
1068 if (L) {
1069 raw_string_ostream OS(Result);
1070 if (const DebugLoc LoopDbgLoc = L->getStartLoc())
1071 LoopDbgLoc.print(OS);
1072 else
1073 // Just print the module name.
1074 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
1075 OS.flush();
1076 }
1077 return Result;
1078}
1079#endif
1080
1081void InnerLoopVectorizer::addNewMetadata(Instruction *To,
1082 const Instruction *Orig) {
1083 // If the loop was versioned with memchecks, add the corresponding no-alias
1084 // metadata.
1085 if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
1086 LVer->annotateInstWithNoAlias(To, Orig);
1087}
1088
1089void InnerLoopVectorizer::collectPoisonGeneratingRecipes(
1090 VPTransformState &State) {
1091
1092 // Collect recipes in the backward slice of `Root` that may generate a poison
1093 // value that is used after vectorization.
1094 SmallPtrSet<VPRecipeBase *, 16> Visited;
1095 auto collectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
1096 SmallVector<VPRecipeBase *, 16> Worklist;
1097 Worklist.push_back(Root);
1098
1099 // Traverse the backward slice of Root through its use-def chain.
1100 while (!Worklist.empty()) {
1101 VPRecipeBase *CurRec = Worklist.back();
1102 Worklist.pop_back();
1103
1104 if (!Visited.insert(CurRec).second)
1105 continue;
1106
1107 // Prune search if we find another recipe generating a widen memory
1108 // instruction. Widen memory instructions involved in address computation
1109 // will lead to gather/scatter instructions, which don't need to be
1110 // handled.
1111 if (isa<VPWidenMemoryInstructionRecipe>(CurRec) ||
1112 isa<VPInterleaveRecipe>(CurRec) ||
1113 isa<VPScalarIVStepsRecipe>(CurRec) ||
1114 isa<VPCanonicalIVPHIRecipe>(CurRec))
1115 continue;
1116
1117 // This recipe contributes to the address computation of a widen
1118 // load/store. Collect recipe if its underlying instruction has
1119 // poison-generating flags.
1120 Instruction *Instr = CurRec->getUnderlyingInstr();
1121 if (Instr && Instr->hasPoisonGeneratingFlags())
1122 State.MayGeneratePoisonRecipes.insert(CurRec);
1123
1124 // Add new definitions to the worklist.
1125 for (VPValue *operand : CurRec->operands())
1126 if (VPDef *OpDef = operand->getDef())
1127 Worklist.push_back(cast<VPRecipeBase>(OpDef));
1128 }
1129 });
1130
1131 // Traverse all the recipes in the VPlan and collect the poison-generating
1132 // recipes in the backward slice starting at the address of a VPWidenRecipe or
1133 // VPInterleaveRecipe.
1134 auto Iter = depth_first(
1135 VPBlockRecursiveTraversalWrapper<VPBlockBase *>(State.Plan->getEntry()));
1136 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
1137 for (VPRecipeBase &Recipe : *VPBB) {
1138 if (auto *WidenRec = dyn_cast<VPWidenMemoryInstructionRecipe>(&Recipe)) {
1139 Instruction *UnderlyingInstr = WidenRec->getUnderlyingInstr();
1140 VPDef *AddrDef = WidenRec->getAddr()->getDef();
1141 if (AddrDef && WidenRec->isConsecutive() && UnderlyingInstr &&
1142 Legal->blockNeedsPredication(UnderlyingInstr->getParent()))
1143 collectPoisonGeneratingInstrsInBackwardSlice(
1144 cast<VPRecipeBase>(AddrDef));
1145 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(&Recipe)) {
1146 VPDef *AddrDef = InterleaveRec->getAddr()->getDef();
1147 if (AddrDef) {
1148 // Check if any member of the interleave group needs predication.
1149 const InterleaveGroup<Instruction> *InterGroup =
1150 InterleaveRec->getInterleaveGroup();
1151 bool NeedPredication = false;
1152 for (int I = 0, NumMembers = InterGroup->getNumMembers();
1153 I < NumMembers; ++I) {
1154 Instruction *Member = InterGroup->getMember(I);
1155 if (Member)
1156 NeedPredication |=
1157 Legal->blockNeedsPredication(Member->getParent());
1158 }
1159
1160 if (NeedPredication)
1161 collectPoisonGeneratingInstrsInBackwardSlice(
1162 cast<VPRecipeBase>(AddrDef));
1163 }
1164 }
1165 }
1166 }
1167}
1168
1169void InnerLoopVectorizer::addMetadata(Instruction *To,
1170 Instruction *From) {
1171 propagateMetadata(To, From);
1172 addNewMetadata(To, From);
1173}
1174
1175void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,
1176 Instruction *From) {
1177 for (Value *V : To) {
1178 if (Instruction *I = dyn_cast<Instruction>(V))
1179 addMetadata(I, From);
1180 }
1181}
1182
1183PHINode *InnerLoopVectorizer::getReductionResumeValue(
1184 const RecurrenceDescriptor &RdxDesc) {
1185 auto It = ReductionResumeValues.find(&RdxDesc);
1186 assert(It != ReductionResumeValues.end() &&(static_cast <bool> (It != ReductionResumeValues.end() &&
"Expected to find a resume value for the reduction.") ? void
(0) : __assert_fail ("It != ReductionResumeValues.end() && \"Expected to find a resume value for the reduction.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1187, __extension__
__PRETTY_FUNCTION__))
1187 "Expected to find a resume value for the reduction.")(static_cast <bool> (It != ReductionResumeValues.end() &&
"Expected to find a resume value for the reduction.") ? void
(0) : __assert_fail ("It != ReductionResumeValues.end() && \"Expected to find a resume value for the reduction.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1187, __extension__
__PRETTY_FUNCTION__))
;
1188 return It->second;
1189}
1190
1191namespace llvm {
1192
1193// Loop vectorization cost-model hints how the scalar epilogue loop should be
1194// lowered.
1195enum ScalarEpilogueLowering {
1196
1197 // The default: allowing scalar epilogues.
1198 CM_ScalarEpilogueAllowed,
1199
1200 // Vectorization with OptForSize: don't allow epilogues.
1201 CM_ScalarEpilogueNotAllowedOptSize,
1202
1203 // A special case of vectorisation with OptForSize: loops with a very small
1204 // trip count are considered for vectorization under OptForSize, thereby
1205 // making sure the cost of their loop body is dominant, free of runtime
1206 // guards and scalar iteration overheads.
1207 CM_ScalarEpilogueNotAllowedLowTripLoop,
1208
1209 // Loop hint predicate indicating an epilogue is undesired.
1210 CM_ScalarEpilogueNotNeededUsePredicate,
1211
1212 // Directive indicating we must either tail fold or not vectorize
1213 CM_ScalarEpilogueNotAllowedUsePredicate
1214};
1215
1216/// ElementCountComparator creates a total ordering for ElementCount
1217/// for the purposes of using it in a set structure.
1218struct ElementCountComparator {
1219 bool operator()(const ElementCount &LHS, const ElementCount &RHS) const {
1220 return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) <
1221 std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue());
1222 }
1223};
1224using ElementCountSet = SmallSet<ElementCount, 16, ElementCountComparator>;
1225
1226/// LoopVectorizationCostModel - estimates the expected speedups due to
1227/// vectorization.
1228/// In many cases vectorization is not profitable. This can happen because of
1229/// a number of reasons. In this class we mainly attempt to predict the
1230/// expected speedup/slowdowns due to the supported instruction set. We use the
1231/// TargetTransformInfo to query the different backends for the cost of
1232/// different operations.
1233class LoopVectorizationCostModel {
1234public:
1235 LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L,
1236 PredicatedScalarEvolution &PSE, LoopInfo *LI,
1237 LoopVectorizationLegality *Legal,
1238 const TargetTransformInfo &TTI,
1239 const TargetLibraryInfo *TLI, DemandedBits *DB,
1240 AssumptionCache *AC,
1241 OptimizationRemarkEmitter *ORE, const Function *F,
1242 const LoopVectorizeHints *Hints,
1243 InterleavedAccessInfo &IAI)
1244 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
1245 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
1246 Hints(Hints), InterleaveInfo(IAI) {}
1247
1248 /// \return An upper bound for the vectorization factors (both fixed and
1249 /// scalable). If the factors are 0, vectorization and interleaving should be
1250 /// avoided up front.
1251 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
1252
1253 /// \return True if runtime checks are required for vectorization, and false
1254 /// otherwise.
1255 bool runtimeChecksRequired();
1256
1257 /// \return The most profitable vectorization factor and the cost of that VF.
1258 /// This method checks every VF in \p CandidateVFs. If UserVF is not ZERO
1259 /// then this vectorization factor will be selected if vectorization is
1260 /// possible.
1261 VectorizationFactor
1262 selectVectorizationFactor(const ElementCountSet &CandidateVFs);
1263
1264 VectorizationFactor
1265 selectEpilogueVectorizationFactor(const ElementCount MaxVF,
1266 const LoopVectorizationPlanner &LVP);
1267
1268 /// Setup cost-based decisions for user vectorization factor.
1269 /// \return true if the UserVF is a feasible VF to be chosen.
1270 bool selectUserVectorizationFactor(ElementCount UserVF) {
1271 collectUniformsAndScalars(UserVF);
1272 collectInstsToScalarize(UserVF);
1273 return expectedCost(UserVF).first.isValid();
1274 }
1275
1276 /// \return The size (in bits) of the smallest and widest types in the code
1277 /// that needs to be vectorized. We ignore values that remain scalar such as
1278 /// 64 bit loop indices.
1279 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
1280
1281 /// \return The desired interleave count.
1282 /// If interleave count has been specified by metadata it will be returned.
1283 /// Otherwise, the interleave count is computed and returned. VF and LoopCost
1284 /// are the selected vectorization factor and the cost of the selected VF.
1285 unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost);
1286
1287 /// Memory access instruction may be vectorized in more than one way.
1288 /// Form of instruction after vectorization depends on cost.
1289 /// This function takes cost-based decisions for Load/Store instructions
1290 /// and collects them in a map. This decisions map is used for building
1291 /// the lists of loop-uniform and loop-scalar instructions.
1292 /// The calculated cost is saved with widening decision in order to
1293 /// avoid redundant calculations.
1294 void setCostBasedWideningDecision(ElementCount VF);
1295
1296 /// A struct that represents some properties of the register usage
1297 /// of a loop.
1298 struct RegisterUsage {
1299 /// Holds the number of loop invariant values that are used in the loop.
1300 /// The key is ClassID of target-provided register class.
1301 SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
1302 /// Holds the maximum number of concurrent live intervals in the loop.
1303 /// The key is ClassID of target-provided register class.
1304 SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
1305 };
1306
1307 /// \return Returns information about the register usages of the loop for the
1308 /// given vectorization factors.
1309 SmallVector<RegisterUsage, 8>
1310 calculateRegisterUsage(ArrayRef<ElementCount> VFs);
1311
1312 /// Collect values we want to ignore in the cost model.
1313 void collectValuesToIgnore();
1314
1315 /// Collect all element types in the loop for which widening is needed.
1316 void collectElementTypesForWidening();
1317
1318 /// Split reductions into those that happen in the loop, and those that happen
1319 /// outside. In loop reductions are collected into InLoopReductionChains.
1320 void collectInLoopReductions();
1321
1322 /// Returns true if we should use strict in-order reductions for the given
1323 /// RdxDesc. This is true if the -enable-strict-reductions flag is passed,
1324 /// the IsOrdered flag of RdxDesc is set and we do not allow reordering
1325 /// of FP operations.
1326 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) {
1327 return !Hints->allowReordering() && RdxDesc.isOrdered();
1328 }
1329
1330 /// \returns The smallest bitwidth each instruction can be represented with.
1331 /// The vector equivalents of these instructions should be truncated to this
1332 /// type.
1333 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
1334 return MinBWs;
1335 }
1336
1337 /// \returns True if it is more profitable to scalarize instruction \p I for
1338 /// vectorization factor \p VF.
1339 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
1340 assert(VF.isVector() &&(static_cast <bool> (VF.isVector() && "Profitable to scalarize relevant only for VF > 1."
) ? void (0) : __assert_fail ("VF.isVector() && \"Profitable to scalarize relevant only for VF > 1.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1341, __extension__
__PRETTY_FUNCTION__))
1341 "Profitable to scalarize relevant only for VF > 1.")(static_cast <bool> (VF.isVector() && "Profitable to scalarize relevant only for VF > 1."
) ? void (0) : __assert_fail ("VF.isVector() && \"Profitable to scalarize relevant only for VF > 1.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1341, __extension__
__PRETTY_FUNCTION__))
;
1342
1343 // Cost model is not run in the VPlan-native path - return conservative
1344 // result until this changes.
1345 if (EnableVPlanNativePath)
1346 return false;
1347
1348 auto Scalars = InstsToScalarize.find(VF);
1349 assert(Scalars != InstsToScalarize.end() &&(static_cast <bool> (Scalars != InstsToScalarize.end() &&
"VF not yet analyzed for scalarization profitability") ? void
(0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1350, __extension__
__PRETTY_FUNCTION__))
1350 "VF not yet analyzed for scalarization profitability")(static_cast <bool> (Scalars != InstsToScalarize.end() &&
"VF not yet analyzed for scalarization profitability") ? void
(0) : __assert_fail ("Scalars != InstsToScalarize.end() && \"VF not yet analyzed for scalarization profitability\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1350, __extension__
__PRETTY_FUNCTION__))
;
1351 return Scalars->second.find(I) != Scalars->second.end();
1352 }
1353
1354 /// Returns true if \p I is known to be uniform after vectorization.
1355 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
1356 if (VF.isScalar())
1357 return true;
1358
1359 // Cost model is not run in the VPlan-native path - return conservative
1360 // result until this changes.
1361 if (EnableVPlanNativePath)
1362 return false;
1363
1364 auto UniformsPerVF = Uniforms.find(VF);
1365 assert(UniformsPerVF != Uniforms.end() &&(static_cast <bool> (UniformsPerVF != Uniforms.end() &&
"VF not yet analyzed for uniformity") ? void (0) : __assert_fail
("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1366, __extension__
__PRETTY_FUNCTION__))
1366 "VF not yet analyzed for uniformity")(static_cast <bool> (UniformsPerVF != Uniforms.end() &&
"VF not yet analyzed for uniformity") ? void (0) : __assert_fail
("UniformsPerVF != Uniforms.end() && \"VF not yet analyzed for uniformity\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1366, __extension__
__PRETTY_FUNCTION__))
;
1367 return UniformsPerVF->second.count(I);
1368 }
1369
1370 /// Returns true if \p I is known to be scalar after vectorization.
1371 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
1372 if (VF.isScalar())
1373 return true;
1374
1375 // Cost model is not run in the VPlan-native path - return conservative
1376 // result until this changes.
1377 if (EnableVPlanNativePath)
1378 return false;
1379
1380 auto ScalarsPerVF = Scalars.find(VF);
1381 assert(ScalarsPerVF != Scalars.end() &&(static_cast <bool> (ScalarsPerVF != Scalars.end() &&
"Scalar values are not calculated for VF") ? void (0) : __assert_fail
("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1382, __extension__
__PRETTY_FUNCTION__))
1382 "Scalar values are not calculated for VF")(static_cast <bool> (ScalarsPerVF != Scalars.end() &&
"Scalar values are not calculated for VF") ? void (0) : __assert_fail
("ScalarsPerVF != Scalars.end() && \"Scalar values are not calculated for VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1382, __extension__
__PRETTY_FUNCTION__))
;
1383 return ScalarsPerVF->second.count(I);
1384 }
1385
1386 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1387 /// for vectorization factor \p VF.
1388 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
1389 return VF.isVector() && MinBWs.find(I) != MinBWs.end() &&
1390 !isProfitableToScalarize(I, VF) &&
1391 !isScalarAfterVectorization(I, VF);
1392 }
1393
1394 /// Decision that was taken during cost calculation for memory instruction.
1395 enum InstWidening {
1396 CM_Unknown,
1397 CM_Widen, // For consecutive accesses with stride +1.
1398 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1399 CM_Interleave,
1400 CM_GatherScatter,
1401 CM_Scalarize
1402 };
1403
1404 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1405 /// instruction \p I and vector width \p VF.
1406 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
1407 InstructionCost Cost) {
1408 assert(VF.isVector() && "Expected VF >=2")(static_cast <bool> (VF.isVector() && "Expected VF >=2"
) ? void (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1408, __extension__
__PRETTY_FUNCTION__))
;
1409 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1410 }
1411
1412 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1413 /// interleaving group \p Grp and vector width \p VF.
1414 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
1415 ElementCount VF, InstWidening W,
1416 InstructionCost Cost) {
1417 assert(VF.isVector() && "Expected VF >=2")(static_cast <bool> (VF.isVector() && "Expected VF >=2"
) ? void (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1417, __extension__
__PRETTY_FUNCTION__))
;
1418 /// Broadcast this decicion to all instructions inside the group.
1419 /// But the cost will be assigned to one instruction only.
1420 for (unsigned i = 0; i < Grp->getFactor(); ++i) {
1421 if (auto *I = Grp->getMember(i)) {
1422 if (Grp->getInsertPos() == I)
1423 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1424 else
1425 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
1426 }
1427 }
1428 }
1429
1430 /// Return the cost model decision for the given instruction \p I and vector
1431 /// width \p VF. Return CM_Unknown if this instruction did not pass
1432 /// through the cost modeling.
1433 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
1434 assert(VF.isVector() && "Expected VF to be a vector VF")(static_cast <bool> (VF.isVector() && "Expected VF to be a vector VF"
) ? void (0) : __assert_fail ("VF.isVector() && \"Expected VF to be a vector VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1434, __extension__
__PRETTY_FUNCTION__))
;
1435 // Cost model is not run in the VPlan-native path - return conservative
1436 // result until this changes.
1437 if (EnableVPlanNativePath)
1438 return CM_GatherScatter;
1439
1440 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1441 auto Itr = WideningDecisions.find(InstOnVF);
1442 if (Itr == WideningDecisions.end())
1443 return CM_Unknown;
1444 return Itr->second.first;
1445 }
1446
1447 /// Return the vectorization cost for the given instruction \p I and vector
1448 /// width \p VF.
1449 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
1450 assert(VF.isVector() && "Expected VF >=2")(static_cast <bool> (VF.isVector() && "Expected VF >=2"
) ? void (0) : __assert_fail ("VF.isVector() && \"Expected VF >=2\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1450, __extension__
__PRETTY_FUNCTION__))
;
1451 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1452 assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() &&(static_cast <bool> (WideningDecisions.find(InstOnVF) !=
WideningDecisions.end() && "The cost is not calculated"
) ? void (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1453, __extension__
__PRETTY_FUNCTION__))
1453 "The cost is not calculated")(static_cast <bool> (WideningDecisions.find(InstOnVF) !=
WideningDecisions.end() && "The cost is not calculated"
) ? void (0) : __assert_fail ("WideningDecisions.find(InstOnVF) != WideningDecisions.end() && \"The cost is not calculated\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 1453, __extension__
__PRETTY_FUNCTION__))
;
1454 return WideningDecisions[InstOnVF].second;
1455 }
1456
1457 /// Return True if instruction \p I is an optimizable truncate whose operand
1458 /// is an induction variable. Such a truncate will be removed by adding a new
1459 /// induction variable with the destination type.
1460 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
1461 // If the instruction is not a truncate, return false.
1462 auto *Trunc = dyn_cast<TruncInst>(I);
1463 if (!Trunc)
1464 return false;
1465
1466 // Get the source and destination types of the truncate.
1467 Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);
1468 Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);
1469
1470 // If the truncate is free for the given types, return false. Replacing a
1471 // free truncate with an induction variable would add an induction variable
1472 // update instruction to each iteration of the loop. We exclude from this
1473 // check the primary induction variable since it will need an update
1474 // instruction regardless.
1475 Value *Op = Trunc->getOperand(0);
1476 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1477 return false;
1478
1479 // If the truncated value is not an induction variable, return false.
1480 return Legal->isInductionPhi(Op);
1481 }
1482
1483 /// Collects the instructions to scalarize for each predicated instruction in
1484 /// the loop.
1485 void collectInstsToScalarize(ElementCount VF);
1486
1487 /// Collect Uniform and Scalar values for the given \p VF.
1488 /// The sets depend on CM decision for Load/Store instructions
1489 /// that may be vectorized as interleave, gather-scatter or scalarized.
1490 void collectUniformsAndScalars(ElementCount VF) {
1491 // Do the analysis once.
1492 if (VF.isScalar() || Uniforms.find(VF) != Uniforms.end())
1493 return;
1494 setCostBasedWideningDecision(VF);
1495 collectLoopUniforms(VF);
1496 collectLoopScalars(VF);
1497 }
1498
1499 /// Returns true if the target machine supports masked store operation
1500 /// for the given \p DataType and kind of access to \p Ptr.
1501 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const {
1502 return Legal->isConsecutivePtr(DataType, Ptr) &&
1503 TTI.isLegalMaskedStore(DataType, Alignment);
1504 }
1505
1506 /// Returns true if the target machine supports masked load operation
1507 /// for the given \p DataType and kind of access to \p Ptr.
1508 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const {
1509 return Legal->isConsecutivePtr(DataType, Ptr) &&
1510 TTI.isLegalMaskedLoad(DataType, Alignment);
1511 }
1512
1513 /// Returns true if the target machine can represent \p V as a masked gather
1514 /// or scatter operation.
1515 bool isLegalGatherOrScatter(Value *V,
1516 ElementCount VF = ElementCount::getFixed(1)) {
1517 bool LI = isa<LoadInst>(V);
1518 bool SI = isa<StoreInst>(V);
1519 if (!LI && !SI)
1520 return false;
1521 auto *Ty = getLoadStoreType(V);
1522 Align Align = getLoadStoreAlignment(V);
1523 if (VF.isVector())
1524 Ty = VectorType::get(Ty, VF);
1525 return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
1526 (SI && TTI.isLegalMaskedScatter(Ty, Align));
1527 }
1528
1529 /// Returns true if the target machine supports all of the reduction
1530 /// variables found for the given VF.
1531 bool canVectorizeReductions(ElementCount VF) const {
1532 return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool {
1533 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1534 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1535 }));
1536 }
1537
1538 /// Returns true if \p I is an instruction that will be scalarized with
1539 /// predication when vectorizing \p I with vectorization factor \p VF. Such
1540 /// instructions include conditional stores and instructions that may divide
1541 /// by zero.
1542 bool isScalarWithPredication(Instruction *I, ElementCount VF) const;
1543
1544 // Returns true if \p I is an instruction that will be predicated either
1545 // through scalar predication or masked load/store or masked gather/scatter.
1546 // \p VF is the vectorization factor that will be used to vectorize \p I.
1547 // Superset of instructions that return true for isScalarWithPredication.
1548 bool isPredicatedInst(Instruction *I, ElementCount VF,
1549 bool IsKnownUniform = false) {
1550 // When we know the load is uniform and the original scalar loop was not
1551 // predicated we don't need to mark it as a predicated instruction. Any
1552 // vectorised blocks created when tail-folding are something artificial we
1553 // have introduced and we know there is always at least one active lane.
1554 // That's why we call Legal->blockNeedsPredication here because it doesn't
1555 // query tail-folding.
1556 if (IsKnownUniform && isa<LoadInst>(I) &&
1557 !Legal->blockNeedsPredication(I->getParent()))
1558 return false;
1559 if (!blockNeedsPredicationForAnyReason(I->getParent()))
1560 return false;
1561 // Loads and stores that need some form of masked operation are predicated
1562 // instructions.
1563 if (isa<LoadInst>(I) || isa<StoreInst>(I))
1564 return Legal->isMaskRequired(I);
1565 return isScalarWithPredication(I, VF);
1566 }
1567
1568 /// Returns true if \p I is a memory instruction with consecutive memory
1569 /// access that can be widened.
1570 bool
1571 memoryInstructionCanBeWidened(Instruction *I,
1572 ElementCount VF = ElementCount::getFixed(1));
1573
1574 /// Returns true if \p I is a memory instruction in an interleaved-group
1575 /// of memory accesses that can be vectorized with wide vector loads/stores
1576 /// and shuffles.
1577 bool
1578 interleavedAccessCanBeWidened(Instruction *I,
1579 ElementCount VF = ElementCount::getFixed(1));
1580
1581 /// Check if \p Instr belongs to any interleaved access group.
1582 bool isAccessInterleaved(Instruction *Instr) {
1583 return InterleaveInfo.isInterleaved(Instr);
1584 }
1585
1586 /// Get the interleaved access group that \p Instr belongs to.
1587 const InterleaveGroup<Instruction> *
1588 getInterleavedAccessGroup(Instruction *Instr) {
1589 return InterleaveInfo.getInterleaveGroup(Instr);
1590 }
1591
1592 /// Returns true if we're required to use a scalar epilogue for at least
1593 /// the final iteration of the original loop.
1594 bool requiresScalarEpilogue(ElementCount VF) const {
1595 if (!isScalarEpilogueAllowed())
1596 return false;
1597 // If we might exit from anywhere but the latch, must run the exiting
1598 // iteration in scalar form.
1599 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch())
1600 return true;
1601 return VF.isVector() && InterleaveInfo.requiresScalarEpilogue();
1602 }
1603
1604 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1605 /// loop hint annotation.
1606 bool isScalarEpilogueAllowed() const {
1607 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1608 }
1609
1610 /// Returns true if all loop blocks should be masked to fold tail loop.
1611 bool foldTailByMasking() const { return FoldTailByMasking; }
1612
1613 /// Returns true if the instructions in this block requires predication
1614 /// for any reason, e.g. because tail folding now requires a predicate
1615 /// or because the block in the original loop was predicated.
1616 bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const {
1617 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1618 }
1619
1620 /// A SmallMapVector to store the InLoop reduction op chains, mapping phi
1621 /// nodes to the chain of instructions representing the reductions. Uses a
1622 /// MapVector to ensure deterministic iteration order.
1623 using ReductionChainMap =
1624 SmallMapVector<PHINode *, SmallVector<Instruction *, 4>, 4>;
1625
1626 /// Return the chain of instructions representing an inloop reduction.
1627 const ReductionChainMap &getInLoopReductionChains() const {
1628 return InLoopReductionChains;
1629 }
1630
1631 /// Returns true if the Phi is part of an inloop reduction.
1632 bool isInLoopReduction(PHINode *Phi) const {
1633 return InLoopReductionChains.count(Phi);
1634 }
1635
1636 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1637 /// with factor VF. Return the cost of the instruction, including
1638 /// scalarization overhead if it's needed.
1639 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1640
1641 /// Estimate cost of a call instruction CI if it were vectorized with factor
1642 /// VF. Return the cost of the instruction, including scalarization overhead
1643 /// if it's needed. The flag NeedToScalarize shows if the call needs to be
1644 /// scalarized -
1645 /// i.e. either vector version isn't available, or is too expensive.
1646 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF,
1647 bool &NeedToScalarize) const;
1648
1649 /// Returns true if the per-lane cost of VectorizationFactor A is lower than
1650 /// that of B.
1651 bool isMoreProfitable(const VectorizationFactor &A,
1652 const VectorizationFactor &B) const;
1653
1654 /// Invalidates decisions already taken by the cost model.
1655 void invalidateCostModelingDecisions() {
1656 WideningDecisions.clear();
1657 Uniforms.clear();
1658 Scalars.clear();
1659 }
1660
1661private:
1662 unsigned NumPredStores = 0;
1663
1664 /// Convenience function that returns the value of vscale_range iff
1665 /// vscale_range.min == vscale_range.max or otherwise returns the value
1666 /// returned by the corresponding TLI method.
1667 Optional<unsigned> getVScaleForTuning() const;
1668
1669 /// \return An upper bound for the vectorization factors for both
1670 /// fixed and scalable vectorization, where the minimum-known number of
1671 /// elements is a power-of-2 larger than zero. If scalable vectorization is
1672 /// disabled or unsupported, then the scalable part will be equal to
1673 /// ElementCount::getScalable(0).
1674 FixedScalableVFPair computeFeasibleMaxVF(unsigned ConstTripCount,
1675 ElementCount UserVF,
1676 bool FoldTailByMasking);
1677
1678 /// \return the maximized element count based on the targets vector
1679 /// registers and the loop trip-count, but limited to a maximum safe VF.
1680 /// This is a helper function of computeFeasibleMaxVF.
1681 /// FIXME: MaxSafeVF is currently passed by reference to avoid some obscure
1682 /// issue that occurred on one of the buildbots which cannot be reproduced
1683 /// without having access to the properietary compiler (see comments on
1684 /// D98509). The issue is currently under investigation and this workaround
1685 /// will be removed as soon as possible.
1686 ElementCount getMaximizedVFForTarget(unsigned ConstTripCount,
1687 unsigned SmallestType,
1688 unsigned WidestType,
1689 const ElementCount &MaxSafeVF,
1690 bool FoldTailByMasking);
1691
1692 /// \return the maximum legal scalable VF, based on the safe max number
1693 /// of elements.
1694 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
1695
1696 /// The vectorization cost is a combination of the cost itself and a boolean
1697 /// indicating whether any of the contributing operations will actually
1698 /// operate on vector values after type legalization in the backend. If this
1699 /// latter value is false, then all operations will be scalarized (i.e. no
1700 /// vectorization has actually taken place).
1701 using VectorizationCostTy = std::pair<InstructionCost, bool>;
1702
1703 /// Returns the expected execution cost. The unit of the cost does
1704 /// not matter because we use the 'cost' units to compare different
1705 /// vector widths. The cost that is returned is *not* normalized by
1706 /// the factor width. If \p Invalid is not nullptr, this function
1707 /// will add a pair(Instruction*, ElementCount) to \p Invalid for
1708 /// each instruction that has an Invalid cost for the given VF.
1709 using InstructionVFPair = std::pair<Instruction *, ElementCount>;
1710 VectorizationCostTy
1711 expectedCost(ElementCount VF,
1712 SmallVectorImpl<InstructionVFPair> *Invalid = nullptr);
1713
1714 /// Returns the execution time cost of an instruction for a given vector
1715 /// width. Vector width of one means scalar.
1716 VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF);
1717
1718 /// The cost-computation logic from getInstructionCost which provides
1719 /// the vector type as an output parameter.
1720 InstructionCost getInstructionCost(Instruction *I, ElementCount VF,
1721 Type *&VectorTy);
1722
1723 /// Return the cost of instructions in an inloop reduction pattern, if I is
1724 /// part of that pattern.
1725 Optional<InstructionCost>
1726 getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy,
1727 TTI::TargetCostKind CostKind);
1728
1729 /// Calculate vectorization cost of memory instruction \p I.
1730 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1731
1732 /// The cost computation for scalarized memory instruction.
1733 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1734
1735 /// The cost computation for interleaving group of memory instructions.
1736 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1737
1738 /// The cost computation for Gather/Scatter instruction.
1739 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1740
1741 /// The cost computation for widening instruction \p I with consecutive
1742 /// memory access.
1743 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1744
1745 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1746 /// Load: scalar load + broadcast.
1747 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1748 /// element)
1749 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1750
1751 /// Estimate the overhead of scalarizing an instruction. This is a
1752 /// convenience wrapper for the type-based getScalarizationOverhead API.
1753 InstructionCost getScalarizationOverhead(Instruction *I,
1754 ElementCount VF) const;
1755
1756 /// Returns whether the instruction is a load or store and will be a emitted
1757 /// as a vector operation.
1758 bool isConsecutiveLoadOrStore(Instruction *I);
1759
1760 /// Returns true if an artificially high cost for emulated masked memrefs
1761 /// should be used.
1762 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
1763
1764 /// Map of scalar integer values to the smallest bitwidth they can be legally
1765 /// represented as. The vector equivalents of these values should be truncated
1766 /// to this type.
1767 MapVector<Instruction *, uint64_t> MinBWs;
1768
1769 /// A type representing the costs for instructions if they were to be
1770 /// scalarized rather than vectorized. The entries are Instruction-Cost
1771 /// pairs.
1772 using ScalarCostsTy = DenseMap<Instruction *, InstructionCost>;
1773
1774 /// A set containing all BasicBlocks that are known to present after
1775 /// vectorization as a predicated block.
1776 SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;
1777
1778 /// Records whether it is allowed to have the original scalar loop execute at
1779 /// least once. This may be needed as a fallback loop in case runtime
1780 /// aliasing/dependence checks fail, or to handle the tail/remainder
1781 /// iterations when the trip count is unknown or doesn't divide by the VF,
1782 /// or as a peel-loop to handle gaps in interleave-groups.
1783 /// Under optsize and when the trip count is very small we don't allow any
1784 /// iterations to execute in the scalar loop.
1785 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1786
1787 /// All blocks of loop are to be masked to fold tail of scalar iterations.
1788 bool FoldTailByMasking = false;
1789
1790 /// A map holding scalar costs for different vectorization factors. The
1791 /// presence of a cost for an instruction in the mapping indicates that the
1792 /// instruction will be scalarized when vectorizing with the associated
1793 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1794 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
1795
1796 /// Holds the instructions known to be uniform after vectorization.
1797 /// The data is collected per VF.
1798 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1799
1800 /// Holds the instructions known to be scalar after vectorization.
1801 /// The data is collected per VF.
1802 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1803
1804 /// Holds the instructions (address computations) that are forced to be
1805 /// scalarized.
1806 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1807
1808 /// PHINodes of the reductions that should be expanded in-loop along with
1809 /// their associated chains of reduction operations, in program order from top
1810 /// (PHI) to bottom
1811 ReductionChainMap InLoopReductionChains;
1812
1813 /// A Map of inloop reduction operations and their immediate chain operand.
1814 /// FIXME: This can be removed once reductions can be costed correctly in
1815 /// vplan. This was added to allow quick lookup to the inloop operations,
1816 /// without having to loop through InLoopReductionChains.
1817 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1818
1819 /// Returns the expected difference in cost from scalarizing the expression
1820 /// feeding a predicated instruction \p PredInst. The instructions to
1821 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1822 /// non-negative return value implies the expression will be scalarized.
1823 /// Currently, only single-use chains are considered for scalarization.
1824 int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,
1825 ElementCount VF);
1826
1827 /// Collect the instructions that are uniform after vectorization. An
1828 /// instruction is uniform if we represent it with a single scalar value in
1829 /// the vectorized loop corresponding to each vector iteration. Examples of
1830 /// uniform instructions include pointer operands of consecutive or
1831 /// interleaved memory accesses. Note that although uniformity implies an
1832 /// instruction will be scalar, the reverse is not true. In general, a
1833 /// scalarized instruction will be represented by VF scalar values in the
1834 /// vectorized loop, each corresponding to an iteration of the original
1835 /// scalar loop.
1836 void collectLoopUniforms(ElementCount VF);
1837
1838 /// Collect the instructions that are scalar after vectorization. An
1839 /// instruction is scalar if it is known to be uniform or will be scalarized
1840 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1841 /// to the list if they are used by a load/store instruction that is marked as
1842 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1843 /// VF values in the vectorized loop, each corresponding to an iteration of
1844 /// the original scalar loop.
1845 void collectLoopScalars(ElementCount VF);
1846
1847 /// Keeps cost model vectorization decision and cost for instructions.
1848 /// Right now it is used for memory instructions only.
1849 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1850 std::pair<InstWidening, InstructionCost>>;
1851
1852 DecisionList WideningDecisions;
1853
1854 /// Returns true if \p V is expected to be vectorized and it needs to be
1855 /// extracted.
1856 bool needsExtract(Value *V, ElementCount VF) const {
1857 Instruction *I = dyn_cast<Instruction>(V);
1858 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1859 TheLoop->isLoopInvariant(I))
1860 return false;
1861
1862 // Assume we can vectorize V (and hence we need extraction) if the
1863 // scalars are not computed yet. This can happen, because it is called
1864 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1865 // the scalars are collected. That should be a safe assumption in most
1866 // cases, because we check if the operands have vectorizable types
1867 // beforehand in LoopVectorizationLegality.
1868 return Scalars.find(VF) == Scalars.end() ||
1869 !isScalarAfterVectorization(I, VF);
1870 };
1871
1872 /// Returns a range containing only operands needing to be extracted.
1873 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1874 ElementCount VF) const {
1875 return SmallVector<Value *, 4>(make_filter_range(
1876 Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
1877 }
1878
1879 /// Determines if we have the infrastructure to vectorize loop \p L and its
1880 /// epilogue, assuming the main loop is vectorized by \p VF.
1881 bool isCandidateForEpilogueVectorization(const Loop &L,
1882 const ElementCount VF) const;
1883
1884 /// Returns true if epilogue vectorization is considered profitable, and
1885 /// false otherwise.
1886 /// \p VF is the vectorization factor chosen for the original loop.
1887 bool isEpilogueVectorizationProfitable(const ElementCount VF) const;
1888
1889public:
1890 /// The loop that we evaluate.
1891 Loop *TheLoop;
1892
1893 /// Predicated scalar evolution analysis.
1894 PredicatedScalarEvolution &PSE;
1895
1896 /// Loop Info analysis.
1897 LoopInfo *LI;
1898
1899 /// Vectorization legality.
1900 LoopVectorizationLegality *Legal;
1901
1902 /// Vector target information.
1903 const TargetTransformInfo &TTI;
1904
1905 /// Target Library Info.
1906 const TargetLibraryInfo *TLI;
1907
1908 /// Demanded bits analysis.
1909 DemandedBits *DB;
1910
1911 /// Assumption cache.
1912 AssumptionCache *AC;
1913
1914 /// Interface to emit optimization remarks.
1915 OptimizationRemarkEmitter *ORE;
1916
1917 const Function *TheFunction;
1918
1919 /// Loop Vectorize Hint.
1920 const LoopVectorizeHints *Hints;
1921
1922 /// The interleave access information contains groups of interleaved accesses
1923 /// with the same stride and close to each other.
1924 InterleavedAccessInfo &InterleaveInfo;
1925
1926 /// Values to ignore in the cost model.
1927 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1928
1929 /// Values to ignore in the cost model when VF > 1.
1930 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1931
1932 /// All element types found in the loop.
1933 SmallPtrSet<Type *, 16> ElementTypesInLoop;
1934
1935 /// Profitable vector factors.
1936 SmallVector<VectorizationFactor, 8> ProfitableVFs;
1937};
1938} // end namespace llvm
1939
1940/// Helper struct to manage generating runtime checks for vectorization.
1941///
1942/// The runtime checks are created up-front in temporary blocks to allow better
1943/// estimating the cost and un-linked from the existing IR. After deciding to
1944/// vectorize, the checks are moved back. If deciding not to vectorize, the
1945/// temporary blocks are completely removed.
1946class GeneratedRTChecks {
1947 /// Basic block which contains the generated SCEV checks, if any.
1948 BasicBlock *SCEVCheckBlock = nullptr;
1949
1950 /// The value representing the result of the generated SCEV checks. If it is
1951 /// nullptr, either no SCEV checks have been generated or they have been used.
1952 Value *SCEVCheckCond = nullptr;
1953
1954 /// Basic block which contains the generated memory runtime checks, if any.
1955 BasicBlock *MemCheckBlock = nullptr;
1956
1957 /// The value representing the result of the generated memory runtime checks.
1958 /// If it is nullptr, either no memory runtime checks have been generated or
1959 /// they have been used.
1960 Value *MemRuntimeCheckCond = nullptr;
1961
1962 DominatorTree *DT;
1963 LoopInfo *LI;
1964
1965 SCEVExpander SCEVExp;
1966 SCEVExpander MemCheckExp;
1967
1968public:
1969 GeneratedRTChecks(ScalarEvolution &SE, DominatorTree *DT, LoopInfo *LI,
1970 const DataLayout &DL)
1971 : DT(DT), LI(LI), SCEVExp(SE, DL, "scev.check"),
1972 MemCheckExp(SE, DL, "scev.check") {}
1973
1974 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1975 /// accurately estimate the cost of the runtime checks. The blocks are
1976 /// un-linked from the IR and is added back during vector code generation. If
1977 /// there is no vector code generation, the check blocks are removed
1978 /// completely.
1979 void Create(Loop *L, const LoopAccessInfo &LAI,
1980 const SCEVPredicate &Pred) {
1981
1982 BasicBlock *LoopHeader = L->getHeader();
1983 BasicBlock *Preheader = L->getLoopPreheader();
1984
1985 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1986 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1987 // may be used by SCEVExpander. The blocks will be un-linked from their
1988 // predecessors and removed from LI & DT at the end of the function.
1989 if (!Pred.isAlwaysTrue()) {
1990 SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1991 nullptr, "vector.scevcheck");
1992
1993 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1994 &Pred, SCEVCheckBlock->getTerminator());
1995 }
1996
1997 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1998 if (RtPtrChecking.Need) {
1999 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
2000 MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
2001 "vector.memcheck");
2002
2003 MemRuntimeCheckCond =
2004 addRuntimeChecks(MemCheckBlock->getTerminator(), L,
2005 RtPtrChecking.getChecks(), MemCheckExp);
2006 assert(MemRuntimeCheckCond &&(static_cast <bool> (MemRuntimeCheckCond && "no RT checks generated although RtPtrChecking "
"claimed checks are required") ? void (0) : __assert_fail ("MemRuntimeCheckCond && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2008, __extension__
__PRETTY_FUNCTION__))
2007 "no RT checks generated although RtPtrChecking "(static_cast <bool> (MemRuntimeCheckCond && "no RT checks generated although RtPtrChecking "
"claimed checks are required") ? void (0) : __assert_fail ("MemRuntimeCheckCond && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2008, __extension__
__PRETTY_FUNCTION__))
2008 "claimed checks are required")(static_cast <bool> (MemRuntimeCheckCond && "no RT checks generated although RtPtrChecking "
"claimed checks are required") ? void (0) : __assert_fail ("MemRuntimeCheckCond && \"no RT checks generated although RtPtrChecking \" \"claimed checks are required\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2008, __extension__
__PRETTY_FUNCTION__))
;
2009 }
2010
2011 if (!MemCheckBlock && !SCEVCheckBlock)
2012 return;
2013
2014 // Unhook the temporary block with the checks, update various places
2015 // accordingly.
2016 if (SCEVCheckBlock)
2017 SCEVCheckBlock->replaceAllUsesWith(Preheader);
2018 if (MemCheckBlock)
2019 MemCheckBlock->replaceAllUsesWith(Preheader);
2020
2021 if (SCEVCheckBlock) {
2022 SCEVCheckBlock->getTerminator()->moveBefore(Preheader->getTerminator());
2023 new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
2024 Preheader->getTerminator()->eraseFromParent();
2025 }
2026 if (MemCheckBlock) {
2027 MemCheckBlock->getTerminator()->moveBefore(Preheader->getTerminator());
2028 new UnreachableInst(Preheader->getContext(), MemCheckBlock);
2029 Preheader->getTerminator()->eraseFromParent();
2030 }
2031
2032 DT->changeImmediateDominator(LoopHeader, Preheader);
2033 if (MemCheckBlock) {
2034 DT->eraseNode(MemCheckBlock);
2035 LI->removeBlock(MemCheckBlock);
2036 }
2037 if (SCEVCheckBlock) {
2038 DT->eraseNode(SCEVCheckBlock);
2039 LI->removeBlock(SCEVCheckBlock);
2040 }
2041 }
2042
2043 /// Remove the created SCEV & memory runtime check blocks & instructions, if
2044 /// unused.
2045 ~GeneratedRTChecks() {
2046 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
2047 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
2048 if (!SCEVCheckCond)
2049 SCEVCleaner.markResultUsed();
2050
2051 if (!MemRuntimeCheckCond)
2052 MemCheckCleaner.markResultUsed();
2053
2054 if (MemRuntimeCheckCond) {
2055 auto &SE = *MemCheckExp.getSE();
2056 // Memory runtime check generation creates compares that use expanded
2057 // values. Remove them before running the SCEVExpanderCleaners.
2058 for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
2059 if (MemCheckExp.isInsertedInstruction(&I))
2060 continue;
2061 SE.forgetValue(&I);
2062 I.eraseFromParent();
2063 }
2064 }
2065 MemCheckCleaner.cleanup();
2066 SCEVCleaner.cleanup();
2067
2068 if (SCEVCheckCond)
2069 SCEVCheckBlock->eraseFromParent();
2070 if (MemRuntimeCheckCond)
2071 MemCheckBlock->eraseFromParent();
2072 }
2073
2074 /// Adds the generated SCEVCheckBlock before \p LoopVectorPreHeader and
2075 /// adjusts the branches to branch to the vector preheader or \p Bypass,
2076 /// depending on the generated condition.
2077 BasicBlock *emitSCEVChecks(BasicBlock *Bypass,
2078 BasicBlock *LoopVectorPreHeader,
2079 BasicBlock *LoopExitBlock) {
2080 if (!SCEVCheckCond)
2081 return nullptr;
2082 if (auto *C = dyn_cast<ConstantInt>(SCEVCheckCond))
2083 if (C->isZero())
2084 return nullptr;
2085
2086 auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
2087
2088 BranchInst::Create(LoopVectorPreHeader, SCEVCheckBlock);
2089 // Create new preheader for vector loop.
2090 if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
2091 PL->addBasicBlockToLoop(SCEVCheckBlock, *LI);
2092
2093 SCEVCheckBlock->getTerminator()->eraseFromParent();
2094 SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
2095 Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
2096 SCEVCheckBlock);
2097
2098 DT->addNewBlock(SCEVCheckBlock, Pred);
2099 DT->changeImmediateDominator(LoopVectorPreHeader, SCEVCheckBlock);
2100
2101 ReplaceInstWithInst(
2102 SCEVCheckBlock->getTerminator(),
2103 BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheckCond));
2104 // Mark the check as used, to prevent it from being removed during cleanup.
2105 SCEVCheckCond = nullptr;
2106 return SCEVCheckBlock;
2107 }
2108
2109 /// Adds the generated MemCheckBlock before \p LoopVectorPreHeader and adjusts
2110 /// the branches to branch to the vector preheader or \p Bypass, depending on
2111 /// the generated condition.
2112 BasicBlock *emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass,
2113 BasicBlock *LoopVectorPreHeader) {
2114 // Check if we generated code that checks in runtime if arrays overlap.
2115 if (!MemRuntimeCheckCond)
2116 return nullptr;
2117
2118 auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
2119 Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
2120 MemCheckBlock);
2121
2122 DT->addNewBlock(MemCheckBlock, Pred);
2123 DT->changeImmediateDominator(LoopVectorPreHeader, MemCheckBlock);
2124 MemCheckBlock->moveBefore(LoopVectorPreHeader);
2125
2126 if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
2127 PL->addBasicBlockToLoop(MemCheckBlock, *LI);
2128
2129 ReplaceInstWithInst(
2130 MemCheckBlock->getTerminator(),
2131 BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond));
2132 MemCheckBlock->getTerminator()->setDebugLoc(
2133 Pred->getTerminator()->getDebugLoc());
2134
2135 // Mark the check as used, to prevent it from being removed during cleanup.
2136 MemRuntimeCheckCond = nullptr;
2137 return MemCheckBlock;
2138 }
2139};
2140
2141// Return true if \p OuterLp is an outer loop annotated with hints for explicit
2142// vectorization. The loop needs to be annotated with #pragma omp simd
2143// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
2144// vector length information is not provided, vectorization is not considered
2145// explicit. Interleave hints are not allowed either. These limitations will be
2146// relaxed in the future.
2147// Please, note that we are currently forced to abuse the pragma 'clang
2148// vectorize' semantics. This pragma provides *auto-vectorization hints*
2149// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
2150// provides *explicit vectorization hints* (LV can bypass legal checks and
2151// assume that vectorization is legal). However, both hints are implemented
2152// using the same metadata (llvm.loop.vectorize, processed by
2153// LoopVectorizeHints). This will be fixed in the future when the native IR
2154// representation for pragma 'omp simd' is introduced.
2155static bool isExplicitVecOuterLoop(Loop *OuterLp,
2156 OptimizationRemarkEmitter *ORE) {
2157 assert(!OuterLp->isInnermost() && "This is not an outer loop")(static_cast <bool> (!OuterLp->isInnermost() &&
"This is not an outer loop") ? void (0) : __assert_fail ("!OuterLp->isInnermost() && \"This is not an outer loop\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2157, __extension__
__PRETTY_FUNCTION__))
;
2158 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
2159
2160 // Only outer loops with an explicit vectorization hint are supported.
2161 // Unannotated outer loops are ignored.
2162 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
2163 return false;
2164
2165 Function *Fn = OuterLp->getHeader()->getParent();
2166 if (!Hints.allowVectorization(Fn, OuterLp,
2167 true /*VectorizeOnlyWhenForced*/)) {
2168 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Loop hints prevent outer loop vectorization.\n"
; } } while (false)
;
2169 return false;
2170 }
2171
2172 if (Hints.getInterleave() > 1) {
2173 // TODO: Interleave support is future work.
2174 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n"; } } while (false)
2175 "outer loops.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not vectorizing: Interleave is not supported for "
"outer loops.\n"; } } while (false)
;
2176 Hints.emitRemarkWithHints();
2177 return false;
2178 }
2179
2180 return true;
2181}
2182
2183static void collectSupportedLoops(Loop &L, LoopInfo *LI,
2184 OptimizationRemarkEmitter *ORE,
2185 SmallVectorImpl<Loop *> &V) {
2186 // Collect inner loops and outer loops without irreducible control flow. For
2187 // now, only collect outer loops that have explicit vectorization hints. If we
2188 // are stress testing the VPlan H-CFG construction, we collect the outermost
2189 // loop of every loop nest.
2190 if (L.isInnermost() || VPlanBuildStressTest ||
2191 (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
2192 LoopBlocksRPO RPOT(&L);
2193 RPOT.perform(LI);
2194 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
2195 V.push_back(&L);
2196 // TODO: Collect inner loops inside marked outer loops in case
2197 // vectorization fails for the outer loop. Do not invoke
2198 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
2199 // already known to be reducible. We can use an inherited attribute for
2200 // that.
2201 return;
2202 }
2203 }
2204 for (Loop *InnerL : L)
2205 collectSupportedLoops(*InnerL, LI, ORE, V);
2206}
2207
2208namespace {
2209
2210/// The LoopVectorize Pass.
2211struct LoopVectorize : public FunctionPass {
2212 /// Pass identification, replacement for typeid
2213 static char ID;
2214
2215 LoopVectorizePass Impl;
2216
2217 explicit LoopVectorize(bool InterleaveOnlyWhenForced = false,
2218 bool VectorizeOnlyWhenForced = false)
2219 : FunctionPass(ID),
2220 Impl({InterleaveOnlyWhenForced, VectorizeOnlyWhenForced}) {
2221 initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
2222 }
2223
2224 bool runOnFunction(Function &F) override {
2225 if (skipFunction(F))
2226 return false;
2227
2228 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
2229 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
2230 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2231 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
2232 auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
2233 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
2234 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
2235 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2236 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
2237 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
2238 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
2239 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
2240 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
2241
2242 std::function<const LoopAccessInfo &(Loop &)> GetLAA =
2243 [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };
2244
2245 return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,
2246 GetLAA, *ORE, PSI).MadeAnyChange;
2247 }
2248
2249 void getAnalysisUsage(AnalysisUsage &AU) const override {
2250 AU.addRequired<AssumptionCacheTracker>();
2251 AU.addRequired<BlockFrequencyInfoWrapperPass>();
2252 AU.addRequired<DominatorTreeWrapperPass>();
2253 AU.addRequired<LoopInfoWrapperPass>();
2254 AU.addRequired<ScalarEvolutionWrapperPass>();
2255 AU.addRequired<TargetTransformInfoWrapperPass>();
2256 AU.addRequired<AAResultsWrapperPass>();
2257 AU.addRequired<LoopAccessLegacyAnalysis>();
2258 AU.addRequired<DemandedBitsWrapperPass>();
2259 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
2260 AU.addRequired<InjectTLIMappingsLegacy>();
2261
2262 // We currently do not preserve loopinfo/dominator analyses with outer loop
2263 // vectorization. Until this is addressed, mark these analyses as preserved
2264 // only for non-VPlan-native path.
2265 // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
2266 if (!EnableVPlanNativePath) {
2267 AU.addPreserved<LoopInfoWrapperPass>();
2268 AU.addPreserved<DominatorTreeWrapperPass>();
2269 }
2270
2271 AU.addPreserved<BasicAAWrapperPass>();
2272 AU.addPreserved<GlobalsAAWrapperPass>();
2273 AU.addRequired<ProfileSummaryInfoWrapperPass>();
2274 }
2275};
2276
2277} // end anonymous namespace
2278
2279//===----------------------------------------------------------------------===//
2280// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2281// LoopVectorizationCostModel and LoopVectorizationPlanner.
2282//===----------------------------------------------------------------------===//
2283
2284Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
2285 // We need to place the broadcast of invariant variables outside the loop,
2286 // but only if it's proven safe to do so. Else, broadcast will be inside
2287 // vector loop body.
2288 Instruction *Instr = dyn_cast<Instruction>(V);
2289 bool SafeToHoist = OrigLoop->isLoopInvariant(V) &&
2290 (!Instr ||
2291 DT->dominates(Instr->getParent(), LoopVectorPreHeader));
2292 // Place the code for broadcasting invariant variables in the new preheader.
2293 IRBuilder<>::InsertPointGuard Guard(Builder);
2294 if (SafeToHoist)
2295 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
2296
2297 // Broadcast the scalar into all locations in the vector.
2298 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
2299
2300 return Shuf;
2301}
2302
2303/// This function adds
2304/// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
2305/// to each vector element of Val. The sequence starts at StartIndex.
2306/// \p Opcode is relevant for FP induction variable.
2307static Value *getStepVector(Value *Val, Value *StartIdx, Value *Step,
2308 Instruction::BinaryOps BinOp, ElementCount VF,
2309 IRBuilderBase &Builder) {
2310 assert(VF.isVector() && "only vector VFs are supported")(static_cast <bool> (VF.isVector() && "only vector VFs are supported"
) ? void (0) : __assert_fail ("VF.isVector() && \"only vector VFs are supported\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2310, __extension__
__PRETTY_FUNCTION__))
;
2311
2312 // Create and check the types.
2313 auto *ValVTy = cast<VectorType>(Val->getType());
2314 ElementCount VLen = ValVTy->getElementCount();
2315
2316 Type *STy = Val->getType()->getScalarType();
2317 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&(static_cast <bool> ((STy->isIntegerTy() || STy->
isFloatingPointTy()) && "Induction Step must be an integer or FP"
) ? void (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2318, __extension__
__PRETTY_FUNCTION__))
2318 "Induction Step must be an integer or FP")(static_cast <bool> ((STy->isIntegerTy() || STy->
isFloatingPointTy()) && "Induction Step must be an integer or FP"
) ? void (0) : __assert_fail ("(STy->isIntegerTy() || STy->isFloatingPointTy()) && \"Induction Step must be an integer or FP\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2318, __extension__
__PRETTY_FUNCTION__))
;
2319 assert(Step->getType() == STy && "Step has wrong type")(static_cast <bool> (Step->getType() == STy &&
"Step has wrong type") ? void (0) : __assert_fail ("Step->getType() == STy && \"Step has wrong type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2319, __extension__
__PRETTY_FUNCTION__))
;
2320
2321 SmallVector<Constant *, 8> Indices;
2322
2323 // Create a vector of consecutive numbers from zero to VF.
2324 VectorType *InitVecValVTy = ValVTy;
2325 if (STy->isFloatingPointTy()) {
2326 Type *InitVecValSTy =
2327 IntegerType::get(STy->getContext(), STy->getScalarSizeInBits());
2328 InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
2329 }
2330 Value *InitVec = Builder.CreateStepVector(InitVecValVTy);
2331
2332 // Splat the StartIdx
2333 Value *StartIdxSplat = Builder.CreateVectorSplat(VLen, StartIdx);
2334
2335 if (STy->isIntegerTy()) {
2336 InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);
2337 Step = Builder.CreateVectorSplat(VLen, Step);
2338 assert(Step->getType() == Val->getType() && "Invalid step vec")(static_cast <bool> (Step->getType() == Val->getType
() && "Invalid step vec") ? void (0) : __assert_fail (
"Step->getType() == Val->getType() && \"Invalid step vec\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2338, __extension__
__PRETTY_FUNCTION__))
;
2339 // FIXME: The newly created binary instructions should contain nsw/nuw
2340 // flags, which can be found from the original scalar operations.
2341 Step = Builder.CreateMul(InitVec, Step);
2342 return Builder.CreateAdd(Val, Step, "induction");
2343 }
2344
2345 // Floating point induction.
2346 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&(static_cast <bool> ((BinOp == Instruction::FAdd || BinOp
== Instruction::FSub) && "Binary Opcode should be specified for FP induction"
) ? void (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2347, __extension__
__PRETTY_FUNCTION__))
2347 "Binary Opcode should be specified for FP induction")(static_cast <bool> ((BinOp == Instruction::FAdd || BinOp
== Instruction::FSub) && "Binary Opcode should be specified for FP induction"
) ? void (0) : __assert_fail ("(BinOp == Instruction::FAdd || BinOp == Instruction::FSub) && \"Binary Opcode should be specified for FP induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2347, __extension__
__PRETTY_FUNCTION__))
;
2348 InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
2349 InitVec = Builder.CreateFAdd(InitVec, StartIdxSplat);
2350
2351 Step = Builder.CreateVectorSplat(VLen, Step);
2352 Value *MulOp = Builder.CreateFMul(InitVec, Step);
2353 return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
2354}
2355
2356/// Compute scalar induction steps. \p ScalarIV is the scalar induction
2357/// variable on which to base the steps, \p Step is the size of the step.
2358static void buildScalarSteps(Value *ScalarIV, Value *Step,
2359 const InductionDescriptor &ID, VPValue *Def,
2360 VPTransformState &State) {
2361 IRBuilderBase &Builder = State.Builder;
2362 // We shouldn't have to build scalar steps if we aren't vectorizing.
2363 assert(State.VF.isVector() && "VF should be greater than one")(static_cast <bool> (State.VF.isVector() && "VF should be greater than one"
) ? void (0) : __assert_fail ("State.VF.isVector() && \"VF should be greater than one\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2363, __extension__
__PRETTY_FUNCTION__))
;
2364 // Get the value type and ensure it and the step have the same integer type.
2365 Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
2366 assert(ScalarIVTy == Step->getType() &&(static_cast <bool> (ScalarIVTy == Step->getType() &&
"Val and Step should have the same type") ? void (0) : __assert_fail
("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2367, __extension__
__PRETTY_FUNCTION__))
2367 "Val and Step should have the same type")(static_cast <bool> (ScalarIVTy == Step->getType() &&
"Val and Step should have the same type") ? void (0) : __assert_fail
("ScalarIVTy == Step->getType() && \"Val and Step should have the same type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2367, __extension__
__PRETTY_FUNCTION__))
;
2368
2369 // We build scalar steps for both integer and floating-point induction
2370 // variables. Here, we determine the kind of arithmetic we will perform.
2371 Instruction::BinaryOps AddOp;
2372 Instruction::BinaryOps MulOp;
2373 if (ScalarIVTy->isIntegerTy()) {
2374 AddOp = Instruction::Add;
2375 MulOp = Instruction::Mul;
2376 } else {
2377 AddOp = ID.getInductionOpcode();
2378 MulOp = Instruction::FMul;
2379 }
2380
2381 // Determine the number of scalars we need to generate for each unroll
2382 // iteration.
2383 bool FirstLaneOnly = vputils::onlyFirstLaneUsed(Def);
2384 unsigned Lanes = FirstLaneOnly ? 1 : State.VF.getKnownMinValue();
2385 // Compute the scalar steps and save the results in State.
2386 Type *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
2387 ScalarIVTy->getScalarSizeInBits());
2388 Type *VecIVTy = nullptr;
2389 Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;
2390 if (!FirstLaneOnly && State.VF.isScalable()) {
2391 VecIVTy = VectorType::get(ScalarIVTy, State.VF);
2392 UnitStepVec =
2393 Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF));
2394 SplatStep = Builder.CreateVectorSplat(State.VF, Step);
2395 SplatIV = Builder.CreateVectorSplat(State.VF, ScalarIV);
2396 }
2397
2398 for (unsigned Part = 0; Part < State.UF; ++Part) {
2399 Value *StartIdx0 = createStepForVF(Builder, IntStepTy, State.VF, Part);
2400
2401 if (!FirstLaneOnly && State.VF.isScalable()) {
2402 auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0);
2403 auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
2404 if (ScalarIVTy->isFloatingPointTy())
2405 InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
2406 auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
2407 auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
2408 State.set(Def, Add, Part);
2409 // It's useful to record the lane values too for the known minimum number
2410 // of elements so we do those below. This improves the code quality when
2411 // trying to extract the first element, for example.
2412 }
2413
2414 if (ScalarIVTy->isFloatingPointTy())
2415 StartIdx0 = Builder.CreateSIToFP(StartIdx0, ScalarIVTy);
2416
2417 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2418 Value *StartIdx = Builder.CreateBinOp(
2419 AddOp, StartIdx0, getSignedIntOrFpConstant(ScalarIVTy, Lane));
2420 // The step returned by `createStepForVF` is a runtime-evaluated value
2421 // when VF is scalable. Otherwise, it should be folded into a Constant.
2422 assert((State.VF.isScalable() || isa<Constant>(StartIdx)) &&(static_cast <bool> ((State.VF.isScalable() || isa<Constant
>(StartIdx)) && "Expected StartIdx to be folded to a constant when VF is not "
"scalable") ? void (0) : __assert_fail ("(State.VF.isScalable() || isa<Constant>(StartIdx)) && \"Expected StartIdx to be folded to a constant when VF is not \" \"scalable\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2424, __extension__
__PRETTY_FUNCTION__))
2423 "Expected StartIdx to be folded to a constant when VF is not "(static_cast <bool> ((State.VF.isScalable() || isa<Constant
>(StartIdx)) && "Expected StartIdx to be folded to a constant when VF is not "
"scalable") ? void (0) : __assert_fail ("(State.VF.isScalable() || isa<Constant>(StartIdx)) && \"Expected StartIdx to be folded to a constant when VF is not \" \"scalable\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2424, __extension__
__PRETTY_FUNCTION__))
2424 "scalable")(static_cast <bool> ((State.VF.isScalable() || isa<Constant
>(StartIdx)) && "Expected StartIdx to be folded to a constant when VF is not "
"scalable") ? void (0) : __assert_fail ("(State.VF.isScalable() || isa<Constant>(StartIdx)) && \"Expected StartIdx to be folded to a constant when VF is not \" \"scalable\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2424, __extension__
__PRETTY_FUNCTION__))
;
2425 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
2426 auto *Add = Builder.CreateBinOp(AddOp, ScalarIV, Mul);
2427 State.set(Def, Add, VPIteration(Part, Lane));
2428 }
2429 }
2430}
2431
2432// Generate code for the induction step. Note that induction steps are
2433// required to be loop-invariant
2434static Value *CreateStepValue(const SCEV *Step, ScalarEvolution &SE,
2435 Instruction *InsertBefore,
2436 Loop *OrigLoop = nullptr) {
2437 const DataLayout &DL = SE.getDataLayout();
2438 assert((!OrigLoop || SE.isLoopInvariant(Step, OrigLoop)) &&(static_cast <bool> ((!OrigLoop || SE.isLoopInvariant(Step
, OrigLoop)) && "Induction step should be loop invariant"
) ? void (0) : __assert_fail ("(!OrigLoop || SE.isLoopInvariant(Step, OrigLoop)) && \"Induction step should be loop invariant\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2439, __extension__
__PRETTY_FUNCTION__))
2439 "Induction step should be loop invariant")(static_cast <bool> ((!OrigLoop || SE.isLoopInvariant(Step
, OrigLoop)) && "Induction step should be loop invariant"
) ? void (0) : __assert_fail ("(!OrigLoop || SE.isLoopInvariant(Step, OrigLoop)) && \"Induction step should be loop invariant\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2439, __extension__
__PRETTY_FUNCTION__))
;
2440 if (auto *E = dyn_cast<SCEVUnknown>(Step))
2441 return E->getValue();
2442
2443 SCEVExpander Exp(SE, DL, "induction");
2444 return Exp.expandCodeFor(Step, Step->getType(), InsertBefore);
2445}
2446
2447/// Compute the transformed value of Index at offset StartValue using step
2448/// StepValue.
2449/// For integer induction, returns StartValue + Index * StepValue.
2450/// For pointer induction, returns StartValue[Index * StepValue].
2451/// FIXME: The newly created binary instructions should contain nsw/nuw
2452/// flags, which can be found from the original scalar operations.
2453static Value *emitTransformedIndex(IRBuilderBase &B, Value *Index,
2454 Value *StartValue, Value *Step,
2455 const InductionDescriptor &ID) {
2456 assert(Index->getType()->getScalarType() == Step->getType() &&(static_cast <bool> (Index->getType()->getScalarType
() == Step->getType() && "Index scalar type does not match StepValue type"
) ? void (0) : __assert_fail ("Index->getType()->getScalarType() == Step->getType() && \"Index scalar type does not match StepValue type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2457, __extension__
__PRETTY_FUNCTION__))
2457 "Index scalar type does not match StepValue type")(static_cast <bool> (Index->getType()->getScalarType
() == Step->getType() && "Index scalar type does not match StepValue type"
) ? void (0) : __assert_fail ("Index->getType()->getScalarType() == Step->getType() && \"Index scalar type does not match StepValue type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2457, __extension__
__PRETTY_FUNCTION__))
;
2458
2459 // Note: the IR at this point is broken. We cannot use SE to create any new
2460 // SCEV and then expand it, hoping that SCEV's simplification will give us
2461 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
2462 // lead to various SCEV crashes. So all we can do is to use builder and rely
2463 // on InstCombine for future simplifications. Here we handle some trivial
2464 // cases only.
2465 auto CreateAdd = [&B](Value *X, Value *Y) {
2466 assert(X->getType() == Y->getType() && "Types don't match!")(static_cast <bool> (X->getType() == Y->getType()
&& "Types don't match!") ? void (0) : __assert_fail (
"X->getType() == Y->getType() && \"Types don't match!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2466, __extension__
__PRETTY_FUNCTION__))
;
2467 if (auto *CX = dyn_cast<ConstantInt>(X))
2468 if (CX->isZero())
2469 return Y;
2470 if (auto *CY = dyn_cast<ConstantInt>(Y))
2471 if (CY->isZero())
2472 return X;
2473 return B.CreateAdd(X, Y);
2474 };
2475
2476 // We allow X to be a vector type, in which case Y will potentially be
2477 // splatted into a vector with the same element count.
2478 auto CreateMul = [&B](Value *X, Value *Y) {
2479 assert(X->getType()->getScalarType() == Y->getType() &&(static_cast <bool> (X->getType()->getScalarType(
) == Y->getType() && "Types don't match!") ? void (
0) : __assert_fail ("X->getType()->getScalarType() == Y->getType() && \"Types don't match!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2480, __extension__
__PRETTY_FUNCTION__))
2480 "Types don't match!")(static_cast <bool> (X->getType()->getScalarType(
) == Y->getType() && "Types don't match!") ? void (
0) : __assert_fail ("X->getType()->getScalarType() == Y->getType() && \"Types don't match!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2480, __extension__
__PRETTY_FUNCTION__))
;
2481 if (auto *CX = dyn_cast<ConstantInt>(X))
2482 if (CX->isOne())
2483 return Y;
2484 if (auto *CY = dyn_cast<ConstantInt>(Y))
2485 if (CY->isOne())
2486 return X;
2487 VectorType *XVTy = dyn_cast<VectorType>(X->getType());
2488 if (XVTy && !isa<VectorType>(Y->getType()))
2489 Y = B.CreateVectorSplat(XVTy->getElementCount(), Y);
2490 return B.CreateMul(X, Y);
2491 };
2492
2493 switch (ID.getKind()) {
2494 case InductionDescriptor::IK_IntInduction: {
2495 assert(!isa<VectorType>(Index->getType()) &&(static_cast <bool> (!isa<VectorType>(Index->getType
()) && "Vector indices not supported for integer inductions yet"
) ? void (0) : __assert_fail ("!isa<VectorType>(Index->getType()) && \"Vector indices not supported for integer inductions yet\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2496, __extension__
__PRETTY_FUNCTION__))
2496 "Vector indices not supported for integer inductions yet")(static_cast <bool> (!isa<VectorType>(Index->getType
()) && "Vector indices not supported for integer inductions yet"
) ? void (0) : __assert_fail ("!isa<VectorType>(Index->getType()) && \"Vector indices not supported for integer inductions yet\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2496, __extension__
__PRETTY_FUNCTION__))
;
2497 assert(Index->getType() == StartValue->getType() &&(static_cast <bool> (Index->getType() == StartValue->
getType() && "Index type does not match StartValue type"
) ? void (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2498, __extension__
__PRETTY_FUNCTION__))
2498 "Index type does not match StartValue type")(static_cast <bool> (Index->getType() == StartValue->
getType() && "Index type does not match StartValue type"
) ? void (0) : __assert_fail ("Index->getType() == StartValue->getType() && \"Index type does not match StartValue type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2498, __extension__
__PRETTY_FUNCTION__))
;
2499 if (isa<ConstantInt>(Step) && cast<ConstantInt>(Step)->isMinusOne())
2500 return B.CreateSub(StartValue, Index);
2501 auto *Offset = CreateMul(Index, Step);
2502 return CreateAdd(StartValue, Offset);
2503 }
2504 case InductionDescriptor::IK_PtrInduction: {
2505 assert(isa<Constant>(Step) &&(static_cast <bool> (isa<Constant>(Step) &&
"Expected constant step for pointer induction") ? void (0) :
__assert_fail ("isa<Constant>(Step) && \"Expected constant step for pointer induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2506, __extension__
__PRETTY_FUNCTION__))
2506 "Expected constant step for pointer induction")(static_cast <bool> (isa<Constant>(Step) &&
"Expected constant step for pointer induction") ? void (0) :
__assert_fail ("isa<Constant>(Step) && \"Expected constant step for pointer induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2506, __extension__
__PRETTY_FUNCTION__))
;
2507 return B.CreateGEP(ID.getElementType(), StartValue, CreateMul(Index, Step));
2508 }
2509 case InductionDescriptor::IK_FpInduction: {
2510 assert(!isa<VectorType>(Index->getType()) &&(static_cast <bool> (!isa<VectorType>(Index->getType
()) && "Vector indices not supported for FP inductions yet"
) ? void (0) : __assert_fail ("!isa<VectorType>(Index->getType()) && \"Vector indices not supported for FP inductions yet\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2511, __extension__
__PRETTY_FUNCTION__))
2511 "Vector indices not supported for FP inductions yet")(static_cast <bool> (!isa<VectorType>(Index->getType
()) && "Vector indices not supported for FP inductions yet"
) ? void (0) : __assert_fail ("!isa<VectorType>(Index->getType()) && \"Vector indices not supported for FP inductions yet\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2511, __extension__
__PRETTY_FUNCTION__))
;
2512 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value")(static_cast <bool> (Step->getType()->isFloatingPointTy
() && "Expected FP Step value") ? void (0) : __assert_fail
("Step->getType()->isFloatingPointTy() && \"Expected FP Step value\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2512, __extension__
__PRETTY_FUNCTION__))
;
2513 auto InductionBinOp = ID.getInductionBinOp();
2514 assert(InductionBinOp &&(static_cast <bool> (InductionBinOp && (InductionBinOp
->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode
() == Instruction::FSub) && "Original bin op should be defined for FP induction"
) ? void (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2517, __extension__
__PRETTY_FUNCTION__))
2515 (InductionBinOp->getOpcode() == Instruction::FAdd ||(static_cast <bool> (InductionBinOp && (InductionBinOp
->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode
() == Instruction::FSub) && "Original bin op should be defined for FP induction"
) ? void (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2517, __extension__
__PRETTY_FUNCTION__))
2516 InductionBinOp->getOpcode() == Instruction::FSub) &&(static_cast <bool> (InductionBinOp && (InductionBinOp
->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode
() == Instruction::FSub) && "Original bin op should be defined for FP induction"
) ? void (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2517, __extension__
__PRETTY_FUNCTION__))
2517 "Original bin op should be defined for FP induction")(static_cast <bool> (InductionBinOp && (InductionBinOp
->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode
() == Instruction::FSub) && "Original bin op should be defined for FP induction"
) ? void (0) : __assert_fail ("InductionBinOp && (InductionBinOp->getOpcode() == Instruction::FAdd || InductionBinOp->getOpcode() == Instruction::FSub) && \"Original bin op should be defined for FP induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2517, __extension__
__PRETTY_FUNCTION__))
;
2518
2519 Value *MulExp = B.CreateFMul(Step, Index);
2520 return B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
2521 "induction");
2522 }
2523 case InductionDescriptor::IK_NoInduction:
2524 return nullptr;
2525 }
2526 llvm_unreachable("invalid enum")::llvm::llvm_unreachable_internal("invalid enum", "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 2526)
;
2527}
2528
2529void InnerLoopVectorizer::packScalarIntoVectorValue(VPValue *Def,
2530 const VPIteration &Instance,
2531 VPTransformState &State) {
2532 Value *ScalarInst = State.get(Def, Instance);
2533 Value *VectorValue = State.get(Def, Instance.Part);
2534 VectorValue = Builder.CreateInsertElement(
2535 VectorValue, ScalarInst,
2536 Instance.Lane.getAsRuntimeExpr(State.Builder, VF));
2537 State.set(Def, VectorValue, Instance.Part);
2538}
2539
2540// Return whether we allow using masked interleave-groups (for dealing with
2541// strided loads/stores that reside in predicated blocks, or for dealing
2542// with gaps).
2543static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
2544 // If an override option has been passed in for interleaved accesses, use it.
2545 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2546 return EnableMaskedInterleavedMemAccesses;
2547
2548 return TTI.enableMaskedInterleavedAccessVectorization();
2549}
2550
2551// Try to vectorize the interleave group that \p Instr belongs to.
2552//
2553// E.g. Translate following interleaved load group (factor = 3):
2554// for (i = 0; i < N; i+=3) {
2555// R = Pic[i]; // Member of index 0
2556// G = Pic[i+1]; // Member of index 1
2557// B = Pic[i+2]; // Member of index 2
2558// ... // do something to R, G, B
2559// }
2560// To:
2561// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
2562// %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9> ; R elements
2563// %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10> ; G elements
2564// %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11> ; B elements
2565//
2566// Or translate following interleaved store group (factor = 3):
2567// for (i = 0; i < N; i+=3) {
2568// ... do something to R, G, B
2569// Pic[i] = R; // Member of index 0
2570// Pic[i+1] = G; // Member of index 1
2571// Pic[i+2] = B; // Member of index 2
2572// }
2573// To:
2574// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
2575// %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
2576// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
2577// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
2578// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
2579void InnerLoopVectorizer::vectorizeInterleaveGroup(
2580 const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs,
2581 VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues,
2582 VPValue *BlockInMask) {
2583 Instruction *Instr = Group->getInsertPos();
2584 const DataLayout &DL = Instr->getModule()->getDataLayout();
2585
2586 // Prepare for the vector type of the interleaved load/store.
2587 Type *ScalarTy = getLoadStoreType(Instr);
2588 unsigned InterleaveFactor = Group->getFactor();
2589 assert(!VF.isScalable() && "scalable vectors not yet supported.")(static_cast <bool> (!VF.isScalable() && "scalable vectors not yet supported."
) ? void (0) : __assert_fail ("!VF.isScalable() && \"scalable vectors not yet supported.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2589, __extension__
__PRETTY_FUNCTION__))
;
2590 auto *VecTy = VectorType::get(ScalarTy, VF * InterleaveFactor);
2591
2592 // Prepare for the new pointers.
2593 SmallVector<Value *, 2> AddrParts;
2594 unsigned Index = Group->getIndex(Instr);
2595
2596 // TODO: extend the masked interleaved-group support to reversed access.
2597 assert((!BlockInMask || !Group->isReverse()) &&(static_cast <bool> ((!BlockInMask || !Group->isReverse
()) && "Reversed masked interleave-group not supported."
) ? void (0) : __assert_fail ("(!BlockInMask || !Group->isReverse()) && \"Reversed masked interleave-group not supported.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2598, __extension__
__PRETTY_FUNCTION__))
2598 "Reversed masked interleave-group not supported.")(static_cast <bool> ((!BlockInMask || !Group->isReverse
()) && "Reversed masked interleave-group not supported."
) ? void (0) : __assert_fail ("(!BlockInMask || !Group->isReverse()) && \"Reversed masked interleave-group not supported.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2598, __extension__
__PRETTY_FUNCTION__))
;
2599
2600 // If the group is reverse, adjust the index to refer to the last vector lane
2601 // instead of the first. We adjust the index from the first vector lane,
2602 // rather than directly getting the pointer for lane VF - 1, because the
2603 // pointer operand of the interleaved access is supposed to be uniform. For
2604 // uniform instructions, we're only required to generate a value for the
2605 // first vector lane in each unroll iteration.
2606 if (Group->isReverse())
2607 Index += (VF.getKnownMinValue() - 1) * Group->getFactor();
2608
2609 for (unsigned Part = 0; Part < UF; Part++) {
2610 Value *AddrPart = State.get(Addr, VPIteration(Part, 0));
2611 setDebugLocFromInst(AddrPart);
2612
2613 // Notice current instruction could be any index. Need to adjust the address
2614 // to the member of index 0.
2615 //
2616 // E.g. a = A[i+1]; // Member of index 1 (Current instruction)
2617 // b = A[i]; // Member of index 0
2618 // Current pointer is pointed to A[i+1], adjust it to A[i].
2619 //
2620 // E.g. A[i+1] = a; // Member of index 1
2621 // A[i] = b; // Member of index 0
2622 // A[i+2] = c; // Member of index 2 (Current instruction)
2623 // Current pointer is pointed to A[i+2], adjust it to A[i].
2624
2625 bool InBounds = false;
2626 if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
2627 InBounds = gep->isInBounds();
2628 AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Builder.getInt32(-Index));
2629 cast<GetElementPtrInst>(AddrPart)->setIsInBounds(InBounds);
2630
2631 // Cast to the vector pointer type.
2632 unsigned AddressSpace = AddrPart->getType()->getPointerAddressSpace();
2633 Type *PtrTy = VecTy->getPointerTo(AddressSpace);
2634 AddrParts.push_back(Builder.CreateBitCast(AddrPart, PtrTy));
2635 }
2636
2637 setDebugLocFromInst(Instr);
2638 Value *PoisonVec = PoisonValue::get(VecTy);
2639
2640 Value *MaskForGaps = nullptr;
2641 if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) {
2642 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
2643 assert(MaskForGaps && "Mask for Gaps is required but it is null")(static_cast <bool> (MaskForGaps && "Mask for Gaps is required but it is null"
) ? void (0) : __assert_fail ("MaskForGaps && \"Mask for Gaps is required but it is null\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2643, __extension__
__PRETTY_FUNCTION__))
;
2644 }
2645
2646 // Vectorize the interleaved load group.
2647 if (isa<LoadInst>(Instr)) {
2648 // For each unroll part, create a wide load for the group.
2649 SmallVector<Value *, 2> NewLoads;
2650 for (unsigned Part = 0; Part < UF; Part++) {
2651 Instruction *NewLoad;
2652 if (BlockInMask || MaskForGaps) {
2653 assert(useMaskedInterleavedAccesses(*TTI) &&(static_cast <bool> (useMaskedInterleavedAccesses(*TTI)
&& "masked interleaved groups are not allowed.") ? void
(0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2654, __extension__
__PRETTY_FUNCTION__))
2654 "masked interleaved groups are not allowed.")(static_cast <bool> (useMaskedInterleavedAccesses(*TTI)
&& "masked interleaved groups are not allowed.") ? void
(0) : __assert_fail ("useMaskedInterleavedAccesses(*TTI) && \"masked interleaved groups are not allowed.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2654, __extension__
__PRETTY_FUNCTION__))
;
2655 Value *GroupMask = MaskForGaps;
2656 if (BlockInMask) {
2657 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2658 Value *ShuffledMask = Builder.CreateShuffleVector(
2659 BlockInMaskPart,
2660 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2661 "interleaved.mask");
2662 GroupMask = MaskForGaps
2663 ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
2664 MaskForGaps)
2665 : ShuffledMask;
2666 }
2667 NewLoad =
2668 Builder.CreateMaskedLoad(VecTy, AddrParts[Part], Group->getAlign(),
2669 GroupMask, PoisonVec, "wide.masked.vec");
2670 }
2671 else
2672 NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part],
2673 Group->getAlign(), "wide.vec");
2674 Group->addMetadata(NewLoad);
2675 NewLoads.push_back(NewLoad);
2676 }
2677
2678 // For each member in the group, shuffle out the appropriate data from the
2679 // wide loads.
2680 unsigned J = 0;
2681 for (unsigned I = 0; I < InterleaveFactor; ++I) {
2682 Instruction *Member = Group->getMember(I);
2683
2684 // Skip the gaps in the group.
2685 if (!Member)
2686 continue;
2687
2688 auto StrideMask =
2689 createStrideMask(I, InterleaveFactor, VF.getKnownMinValue());
2690 for (unsigned Part = 0; Part < UF; Part++) {
2691 Value *StridedVec = Builder.CreateShuffleVector(
2692 NewLoads[Part], StrideMask, "strided.vec");
2693
2694 // If this member has different type, cast the result type.
2695 if (Member->getType() != ScalarTy) {
2696 assert(!VF.isScalable() && "VF is assumed to be non scalable.")(static_cast <bool> (!VF.isScalable() && "VF is assumed to be non scalable."
) ? void (0) : __assert_fail ("!VF.isScalable() && \"VF is assumed to be non scalable.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2696, __extension__
__PRETTY_FUNCTION__))
;
2697 VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
2698 StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
2699 }
2700
2701 if (Group->isReverse())
2702 StridedVec = Builder.CreateVectorReverse(StridedVec, "reverse");
2703
2704 State.set(VPDefs[J], StridedVec, Part);
2705 }
2706 ++J;
2707 }
2708 return;
2709 }
2710
2711 // The sub vector type for current instruction.
2712 auto *SubVT = VectorType::get(ScalarTy, VF);
2713
2714 // Vectorize the interleaved store group.
2715 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
2716 assert((!MaskForGaps || useMaskedInterleavedAccesses(*TTI)) &&(static_cast <bool> ((!MaskForGaps || useMaskedInterleavedAccesses
(*TTI)) && "masked interleaved groups are not allowed."
) ? void (0) : __assert_fail ("(!MaskForGaps || useMaskedInterleavedAccesses(*TTI)) && \"masked interleaved groups are not allowed.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2717, __extension__
__PRETTY_FUNCTION__))
2717 "masked interleaved groups are not allowed.")(static_cast <bool> ((!MaskForGaps || useMaskedInterleavedAccesses
(*TTI)) && "masked interleaved groups are not allowed."
) ? void (0) : __assert_fail ("(!MaskForGaps || useMaskedInterleavedAccesses(*TTI)) && \"masked interleaved groups are not allowed.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2717, __extension__
__PRETTY_FUNCTION__))
;
2718 assert((!MaskForGaps || !VF.isScalable()) &&(static_cast <bool> ((!MaskForGaps || !VF.isScalable())
&& "masking gaps for scalable vectors is not yet supported."
) ? void (0) : __assert_fail ("(!MaskForGaps || !VF.isScalable()) && \"masking gaps for scalable vectors is not yet supported.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2719, __extension__
__PRETTY_FUNCTION__))
2719 "masking gaps for scalable vectors is not yet supported.")(static_cast <bool> ((!MaskForGaps || !VF.isScalable())
&& "masking gaps for scalable vectors is not yet supported."
) ? void (0) : __assert_fail ("(!MaskForGaps || !VF.isScalable()) && \"masking gaps for scalable vectors is not yet supported.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2719, __extension__
__PRETTY_FUNCTION__))
;
2720 for (unsigned Part = 0; Part < UF; Part++) {
2721 // Collect the stored vector from each member.
2722 SmallVector<Value *, 4> StoredVecs;
2723 for (unsigned i = 0; i < InterleaveFactor; i++) {
2724 assert((Group->getMember(i) || MaskForGaps) &&(static_cast <bool> ((Group->getMember(i) || MaskForGaps
) && "Fail to get a member from an interleaved store group"
) ? void (0) : __assert_fail ("(Group->getMember(i) || MaskForGaps) && \"Fail to get a member from an interleaved store group\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2725, __extension__
__PRETTY_FUNCTION__))
2725 "Fail to get a member from an interleaved store group")(static_cast <bool> ((Group->getMember(i) || MaskForGaps
) && "Fail to get a member from an interleaved store group"
) ? void (0) : __assert_fail ("(Group->getMember(i) || MaskForGaps) && \"Fail to get a member from an interleaved store group\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2725, __extension__
__PRETTY_FUNCTION__))
;
2726 Instruction *Member = Group->getMember(i);
2727
2728 // Skip the gaps in the group.
2729 if (!Member) {
2730 Value *Undef = PoisonValue::get(SubVT);
2731 StoredVecs.push_back(Undef);
2732 continue;
2733 }
2734
2735 Value *StoredVec = State.get(StoredValues[i], Part);
2736
2737 if (Group->isReverse())
2738 StoredVec = Builder.CreateVectorReverse(StoredVec, "reverse");
2739
2740 // If this member has different type, cast it to a unified type.
2741
2742 if (StoredVec->getType() != SubVT)
2743 StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);
2744
2745 StoredVecs.push_back(StoredVec);
2746 }
2747
2748 // Concatenate all vectors into a wide vector.
2749 Value *WideVec = concatenateVectors(Builder, StoredVecs);
2750
2751 // Interleave the elements in the wide vector.
2752 Value *IVec = Builder.CreateShuffleVector(
2753 WideVec, createInterleaveMask(VF.getKnownMinValue(), InterleaveFactor),
2754 "interleaved.vec");
2755
2756 Instruction *NewStoreInstr;
2757 if (BlockInMask || MaskForGaps) {
2758 Value *GroupMask = MaskForGaps;
2759 if (BlockInMask) {
2760 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2761 Value *ShuffledMask = Builder.CreateShuffleVector(
2762 BlockInMaskPart,
2763 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2764 "interleaved.mask");
2765 GroupMask = MaskForGaps ? Builder.CreateBinOp(Instruction::And,
2766 ShuffledMask, MaskForGaps)
2767 : ShuffledMask;
2768 }
2769 NewStoreInstr = Builder.CreateMaskedStore(IVec, AddrParts[Part],
2770 Group->getAlign(), GroupMask);
2771 } else
2772 NewStoreInstr =
2773 Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign());
2774
2775 Group->addMetadata(NewStoreInstr);
2776 }
2777}
2778
2779void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr,
2780 VPReplicateRecipe *RepRecipe,
2781 const VPIteration &Instance,
2782 bool IfPredicateInstr,
2783 VPTransformState &State) {
2784 assert(!Instr->getType()->isAggregateType() && "Can't handle vectors")(static_cast <bool> (!Instr->getType()->isAggregateType
() && "Can't handle vectors") ? void (0) : __assert_fail
("!Instr->getType()->isAggregateType() && \"Can't handle vectors\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2784, __extension__
__PRETTY_FUNCTION__))
;
2785
2786 // llvm.experimental.noalias.scope.decl intrinsics must only be duplicated for
2787 // the first lane and part.
2788 if (isa<NoAliasScopeDeclInst>(Instr))
2789 if (!Instance.isFirstIteration())
2790 return;
2791
2792 setDebugLocFromInst(Instr);
2793
2794 // Does this instruction return a value ?
2795 bool IsVoidRetTy = Instr->getType()->isVoidTy();
2796
2797 Instruction *Cloned = Instr->clone();
2798 if (!IsVoidRetTy)
2799 Cloned->setName(Instr->getName() + ".cloned");
2800
2801 // If the scalarized instruction contributes to the address computation of a
2802 // widen masked load/store which was in a basic block that needed predication
2803 // and is not predicated after vectorization, we can't propagate
2804 // poison-generating flags (nuw/nsw, exact, inbounds, etc.). The scalarized
2805 // instruction could feed a poison value to the base address of the widen
2806 // load/store.
2807 if (State.MayGeneratePoisonRecipes.contains(RepRecipe))
2808 Cloned->dropPoisonGeneratingFlags();
2809
2810 State.Builder.SetInsertPoint(Builder.GetInsertBlock(),
2811 Builder.GetInsertPoint());
2812 // Replace the operands of the cloned instructions with their scalar
2813 // equivalents in the new loop.
2814 for (auto &I : enumerate(RepRecipe->operands())) {
2815 auto InputInstance = Instance;
2816 VPValue *Operand = I.value();
2817 VPReplicateRecipe *OperandR = dyn_cast<VPReplicateRecipe>(Operand);
2818 if (OperandR && OperandR->isUniform())
2819 InputInstance.Lane = VPLane::getFirstLane();
2820 Cloned->setOperand(I.index(), State.get(Operand, InputInstance));
2821 }
2822 addNewMetadata(Cloned, Instr);
2823
2824 // Place the cloned scalar in the new loop.
2825 Builder.Insert(Cloned);
2826
2827 State.set(RepRecipe, Cloned, Instance);
2828
2829 // If we just cloned a new assumption, add it the assumption cache.
2830 if (auto *II = dyn_cast<AssumeInst>(Cloned))
2831 AC->registerAssumption(II);
2832
2833 // End if-block.
2834 if (IfPredicateInstr)
2835 PredicatedInstructions.push_back(Cloned);
2836}
2837
2838void InnerLoopVectorizer::createHeaderBranch(Loop *L) {
2839 BasicBlock *Header = L->getHeader();
2840 assert(!L->getLoopLatch() && "loop should not have a latch at this point")(static_cast <bool> (!L->getLoopLatch() && "loop should not have a latch at this point"
) ? void (0) : __assert_fail ("!L->getLoopLatch() && \"loop should not have a latch at this point\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2840, __extension__
__PRETTY_FUNCTION__))
;
2841
2842 IRBuilder<> B(Header->getTerminator());
2843 Instruction *OldInst =
2844 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction());
2845 setDebugLocFromInst(OldInst, &B);
2846
2847 // Connect the header to the exit and header blocks and replace the old
2848 // terminator.
2849 B.CreateCondBr(B.getTrue(), L->getUniqueExitBlock(), Header);
2850
2851 // Now we have two terminators. Remove the old one from the block.
2852 Header->getTerminator()->eraseFromParent();
2853}
2854
2855Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
2856 if (TripCount)
2857 return TripCount;
2858
2859 assert(L && "Create Trip Count for null loop.")(static_cast <bool> (L && "Create Trip Count for null loop."
) ? void (0) : __assert_fail ("L && \"Create Trip Count for null loop.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2859, __extension__
__PRETTY_FUNCTION__))
;
2860 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
2861 // Find the loop boundaries.
2862 ScalarEvolution *SE = PSE.getSE();
2863 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
2864 assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&(static_cast <bool> (!isa<SCEVCouldNotCompute>(BackedgeTakenCount
) && "Invalid loop count") ? void (0) : __assert_fail
("!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && \"Invalid loop count\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2865, __extension__
__PRETTY_FUNCTION__))
2865 "Invalid loop count")(static_cast <bool> (!isa<SCEVCouldNotCompute>(BackedgeTakenCount
) && "Invalid loop count") ? void (0) : __assert_fail
("!isa<SCEVCouldNotCompute>(BackedgeTakenCount) && \"Invalid loop count\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2865, __extension__
__PRETTY_FUNCTION__))
;
2866
2867 Type *IdxTy = Legal->getWidestInductionType();
2868 assert(IdxTy && "No type for induction")(static_cast <bool> (IdxTy && "No type for induction"
) ? void (0) : __assert_fail ("IdxTy && \"No type for induction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2868, __extension__
__PRETTY_FUNCTION__))
;
2869
2870 // The exit count might have the type of i64 while the phi is i32. This can
2871 // happen if we have an induction variable that is sign extended before the
2872 // compare. The only way that we get a backedge taken count is that the
2873 // induction variable was signed and as such will not overflow. In such a case
2874 // truncation is legal.
2875 if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) >
2876 IdxTy->getPrimitiveSizeInBits())
2877 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
2878 BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
2879
2880 // Get the total trip count from the count by adding 1.
2881 const SCEV *ExitCount = SE->getAddExpr(
2882 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
2883
2884 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
2885
2886 // Expand the trip count and place the new instructions in the preheader.
2887 // Notice that the pre-header does not change, only the loop body.
2888 SCEVExpander Exp(*SE, DL, "induction");
2889
2890 // Count holds the overall loop count (N).
2891 TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
2892 L->getLoopPreheader()->getTerminator());
2893
2894 if (TripCount->getType()->isPointerTy())
2895 TripCount =
2896 CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
2897 L->getLoopPreheader()->getTerminator());
2898
2899 return TripCount;
2900}
2901
2902Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
2903 if (VectorTripCount)
5
Assuming field 'VectorTripCount' is null
6
Taking false branch
2904 return VectorTripCount;
2905
2906 Value *TC = getOrCreateTripCount(L);
2907 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
7
Called C++ object pointer is null
2908
2909 Type *Ty = TC->getType();
2910 // This is where we can make the step a runtime constant.
2911 Value *Step = createStepForVF(Builder, Ty, VF, UF);
2912
2913 // If the tail is to be folded by masking, round the number of iterations N
2914 // up to a multiple of Step instead of rounding down. This is done by first
2915 // adding Step-1 and then rounding down. Note that it's ok if this addition
2916 // overflows: the vector induction variable will eventually wrap to zero given
2917 // that it starts at zero and its Step is a power of two; the loop will then
2918 // exit, with the last early-exit vector comparison also producing all-true.
2919 if (Cost->foldTailByMasking()) {
2920 assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&(static_cast <bool> (isPowerOf2_32(VF.getKnownMinValue(
) * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? void (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue() * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2921, __extension__
__PRETTY_FUNCTION__))
2921 "VF*UF must be a power of 2 when folding tail by masking")(static_cast <bool> (isPowerOf2_32(VF.getKnownMinValue(
) * UF) && "VF*UF must be a power of 2 when folding tail by masking"
) ? void (0) : __assert_fail ("isPowerOf2_32(VF.getKnownMinValue() * UF) && \"VF*UF must be a power of 2 when folding tail by masking\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2921, __extension__
__PRETTY_FUNCTION__))
;
2922 Value *NumLanes = getRuntimeVF(Builder, Ty, VF * UF);
2923 TC = Builder.CreateAdd(
2924 TC, Builder.CreateSub(NumLanes, ConstantInt::get(Ty, 1)), "n.rnd.up");
2925 }
2926
2927 // Now we need to generate the expression for the part of the loop that the
2928 // vectorized body will execute. This is equal to N - (N % Step) if scalar
2929 // iterations are not required for correctness, or N - Step, otherwise. Step
2930 // is equal to the vectorization factor (number of SIMD elements) times the
2931 // unroll factor (number of SIMD instructions).
2932 Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
2933
2934 // There are cases where we *must* run at least one iteration in the remainder
2935 // loop. See the cost model for when this can happen. If the step evenly
2936 // divides the trip count, we set the remainder to be equal to the step. If
2937 // the step does not evenly divide the trip count, no adjustment is necessary
2938 // since there will already be scalar iterations. Note that the minimum
2939 // iterations check ensures that N >= Step.
2940 if (Cost->requiresScalarEpilogue(VF)) {
2941 auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
2942 R = Builder.CreateSelect(IsZero, Step, R);
2943 }
2944
2945 VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
2946
2947 return VectorTripCount;
2948}
2949
2950Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
2951 const DataLayout &DL) {
2952 // Verify that V is a vector type with same number of elements as DstVTy.
2953 auto *DstFVTy = cast<FixedVectorType>(DstVTy);
2954 unsigned VF = DstFVTy->getNumElements();
2955 auto *SrcVecTy = cast<FixedVectorType>(V->getType());
2956 assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match")(static_cast <bool> ((VF == SrcVecTy->getNumElements
()) && "Vector dimensions do not match") ? void (0) :
__assert_fail ("(VF == SrcVecTy->getNumElements()) && \"Vector dimensions do not match\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2956, __extension__
__PRETTY_FUNCTION__))
;
2957 Type *SrcElemTy = SrcVecTy->getElementType();
2958 Type *DstElemTy = DstFVTy->getElementType();
2959 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&(static_cast <bool> ((DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) && "Vector elements must have same size"
) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2960, __extension__
__PRETTY_FUNCTION__))
2960 "Vector elements must have same size")(static_cast <bool> ((DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) && "Vector elements must have same size"
) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) && \"Vector elements must have same size\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2960, __extension__
__PRETTY_FUNCTION__))
;
2961
2962 // Do a direct cast if element types are castable.
2963 if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
2964 return Builder.CreateBitOrPointerCast(V, DstFVTy);
2965 }
2966 // V cannot be directly casted to desired vector type.
2967 // May happen when V is a floating point vector but DstVTy is a vector of
2968 // pointers or vice-versa. Handle this using a two-step bitcast using an
2969 // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
2970 assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&(static_cast <bool> ((DstElemTy->isPointerTy() != SrcElemTy
->isPointerTy()) && "Only one type should be a pointer type"
) ? void (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2971, __extension__
__PRETTY_FUNCTION__))
2971 "Only one type should be a pointer type")(static_cast <bool> ((DstElemTy->isPointerTy() != SrcElemTy
->isPointerTy()) && "Only one type should be a pointer type"
) ? void (0) : __assert_fail ("(DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) && \"Only one type should be a pointer type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2971, __extension__
__PRETTY_FUNCTION__))
;
2972 assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&(static_cast <bool> ((DstElemTy->isFloatingPointTy()
!= SrcElemTy->isFloatingPointTy()) && "Only one type should be a floating point type"
) ? void (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2973, __extension__
__PRETTY_FUNCTION__))
2973 "Only one type should be a floating point type")(static_cast <bool> ((DstElemTy->isFloatingPointTy()
!= SrcElemTy->isFloatingPointTy()) && "Only one type should be a floating point type"
) ? void (0) : __assert_fail ("(DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) && \"Only one type should be a floating point type\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 2973, __extension__
__PRETTY_FUNCTION__))
;
2974 Type *IntTy =
2975 IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
2976 auto *VecIntTy = FixedVectorType::get(IntTy, VF);
2977 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
2978 return Builder.CreateBitOrPointerCast(CastVal, DstFVTy);
2979}
2980
2981void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
2982 BasicBlock *Bypass) {
2983 Value *Count = getOrCreateTripCount(L);
2984 // Reuse existing vector loop preheader for TC checks.
2985 // Note that new preheader block is generated for vector loop.
2986 BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
2987 IRBuilder<> Builder(TCCheckBlock->getTerminator());
2988
2989 // Generate code to check if the loop's trip count is less than VF * UF, or
2990 // equal to it in case a scalar epilogue is required; this implies that the
2991 // vector trip count is zero. This check also covers the case where adding one
2992 // to the backedge-taken count overflowed leading to an incorrect trip count
2993 // of zero. In this case we will also jump to the scalar loop.
2994 auto P = Cost->requiresScalarEpilogue(VF) ? ICmpInst::ICMP_ULE
2995 : ICmpInst::ICMP_ULT;
2996
2997 // If tail is to be folded, vector loop takes care of all iterations.
2998 Value *CheckMinIters = Builder.getFalse();
2999 if (!Cost->foldTailByMasking()) {
3000 Value *Step = createStepForVF(Builder, Count->getType(), VF, UF);
3001 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
3002 }
3003 // Create new preheader for vector loop.
3004 LoopVectorPreHeader =
3005 SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), DT, LI, nullptr,
3006 "vector.ph");
3007
3008 assert(DT->properlyDominates(DT->getNode(TCCheckBlock),(static_cast <bool> (DT->properlyDominates(DT->getNode
(TCCheckBlock), DT->getNode(Bypass)->getIDom()) &&
"TC check is expected to dominate Bypass") ? void (0) : __assert_fail
("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3010, __extension__
__PRETTY_FUNCTION__))
3009 DT->getNode(Bypass)->getIDom()) &&(static_cast <bool> (DT->properlyDominates(DT->getNode
(TCCheckBlock), DT->getNode(Bypass)->getIDom()) &&
"TC check is expected to dominate Bypass") ? void (0) : __assert_fail
("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3010, __extension__
__PRETTY_FUNCTION__))
3010 "TC check is expected to dominate Bypass")(static_cast <bool> (DT->properlyDominates(DT->getNode
(TCCheckBlock), DT->getNode(Bypass)->getIDom()) &&
"TC check is expected to dominate Bypass") ? void (0) : __assert_fail
("DT->properlyDominates(DT->getNode(TCCheckBlock), DT->getNode(Bypass)->getIDom()) && \"TC check is expected to dominate Bypass\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3010, __extension__
__PRETTY_FUNCTION__))
;
3011
3012 // Update dominator for Bypass & LoopExit (if needed).
3013 DT->changeImmediateDominator(Bypass, TCCheckBlock);
3014 if (!Cost->requiresScalarEpilogue(VF))
3015 // If there is an epilogue which must run, there's no edge from the
3016 // middle block to exit blocks and thus no need to update the immediate
3017 // dominator of the exit blocks.
3018 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);
3019
3020 ReplaceInstWithInst(
3021 TCCheckBlock->getTerminator(),
3022 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
3023 LoopBypassBlocks.push_back(TCCheckBlock);
3024}
3025
3026BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
3027
3028 BasicBlock *const SCEVCheckBlock =
3029 RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader, LoopExitBlock);
3030 if (!SCEVCheckBlock)
3031 return nullptr;
3032
3033 assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||(static_cast <bool> (!(SCEVCheckBlock->getParent()->
hasOptSize() || (OptForSizeBasedOnProfile && Cost->
Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
"Cannot SCEV check stride or overflow when optimizing for size"
) ? void (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3036, __extension__
__PRETTY_FUNCTION__))
3034 (OptForSizeBasedOnProfile &&(static_cast <bool> (!(SCEVCheckBlock->getParent()->
hasOptSize() || (OptForSizeBasedOnProfile && Cost->
Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
"Cannot SCEV check stride or overflow when optimizing for size"
) ? void (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3036, __extension__
__PRETTY_FUNCTION__))
3035 Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&(static_cast <bool> (!(SCEVCheckBlock->getParent()->
hasOptSize() || (OptForSizeBasedOnProfile && Cost->
Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
"Cannot SCEV check stride or overflow when optimizing for size"
) ? void (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3036, __extension__
__PRETTY_FUNCTION__))
3036 "Cannot SCEV check stride or overflow when optimizing for size")(static_cast <bool> (!(SCEVCheckBlock->getParent()->
hasOptSize() || (OptForSizeBasedOnProfile && Cost->
Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&
"Cannot SCEV check stride or overflow when optimizing for size"
) ? void (0) : __assert_fail ("!(SCEVCheckBlock->getParent()->hasOptSize() || (OptForSizeBasedOnProfile && Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && \"Cannot SCEV check stride or overflow when optimizing for size\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3036, __extension__
__PRETTY_FUNCTION__))
;
3037
3038
3039 // Update dominator only if this is first RT check.
3040 if (LoopBypassBlocks.empty()) {
3041 DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
3042 if (!Cost->requiresScalarEpilogue(VF))
3043 // If there is an epilogue which must run, there's no edge from the
3044 // middle block to exit blocks and thus no need to update the immediate
3045 // dominator of the exit blocks.
3046 DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
3047 }
3048
3049 LoopBypassBlocks.push_back(SCEVCheckBlock);
3050 AddedSafetyChecks = true;
3051 return SCEVCheckBlock;
3052}
3053
3054BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L,
3055 BasicBlock *Bypass) {
3056 // VPlan-native path does not do any analysis for runtime checks currently.
3057 if (EnableVPlanNativePath)
3058 return nullptr;
3059
3060 BasicBlock *const MemCheckBlock =
3061 RTChecks.emitMemRuntimeChecks(L, Bypass, LoopVectorPreHeader);
3062
3063 // Check if we generated code that checks in runtime if arrays overlap. We put
3064 // the checks into a separate block to make the more common case of few
3065 // elements faster.
3066 if (!MemCheckBlock)
3067 return nullptr;
3068
3069 if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
3070 assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&(static_cast <bool> (Cost->Hints->getForce() == LoopVectorizeHints
::FK_Enabled && "Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.") ? void (0) : __assert_fail ("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3072, __extension__
__PRETTY_FUNCTION__))
3071 "Cannot emit memory checks when optimizing for size, unless forced "(static_cast <bool> (Cost->Hints->getForce() == LoopVectorizeHints
::FK_Enabled && "Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.") ? void (0) : __assert_fail ("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3072, __extension__
__PRETTY_FUNCTION__))
3072 "to vectorize.")(static_cast <bool> (Cost->Hints->getForce() == LoopVectorizeHints
::FK_Enabled && "Cannot emit memory checks when optimizing for size, unless forced "
"to vectorize.") ? void (0) : __assert_fail ("Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && \"Cannot emit memory checks when optimizing for size, unless forced \" \"to vectorize.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3072, __extension__
__PRETTY_FUNCTION__))
;
3073 ORE->emit([&]() {
3074 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationCodeSize",
3075 L->getStartLoc(), L->getHeader())
3076 << "Code-size may be reduced by not forcing "
3077 "vectorization, or by source-code modifications "
3078 "eliminating the need for runtime checks "
3079 "(e.g., adding 'restrict').";
3080 });
3081 }
3082
3083 LoopBypassBlocks.push_back(MemCheckBlock);
3084
3085 AddedSafetyChecks = true;
3086
3087 // We currently don't use LoopVersioning for the actual loop cloning but we
3088 // still use it to add the noalias metadata.
3089 LVer = std::make_unique<LoopVersioning>(
3090 *Legal->getLAI(),
3091 Legal->getLAI()->getRuntimePointerChecking()->getChecks(), OrigLoop, LI,
3092 DT, PSE.getSE());
3093 LVer->prepareNoAliasMetadata();
3094 return MemCheckBlock;
3095}
3096
3097Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
3098 LoopScalarBody = OrigLoop->getHeader();
3099 LoopVectorPreHeader = OrigLoop->getLoopPreheader();
3100 assert(LoopVectorPreHeader && "Invalid loop structure")(static_cast <bool> (LoopVectorPreHeader && "Invalid loop structure"
) ? void (0) : __assert_fail ("LoopVectorPreHeader && \"Invalid loop structure\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3100, __extension__
__PRETTY_FUNCTION__))
;
3101 LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr
3102 assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF)) &&(static_cast <bool> ((LoopExitBlock || Cost->requiresScalarEpilogue
(VF)) && "multiple exit loop without required epilogue?"
) ? void (0) : __assert_fail ("(LoopExitBlock || Cost->requiresScalarEpilogue(VF)) && \"multiple exit loop without required epilogue?\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3103, __extension__
__PRETTY_FUNCTION__))
3103 "multiple exit loop without required epilogue?")(static_cast <bool> ((LoopExitBlock || Cost->requiresScalarEpilogue
(VF)) && "multiple exit loop without required epilogue?"
) ? void (0) : __assert_fail ("(LoopExitBlock || Cost->requiresScalarEpilogue(VF)) && \"multiple exit loop without required epilogue?\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3103, __extension__
__PRETTY_FUNCTION__))
;
3104
3105 LoopMiddleBlock =
3106 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3107 LI, nullptr, Twine(Prefix) + "middle.block");
3108 LoopScalarPreHeader =
3109 SplitBlock(LoopMiddleBlock, LoopMiddleBlock->getTerminator(), DT, LI,
3110 nullptr, Twine(Prefix) + "scalar.ph");
3111
3112 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
3113
3114 // Set up the middle block terminator. Two cases:
3115 // 1) If we know that we must execute the scalar epilogue, emit an
3116 // unconditional branch.
3117 // 2) Otherwise, we must have a single unique exit block (due to how we
3118 // implement the multiple exit case). In this case, set up a conditonal
3119 // branch from the middle block to the loop scalar preheader, and the
3120 // exit block. completeLoopSkeleton will update the condition to use an
3121 // iteration check, if required to decide whether to execute the remainder.
3122 BranchInst *BrInst = Cost->requiresScalarEpilogue(VF) ?
3123 BranchInst::Create(LoopScalarPreHeader) :
3124 BranchInst::Create(LoopExitBlock, LoopScalarPreHeader,
3125 Builder.getTrue());
3126 BrInst->setDebugLoc(ScalarLatchTerm->getDebugLoc());
3127 ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst);
3128
3129 // We intentionally don't let SplitBlock to update LoopInfo since
3130 // LoopVectorBody should belong to another loop than LoopVectorPreHeader.
3131 // LoopVectorBody is explicitly added to the correct place few lines later.
3132 BasicBlock *LoopVectorBody =
3133 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3134 nullptr, nullptr, Twine(Prefix) + "vector.body");
3135
3136 // Update dominator for loop exit.
3137 if (!Cost->requiresScalarEpilogue(VF))
3138 // If there is an epilogue which must run, there's no edge from the
3139 // middle block to exit blocks and thus no need to update the immediate
3140 // dominator of the exit blocks.
3141 DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);
3142
3143 // Create and register the new vector loop.
3144 Loop *Lp = LI->AllocateLoop();
3145 Loop *ParentLoop = OrigLoop->getParentLoop();
3146
3147 // Insert the new loop into the loop nest and register the new basic blocks
3148 // before calling any utilities such as SCEV that require valid LoopInfo.
3149 if (ParentLoop) {
3150 ParentLoop->addChildLoop(Lp);
3151 } else {
3152 LI->addTopLevelLoop(Lp);
3153 }
3154 Lp->addBasicBlockToLoop(LoopVectorBody, *LI);
3155 return Lp;
3156}
3157
3158void InnerLoopVectorizer::createInductionResumeValues(
3159 Loop *L, std::pair<BasicBlock *, Value *> AdditionalBypass) {
3160 assert(((AdditionalBypass.first && AdditionalBypass.second) ||(static_cast <bool> (((AdditionalBypass.first &&
AdditionalBypass.second) || (!AdditionalBypass.first &&
!AdditionalBypass.second)) && "Inconsistent information about additional bypass."
) ? void (0) : __assert_fail ("((AdditionalBypass.first && AdditionalBypass.second) || (!AdditionalBypass.first && !AdditionalBypass.second)) && \"Inconsistent information about additional bypass.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3162, __extension__
__PRETTY_FUNCTION__))
3161 (!AdditionalBypass.first && !AdditionalBypass.second)) &&(static_cast <bool> (((AdditionalBypass.first &&
AdditionalBypass.second) || (!AdditionalBypass.first &&
!AdditionalBypass.second)) && "Inconsistent information about additional bypass."
) ? void (0) : __assert_fail ("((AdditionalBypass.first && AdditionalBypass.second) || (!AdditionalBypass.first && !AdditionalBypass.second)) && \"Inconsistent information about additional bypass.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3162, __extension__
__PRETTY_FUNCTION__))
3162 "Inconsistent information about additional bypass.")(static_cast <bool> (((AdditionalBypass.first &&
AdditionalBypass.second) || (!AdditionalBypass.first &&
!AdditionalBypass.second)) && "Inconsistent information about additional bypass."
) ? void (0) : __assert_fail ("((AdditionalBypass.first && AdditionalBypass.second) || (!AdditionalBypass.first && !AdditionalBypass.second)) && \"Inconsistent information about additional bypass.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3162, __extension__
__PRETTY_FUNCTION__))
;
3163
3164 Value *VectorTripCount = getOrCreateVectorTripCount(L);
3165 assert(VectorTripCount && L && "Expected valid arguments")(static_cast <bool> (VectorTripCount && L &&
"Expected valid arguments") ? void (0) : __assert_fail ("VectorTripCount && L && \"Expected valid arguments\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3165, __extension__
__PRETTY_FUNCTION__))
;
3166 // We are going to resume the execution of the scalar loop.
3167 // Go over all of the induction variables that we found and fix the
3168 // PHIs that are left in the scalar version of the loop.
3169 // The starting values of PHI nodes depend on the counter of the last
3170 // iteration in the vectorized loop.
3171 // If we come from a bypass edge then we need to start from the original
3172 // start value.
3173 Instruction *OldInduction = Legal->getPrimaryInduction();
3174 for (auto &InductionEntry : Legal->getInductionVars()) {
3175 PHINode *OrigPhi = InductionEntry.first;
3176 InductionDescriptor II = InductionEntry.second;
3177
3178 // Create phi nodes to merge from the backedge-taken check block.
3179 PHINode *BCResumeVal =
3180 PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
3181 LoopScalarPreHeader->getTerminator());
3182 // Copy original phi DL over to the new one.
3183 BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
3184 Value *&EndValue = IVEndValues[OrigPhi];
3185 Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
3186 if (OrigPhi == OldInduction) {
3187 // We know what the end value is.
3188 EndValue = VectorTripCount;
3189 } else {
3190 IRBuilder<> B(L->getLoopPreheader()->getTerminator());
3191
3192 // Fast-math-flags propagate from the original induction instruction.
3193 if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
3194 B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());
3195
3196 Type *StepType = II.getStep()->getType();
3197 Instruction::CastOps CastOp =
3198 CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
3199 Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd");
3200 Value *Step =
3201 CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint());
3202 EndValue = emitTransformedIndex(B, CRD, II.getStartValue(), Step, II);
3203 EndValue->setName("ind.end");
3204
3205 // Compute the end value for the additional bypass (if applicable).
3206 if (AdditionalBypass.first) {
3207 B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt()));
3208 CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true,
3209 StepType, true);
3210 Value *Step =
3211 CreateStepValue(II.getStep(), *PSE.getSE(), &*B.GetInsertPoint());
3212 CRD =
3213 B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd");
3214 EndValueFromAdditionalBypass =
3215 emitTransformedIndex(B, CRD, II.getStartValue(), Step, II);
3216 EndValueFromAdditionalBypass->setName("ind.end");
3217 }
3218 }
3219 // The new PHI merges the original incoming value, in case of a bypass,
3220 // or the value at the end of the vectorized loop.
3221 BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
3222
3223 // Fix the scalar body counter (PHI node).
3224 // The old induction's phi node in the scalar body needs the truncated
3225 // value.
3226 for (BasicBlock *BB : LoopBypassBlocks)
3227 BCResumeVal->addIncoming(II.getStartValue(), BB);
3228
3229 if (AdditionalBypass.first)
3230 BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
3231 EndValueFromAdditionalBypass);
3232
3233 OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
3234 }
3235}
3236
3237BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L,
3238 MDNode *OrigLoopID) {
3239 assert(L && "Expected valid loop.")(static_cast <bool> (L && "Expected valid loop."
) ? void (0) : __assert_fail ("L && \"Expected valid loop.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3239, __extension__
__PRETTY_FUNCTION__))
;
3240
3241 // The trip counts should be cached by now.
3242 Value *Count = getOrCreateTripCount(L);
3243 Value *VectorTripCount = getOrCreateVectorTripCount(L);
3244
3245 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
3246
3247 // Add a check in the middle block to see if we have completed
3248 // all of the iterations in the first vector loop. Three cases:
3249 // 1) If we require a scalar epilogue, there is no conditional branch as
3250 // we unconditionally branch to the scalar preheader. Do nothing.
3251 // 2) If (N - N%VF) == N, then we *don't* need to run the remainder.
3252 // Thus if tail is to be folded, we know we don't need to run the
3253 // remainder and we can use the previous value for the condition (true).
3254 // 3) Otherwise, construct a runtime check.
3255 if (!Cost->requiresScalarEpilogue(VF) && !Cost->foldTailByMasking()) {
3256 Instruction *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ,
3257 Count, VectorTripCount, "cmp.n",
3258 LoopMiddleBlock->getTerminator());
3259
3260 // Here we use the same DebugLoc as the scalar loop latch terminator instead
3261 // of the corresponding compare because they may have ended up with
3262 // different line numbers and we want to avoid awkward line stepping while
3263 // debugging. Eg. if the compare has got a line number inside the loop.
3264 CmpN->setDebugLoc(ScalarLatchTerm->getDebugLoc());
3265 cast<BranchInst>(LoopMiddleBlock->getTerminator())->setCondition(CmpN);
3266 }
3267
3268 // Get ready to start creating new instructions into the vectorized body.
3269 assert(LoopVectorPreHeader == L->getLoopPreheader() &&(static_cast <bool> (LoopVectorPreHeader == L->getLoopPreheader
() && "Inconsistent vector loop preheader") ? void (0
) : __assert_fail ("LoopVectorPreHeader == L->getLoopPreheader() && \"Inconsistent vector loop preheader\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3270, __extension__
__PRETTY_FUNCTION__))
3270 "Inconsistent vector loop preheader")(static_cast <bool> (LoopVectorPreHeader == L->getLoopPreheader
() && "Inconsistent vector loop preheader") ? void (0
) : __assert_fail ("LoopVectorPreHeader == L->getLoopPreheader() && \"Inconsistent vector loop preheader\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3270, __extension__
__PRETTY_FUNCTION__))
;
3271
3272#ifdef EXPENSIVE_CHECKS
3273 assert(DT->verify(DominatorTree::VerificationLevel::Fast))(static_cast <bool> (DT->verify(DominatorTree::VerificationLevel
::Fast)) ? void (0) : __assert_fail ("DT->verify(DominatorTree::VerificationLevel::Fast)"
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3273, __extension__
__PRETTY_FUNCTION__))
;
3274 LI->verify(*DT);
3275#endif
3276
3277 return LoopVectorPreHeader;
3278}
3279
3280std::pair<BasicBlock *, Value *>
3281InnerLoopVectorizer::createVectorizedLoopSkeleton() {
3282 /*
3283 In this function we generate a new loop. The new loop will contain
3284 the vectorized instructions while the old loop will continue to run the
3285 scalar remainder.
3286
3287 [ ] <-- loop iteration number check.
3288 / |
3289 / v
3290 | [ ] <-- vector loop bypass (may consist of multiple blocks).
3291 | / |
3292 | / v
3293 || [ ] <-- vector pre header.
3294 |/ |
3295 | v
3296 | [ ] \
3297 | [ ]_| <-- vector loop.
3298 | |
3299 | v
3300 \ -[ ] <--- middle-block.
3301 \/ |
3302 /\ v
3303 | ->[ ] <--- new preheader.
3304 | |
3305 (opt) v <-- edge from middle to exit iff epilogue is not required.
3306 | [ ] \
3307 | [ ]_| <-- old scalar loop to handle remainder (scalar epilogue).
3308 \ |
3309 \ v
3310 >[ ] <-- exit block(s).
3311 ...
3312 */
3313
3314 // Get the metadata of the original loop before it gets modified.
3315 MDNode *OrigLoopID = OrigLoop->getLoopID();
3316
3317 // Workaround! Compute the trip count of the original loop and cache it
3318 // before we start modifying the CFG. This code has a systemic problem
3319 // wherein it tries to run analysis over partially constructed IR; this is
3320 // wrong, and not simply for SCEV. The trip count of the original loop
3321 // simply happens to be prone to hitting this in practice. In theory, we
3322 // can hit the same issue for any SCEV, or ValueTracking query done during
3323 // mutation. See PR49900.
3324 getOrCreateTripCount(OrigLoop);
3325
3326 // Create an empty vector loop, and prepare basic blocks for the runtime
3327 // checks.
3328 Loop *Lp = createVectorLoopSkeleton("");
3329
3330 // Now, compare the new count to zero. If it is zero skip the vector loop and
3331 // jump to the scalar loop. This check also covers the case where the
3332 // backedge-taken count is uint##_max: adding one to it will overflow leading
3333 // to an incorrect trip count of zero. In this (rare) case we will also jump
3334 // to the scalar loop.
3335 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader);
3336
3337 // Generate the code to check any assumptions that we've made for SCEV
3338 // expressions.
3339 emitSCEVChecks(LoopScalarPreHeader);
3340
3341 // Generate the code that checks in runtime if arrays overlap. We put the
3342 // checks into a separate block to make the more common case of few elements
3343 // faster.
3344 emitMemRuntimeChecks(Lp, LoopScalarPreHeader);
3345
3346 createHeaderBranch(Lp);
3347
3348 // Emit phis for the new starting index of the scalar loop.
3349 createInductionResumeValues(Lp);
3350
3351 return {completeLoopSkeleton(Lp, OrigLoopID), nullptr};
3352}
3353
3354// Fix up external users of the induction variable. At this point, we are
3355// in LCSSA form, with all external PHIs that use the IV having one input value,
3356// coming from the remainder loop. We need those PHIs to also have a correct
3357// value for the IV when arriving directly from the middle block.
3358void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
3359 const InductionDescriptor &II,
3360 Value *CountRoundDown, Value *EndValue,
3361 BasicBlock *MiddleBlock,
3362 BasicBlock *VectorHeader) {
3363 // There are two kinds of external IV usages - those that use the value
3364 // computed in the last iteration (the PHI) and those that use the penultimate
3365 // value (the value that feeds into the phi from the loop latch).
3366 // We allow both, but they, obviously, have different values.
3367
3368 assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block")(static_cast <bool> (OrigLoop->getUniqueExitBlock() &&
"Expected a single exit block") ? void (0) : __assert_fail (
"OrigLoop->getUniqueExitBlock() && \"Expected a single exit block\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3368, __extension__
__PRETTY_FUNCTION__))
;
3369
3370 DenseMap<Value *, Value *> MissingVals;
3371
3372 // An external user of the last iteration's value should see the value that
3373 // the remainder loop uses to initialize its own IV.
3374 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());
3375 for (User *U : PostInc->users()) {
3376 Instruction *UI = cast<Instruction>(U);
3377 if (!OrigLoop->contains(UI)) {
3378 assert(isa<PHINode>(UI) && "Expected LCSSA form")(static_cast <bool> (isa<PHINode>(UI) && "Expected LCSSA form"
) ? void (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3378, __extension__
__PRETTY_FUNCTION__))
;
3379 MissingVals[UI] = EndValue;
3380 }
3381 }
3382
3383 // An external user of the penultimate value need to see EndValue - Step.
3384 // The simplest way to get this is to recompute it from the constituent SCEVs,
3385 // that is Start + (Step * (CRD - 1)).
3386 for (User *U : OrigPhi->users()) {
3387 auto *UI = cast<Instruction>(U);
3388 if (!OrigLoop->contains(UI)) {
3389 assert(isa<PHINode>(UI) && "Expected LCSSA form")(static_cast <bool> (isa<PHINode>(UI) && "Expected LCSSA form"
) ? void (0) : __assert_fail ("isa<PHINode>(UI) && \"Expected LCSSA form\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3389, __extension__
__PRETTY_FUNCTION__))
;
3390
3391 IRBuilder<> B(MiddleBlock->getTerminator());
3392
3393 // Fast-math-flags propagate from the original induction instruction.
3394 if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
3395 B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());
3396
3397 Value *CountMinusOne = B.CreateSub(
3398 CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));
3399 Value *CMO =
3400 !II.getStep()->getType()->isIntegerTy()
3401 ? B.CreateCast(Instruction::SIToFP, CountMinusOne,
3402 II.getStep()->getType())
3403 : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
3404 CMO->setName("cast.cmo");
3405
3406 Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(),
3407 VectorHeader->getTerminator());
3408 Value *Escape =
3409 emitTransformedIndex(B, CMO, II.getStartValue(), Step, II);
3410 Escape->setName("ind.escape");
3411 MissingVals[UI] = Escape;
3412 }
3413 }
3414
3415 for (auto &I : MissingVals) {
3416 PHINode *PHI = cast<PHINode>(I.first);
3417 // One corner case we have to handle is two IVs "chasing" each-other,
3418 // that is %IV2 = phi [...], [ %IV1, %latch ]
3419 // In this case, if IV1 has an external use, we need to avoid adding both
3420 // "last value of IV1" and "penultimate value of IV2". So, verify that we
3421 // don't already have an incoming value for the middle block.
3422 if (PHI->getBasicBlockIndex(MiddleBlock) == -1)
3423 PHI->addIncoming(I.second, MiddleBlock);
3424 }
3425}
3426
3427namespace {
3428
3429struct CSEDenseMapInfo {
3430 static bool canHandle(const Instruction *I) {
3431 return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
3432 isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
3433 }
3434
3435 static inline Instruction *getEmptyKey() {
3436 return DenseMapInfo<Instruction *>::getEmptyKey();
3437 }
3438
3439 static inline Instruction *getTombstoneKey() {
3440 return DenseMapInfo<Instruction *>::getTombstoneKey();
3441 }
3442
3443 static unsigned getHashValue(const Instruction *I) {
3444 assert(canHandle(I) && "Unknown instruction!")(static_cast <bool> (canHandle(I) && "Unknown instruction!"
) ? void (0) : __assert_fail ("canHandle(I) && \"Unknown instruction!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3444, __extension__
__PRETTY_FUNCTION__))
;
3445 return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
3446 I->value_op_end()));
3447 }
3448
3449 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
3450 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
3451 LHS == getTombstoneKey() || RHS == getTombstoneKey())
3452 return LHS == RHS;
3453 return LHS->isIdenticalTo(RHS);
3454 }
3455};
3456
3457} // end anonymous namespace
3458
3459///Perform cse of induction variable instructions.
3460static void cse(BasicBlock *BB) {
3461 // Perform simple cse.
3462 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
3463 for (Instruction &In : llvm::make_early_inc_range(*BB)) {
3464 if (!CSEDenseMapInfo::canHandle(&In))
3465 continue;
3466
3467 // Check if we can replace this instruction with any of the
3468 // visited instructions.
3469 if (Instruction *V = CSEMap.lookup(&In)) {
3470 In.replaceAllUsesWith(V);
3471 In.eraseFromParent();
3472 continue;
3473 }
3474
3475 CSEMap[&In] = &In;
3476 }
3477}
3478
3479InstructionCost
3480LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF,
3481 bool &NeedToScalarize) const {
3482 Function *F = CI->getCalledFunction();
3483 Type *ScalarRetTy = CI->getType();
3484 SmallVector<Type *, 4> Tys, ScalarTys;
3485 for (auto &ArgOp : CI->args())
3486 ScalarTys.push_back(ArgOp->getType());
3487
3488 // Estimate cost of scalarized vector call. The source operands are assumed
3489 // to be vectors, so we need to extract individual elements from there,
3490 // execute VF scalar calls, and then gather the result into the vector return
3491 // value.
3492 InstructionCost ScalarCallCost =
3493 TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys, TTI::TCK_RecipThroughput);
3494 if (VF.isScalar())
3495 return ScalarCallCost;
3496
3497 // Compute corresponding vector type for return value and arguments.
3498 Type *RetTy = ToVectorTy(ScalarRetTy, VF);
3499 for (Type *ScalarTy : ScalarTys)
3500 Tys.push_back(ToVectorTy(ScalarTy, VF));
3501
3502 // Compute costs of unpacking argument values for the scalar calls and
3503 // packing the return values to a vector.
3504 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
3505
3506 InstructionCost Cost =
3507 ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
3508
3509 // If we can't emit a vector call for this function, then the currently found
3510 // cost is the cost we need to return.
3511 NeedToScalarize = true;
3512 VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
3513 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3514
3515 if (!TLI || CI->isNoBuiltin() || !VecFunc)
3516 return Cost;
3517
3518 // If the corresponding vector cost is cheaper, return its cost.
3519 InstructionCost VectorCallCost =
3520 TTI.getCallInstrCost(nullptr, RetTy, Tys, TTI::TCK_RecipThroughput);
3521 if (VectorCallCost < Cost) {
3522 NeedToScalarize = false;
3523 Cost = VectorCallCost;
3524 }
3525 return Cost;
3526}
3527
3528static Type *MaybeVectorizeType(Type *Elt, ElementCount VF) {
3529 if (VF.isScalar() || (!Elt->isIntOrPtrTy() && !Elt->isFloatingPointTy()))
3530 return Elt;
3531 return VectorType::get(Elt, VF);
3532}
3533
3534InstructionCost
3535LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
3536 ElementCount VF) const {
3537 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3538 assert(ID && "Expected intrinsic call!")(static_cast <bool> (ID && "Expected intrinsic call!"
) ? void (0) : __assert_fail ("ID && \"Expected intrinsic call!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3538, __extension__
__PRETTY_FUNCTION__))
;
3539 Type *RetTy = MaybeVectorizeType(CI->getType(), VF);
3540 FastMathFlags FMF;
3541 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
3542 FMF = FPMO->getFastMathFlags();
3543
3544 SmallVector<const Value *> Arguments(CI->args());
3545 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
3546 SmallVector<Type *> ParamTys;
3547 std::transform(FTy->param_begin(), FTy->param_end(),
3548 std::back_inserter(ParamTys),
3549 [&](Type *Ty) { return MaybeVectorizeType(Ty, VF); });
3550
3551 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
3552 dyn_cast<IntrinsicInst>(CI));
3553 return TTI.getIntrinsicInstrCost(CostAttrs,
3554 TargetTransformInfo::TCK_RecipThroughput);
3555}
3556
3557static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
3558 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3559 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3560 return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
3561}
3562
3563static Type *largestIntegerVectorType(Type *T1, Type *T2) {
3564 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3565 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3566 return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
3567}
3568
3569void InnerLoopVectorizer::truncateToMinimalBitwidths(VPTransformState &State) {
3570 // For every instruction `I` in MinBWs, truncate the operands, create a
3571 // truncated version of `I` and reextend its result. InstCombine runs
3572 // later and will remove any ext/trunc pairs.
3573 SmallPtrSet<Value *, 4> Erased;
3574 for (const auto &KV : Cost->getMinimalBitwidths()) {
3575 // If the value wasn't vectorized, we must maintain the original scalar
3576 // type. The absence of the value from State indicates that it
3577 // wasn't vectorized.
3578 // FIXME: Should not rely on getVPValue at this point.
3579 VPValue *Def = State.Plan->getVPValue(KV.first, true);
3580 if (!State.hasAnyVectorValue(Def))
3581 continue;
3582 for (unsigned Part = 0; Part < UF; ++Part) {
3583 Value *I = State.get(Def, Part);
3584 if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I))
3585 continue;
3586 Type *OriginalTy = I->getType();
3587 Type *ScalarTruncatedTy =
3588 IntegerType::get(OriginalTy->getContext(), KV.second);
3589 auto *TruncatedTy = VectorType::get(
3590 ScalarTruncatedTy, cast<VectorType>(OriginalTy)->getElementCount());
3591 if (TruncatedTy == OriginalTy)
3592 continue;
3593
3594 IRBuilder<> B(cast<Instruction>(I));
3595 auto ShrinkOperand = [&](Value *V) -> Value * {
3596 if (auto *ZI = dyn_cast<ZExtInst>(V))
3597 if (ZI->getSrcTy() == TruncatedTy)
3598 return ZI->getOperand(0);
3599 return B.CreateZExtOrTrunc(V, TruncatedTy);
3600 };
3601
3602 // The actual instruction modification depends on the instruction type,
3603 // unfortunately.
3604 Value *NewI = nullptr;
3605 if (auto *BO = dyn_cast<BinaryOperator>(I)) {
3606 NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),
3607 ShrinkOperand(BO->getOperand(1)));
3608
3609 // Any wrapping introduced by shrinking this operation shouldn't be
3610 // considered undefined behavior. So, we can't unconditionally copy
3611 // arithmetic wrapping flags to NewI.
3612 cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);
3613 } else if (auto *CI = dyn_cast<ICmpInst>(I)) {
3614 NewI =
3615 B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),
3616 ShrinkOperand(CI->getOperand(1)));
3617 } else if (auto *SI = dyn_cast<SelectInst>(I)) {
3618 NewI = B.CreateSelect(SI->getCondition(),
3619 ShrinkOperand(SI->getTrueValue()),
3620 ShrinkOperand(SI->getFalseValue()));
3621 } else if (auto *CI = dyn_cast<CastInst>(I)) {
3622 switch (CI->getOpcode()) {
3623 default:
3624 llvm_unreachable("Unhandled cast!")::llvm::llvm_unreachable_internal("Unhandled cast!", "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 3624)
;
3625 case Instruction::Trunc:
3626 NewI = ShrinkOperand(CI->getOperand(0));
3627 break;
3628 case Instruction::SExt:
3629 NewI = B.CreateSExtOrTrunc(
3630 CI->getOperand(0),
3631 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3632 break;
3633 case Instruction::ZExt:
3634 NewI = B.CreateZExtOrTrunc(
3635 CI->getOperand(0),
3636 smallestIntegerVectorType(OriginalTy, TruncatedTy));
3637 break;
3638 }
3639 } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {
3640 auto Elements0 =
3641 cast<VectorType>(SI->getOperand(0)->getType())->getElementCount();
3642 auto *O0 = B.CreateZExtOrTrunc(
3643 SI->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements0));
3644 auto Elements1 =
3645 cast<VectorType>(SI->getOperand(1)->getType())->getElementCount();
3646 auto *O1 = B.CreateZExtOrTrunc(
3647 SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
3648
3649 NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
3650 } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
3651 // Don't do anything with the operands, just extend the result.
3652 continue;
3653 } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
3654 auto Elements =
3655 cast<VectorType>(IE->getOperand(0)->getType())->getElementCount();
3656 auto *O0 = B.CreateZExtOrTrunc(
3657 IE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));
3658 auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);
3659 NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));
3660 } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
3661 auto Elements =
3662 cast<VectorType>(EE->getOperand(0)->getType())->getElementCount();
3663 auto *O0 = B.CreateZExtOrTrunc(
3664 EE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));
3665 NewI = B.CreateExtractElement(O0, EE->getOperand(2));
3666 } else {
3667 // If we don't know what to do, be conservative and don't do anything.
3668 continue;
3669 }
3670
3671 // Lastly, extend the result.
3672 NewI->takeName(cast<Instruction>(I));
3673 Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
3674 I->replaceAllUsesWith(Res);
3675 cast<Instruction>(I)->eraseFromParent();
3676 Erased.insert(I);
3677 State.reset(Def, Res, Part);
3678 }
3679 }
3680
3681 // We'll have created a bunch of ZExts that are now parentless. Clean up.
3682 for (const auto &KV : Cost->getMinimalBitwidths()) {
3683 // If the value wasn't vectorized, we must maintain the original scalar
3684 // type. The absence of the value from State indicates that it
3685 // wasn't vectorized.
3686 // FIXME: Should not rely on getVPValue at this point.
3687 VPValue *Def = State.Plan->getVPValue(KV.first, true);
3688 if (!State.hasAnyVectorValue(Def))
3689 continue;
3690 for (unsigned Part = 0; Part < UF; ++Part) {
3691 Value *I = State.get(Def, Part);
3692 ZExtInst *Inst = dyn_cast<ZExtInst>(I);
3693 if (Inst && Inst->use_empty()) {
3694 Value *NewI = Inst->getOperand(0);
3695 Inst->eraseFromParent();
3696 State.reset(Def, NewI, Part);
3697 }
3698 }
3699 }
3700}
3701
3702void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
3703 // Insert truncates and extends for any truncated instructions as hints to
3704 // InstCombine.
3705 if (VF.isVector())
3706 truncateToMinimalBitwidths(State);
3707
3708 // Fix widened non-induction PHIs by setting up the PHI operands.
3709 if (OrigPHIsToFix.size()) {
3710 assert(EnableVPlanNativePath &&(static_cast <bool> (EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? void (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3711, __extension__
__PRETTY_FUNCTION__))
3711 "Unexpected non-induction PHIs for fixup in non VPlan-native path")(static_cast <bool> (EnableVPlanNativePath && "Unexpected non-induction PHIs for fixup in non VPlan-native path"
) ? void (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected non-induction PHIs for fixup in non VPlan-native path\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3711, __extension__
__PRETTY_FUNCTION__))
;
3712 fixNonInductionPHIs(State);
3713 }
3714
3715 // At this point every instruction in the original loop is widened to a
3716 // vector form. Now we need to fix the recurrences in the loop. These PHI
3717 // nodes are currently empty because we did not want to introduce cycles.
3718 // This is the second stage of vectorizing recurrences.
3719 fixCrossIterationPHIs(State);
3720
3721 // Forget the original basic block.
3722 PSE.getSE()->forgetLoop(OrigLoop);
3723
3724 Loop *VectorLoop = LI->getLoopFor(State.CFG.PrevBB);
3725 // If we inserted an edge from the middle block to the unique exit block,
3726 // update uses outside the loop (phis) to account for the newly inserted
3727 // edge.
3728 if (!Cost->requiresScalarEpilogue(VF)) {
3729 // Fix-up external users of the induction variables.
3730 for (auto &Entry : Legal->getInductionVars())
3731 fixupIVUsers(
3732 Entry.first, Entry.second, getOrCreateVectorTripCount(VectorLoop),
3733 IVEndValues[Entry.first], LoopMiddleBlock, VectorLoop->getHeader());
3734
3735 fixLCSSAPHIs(State);
3736 }
3737
3738 for (Instruction *PI : PredicatedInstructions)
3739 sinkScalarOperands(&*PI);
3740
3741 // Remove redundant induction instructions.
3742 cse(VectorLoop->getHeader());
3743
3744 // Set/update profile weights for the vector and remainder loops as original
3745 // loop iterations are now distributed among them. Note that original loop
3746 // represented by LoopScalarBody becomes remainder loop after vectorization.
3747 //
3748 // For cases like foldTailByMasking() and requiresScalarEpiloque() we may
3749 // end up getting slightly roughened result but that should be OK since
3750 // profile is not inherently precise anyway. Note also possible bypass of
3751 // vector code caused by legality checks is ignored, assigning all the weight
3752 // to the vector loop, optimistically.
3753 //
3754 // For scalable vectorization we can't know at compile time how many iterations
3755 // of the loop are handled in one vector iteration, so instead assume a pessimistic
3756 // vscale of '1'.
3757 setProfileInfoAfterUnrolling(LI->getLoopFor(LoopScalarBody), VectorLoop,
3758 LI->getLoopFor(LoopScalarBody),
3759 VF.getKnownMinValue() * UF);
3760}
3761
3762void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
3763 // In order to support recurrences we need to be able to vectorize Phi nodes.
3764 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
3765 // stage #2: We now need to fix the recurrences by adding incoming edges to
3766 // the currently empty PHI nodes. At this point every instruction in the
3767 // original loop is widened to a vector form so we can use them to construct
3768 // the incoming edges.
3769 VPBasicBlock *Header = State.Plan->getEntry()->getEntryBasicBlock();
3770 for (VPRecipeBase &R : Header->phis()) {
3771 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R))
3772 fixReduction(ReductionPhi, State);
3773 else if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
3774 fixFirstOrderRecurrence(FOR, State);
3775 }
3776}
3777
3778void InnerLoopVectorizer::fixFirstOrderRecurrence(
3779 VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State) {
3780 // This is the second phase of vectorizing first-order recurrences. An
3781 // overview of the transformation is described below. Suppose we have the
3782 // following loop.
3783 //
3784 // for (int i = 0; i < n; ++i)
3785 // b[i] = a[i] - a[i - 1];
3786 //
3787 // There is a first-order recurrence on "a". For this loop, the shorthand
3788 // scalar IR looks like:
3789 //
3790 // scalar.ph:
3791 // s_init = a[-1]
3792 // br scalar.body
3793 //
3794 // scalar.body:
3795 // i = phi [0, scalar.ph], [i+1, scalar.body]
3796 // s1 = phi [s_init, scalar.ph], [s2, scalar.body]
3797 // s2 = a[i]
3798 // b[i] = s2 - s1
3799 // br cond, scalar.body, ...
3800 //
3801 // In this example, s1 is a recurrence because it's value depends on the
3802 // previous iteration. In the first phase of vectorization, we created a
3803 // vector phi v1 for s1. We now complete the vectorization and produce the
3804 // shorthand vector IR shown below (for VF = 4, UF = 1).
3805 //
3806 // vector.ph:
3807 // v_init = vector(..., ..., ..., a[-1])
3808 // br vector.body
3809 //
3810 // vector.body
3811 // i = phi [0, vector.ph], [i+4, vector.body]
3812 // v1 = phi [v_init, vector.ph], [v2, vector.body]
3813 // v2 = a[i, i+1, i+2, i+3];
3814 // v3 = vector(v1(3), v2(0, 1, 2))
3815 // b[i, i+1, i+2, i+3] = v2 - v3
3816 // br cond, vector.body, middle.block
3817 //
3818 // middle.block:
3819 // x = v2(3)
3820 // br scalar.ph
3821 //
3822 // scalar.ph:
3823 // s_init = phi [x, middle.block], [a[-1], otherwise]
3824 // br scalar.body
3825 //
3826 // After execution completes the vector loop, we extract the next value of
3827 // the recurrence (x) to use as the initial value in the scalar loop.
3828
3829 // Extract the last vector element in the middle block. This will be the
3830 // initial value for the recurrence when jumping to the scalar loop.
3831 VPValue *PreviousDef = PhiR->getBackedgeValue();
3832 Value *Incoming = State.get(PreviousDef, UF - 1);
3833 auto *ExtractForScalar = Incoming;
3834 auto *IdxTy = Builder.getInt32Ty();
3835 if (VF.isVector()) {
3836 auto *One = ConstantInt::get(IdxTy, 1);
3837 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
3838 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF);
3839 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
3840 ExtractForScalar = Builder.CreateExtractElement(ExtractForScalar, LastIdx,
3841 "vector.recur.extract");
3842 }
3843 // Extract the second last element in the middle block if the
3844 // Phi is used outside the loop. We need to extract the phi itself
3845 // and not the last element (the phi update in the current iteration). This
3846 // will be the value when jumping to the exit block from the LoopMiddleBlock,
3847 // when the scalar loop is not run at all.
3848 Value *ExtractForPhiUsedOutsideLoop = nullptr;
3849 if (VF.isVector()) {
3850 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF);
3851 auto *Idx = Builder.CreateSub(RuntimeVF, ConstantInt::get(IdxTy, 2));
3852 ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(
3853 Incoming, Idx, "vector.recur.extract.for.phi");
3854 } else if (UF > 1)
3855 // When loop is unrolled without vectorizing, initialize
3856 // ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value
3857 // of `Incoming`. This is analogous to the vectorized case above: extracting
3858 // the second last element when VF > 1.
3859 ExtractForPhiUsedOutsideLoop = State.get(PreviousDef, UF - 2);
3860
3861 // Fix the initial value of the original recurrence in the scalar loop.
3862 Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
3863 PHINode *Phi = cast<PHINode>(PhiR->getUnderlyingValue());
3864 auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
3865 auto *ScalarInit = PhiR->getStartValue()->getLiveInIRValue();
3866 for (auto *BB : predecessors(LoopScalarPreHeader)) {
3867 auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
3868 Start->addIncoming(Incoming, BB);
3869 }
3870
3871 Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start);
3872 Phi->setName("scalar.recur");
3873
3874 // Finally, fix users of the recurrence outside the loop. The users will need
3875 // either the last value of the scalar recurrence or the last value of the
3876 // vector recurrence we extracted in the middle block. Since the loop is in
3877 // LCSSA form, we just need to find all the phi nodes for the original scalar
3878 // recurrence in the exit block, and then add an edge for the middle block.
3879 // Note that LCSSA does not imply single entry when the original scalar loop
3880 // had multiple exiting edges (as we always run the last iteration in the
3881 // scalar epilogue); in that case, there is no edge from middle to exit and
3882 // and thus no phis which needed updated.
3883 if (!Cost->requiresScalarEpilogue(VF))
3884 for (PHINode &LCSSAPhi : LoopExitBlock->phis())
3885 if (llvm::is_contained(LCSSAPhi.incoming_values(), Phi))
3886 LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
3887}
3888
3889void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
3890 VPTransformState &State) {
3891 PHINode *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
3892 // Get it's reduction variable descriptor.
3893 assert(Legal->isReductionVariable(OrigPhi) &&(static_cast <bool> (Legal->isReductionVariable(OrigPhi
) && "Unable to find the reduction variable") ? void (
0) : __assert_fail ("Legal->isReductionVariable(OrigPhi) && \"Unable to find the reduction variable\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3894, __extension__
__PRETTY_FUNCTION__))
3894 "Unable to find the reduction variable")(static_cast <bool> (Legal->isReductionVariable(OrigPhi
) && "Unable to find the reduction variable") ? void (
0) : __assert_fail ("Legal->isReductionVariable(OrigPhi) && \"Unable to find the reduction variable\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3894, __extension__
__PRETTY_FUNCTION__))
;
3895 const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
3896
3897 RecurKind RK = RdxDesc.getRecurrenceKind();
3898 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
3899 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
3900 setDebugLocFromInst(ReductionStartValue);
3901
3902 VPValue *LoopExitInstDef = PhiR->getBackedgeValue();
3903 // This is the vector-clone of the value that leaves the loop.
3904 Type *VecTy = State.get(LoopExitInstDef, 0)->getType();
3905
3906 // Wrap flags are in general invalid after vectorization, clear them.
3907 clearReductionWrapFlags(RdxDesc, State);
3908
3909 // Before each round, move the insertion point right between
3910 // the PHIs and the values we are going to write.
3911 // This allows us to write both PHINodes and the extractelement
3912 // instructions.
3913 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
3914
3915 setDebugLocFromInst(LoopExitInst);
3916
3917 Type *PhiTy = OrigPhi->getType();
3918 BasicBlock *VectorLoopLatch =
3919 LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch();
3920 // If tail is folded by masking, the vector value to leave the loop should be
3921 // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
3922 // instead of the former. For an inloop reduction the reduction will already
3923 // be predicated, and does not need to be handled here.
3924 if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
3925 for (unsigned Part = 0; Part < UF; ++Part) {
3926 Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
3927 Value *Sel = nullptr;
3928 for (User *U : VecLoopExitInst->users()) {
3929 if (isa<SelectInst>(U)) {
3930 assert(!Sel && "Reduction exit feeding two selects")(static_cast <bool> (!Sel && "Reduction exit feeding two selects"
) ? void (0) : __assert_fail ("!Sel && \"Reduction exit feeding two selects\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3930, __extension__
__PRETTY_FUNCTION__))
;
3931 Sel = U;
3932 } else
3933 assert(isa<PHINode>(U) && "Reduction exit must feed Phi's or select")(static_cast <bool> (isa<PHINode>(U) && "Reduction exit must feed Phi's or select"
) ? void (0) : __assert_fail ("isa<PHINode>(U) && \"Reduction exit must feed Phi's or select\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3933, __extension__
__PRETTY_FUNCTION__))
;
3934 }
3935 assert(Sel && "Reduction exit feeds no select")(static_cast <bool> (Sel && "Reduction exit feeds no select"
) ? void (0) : __assert_fail ("Sel && \"Reduction exit feeds no select\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3935, __extension__
__PRETTY_FUNCTION__))
;
3936 State.reset(LoopExitInstDef, Sel, Part);
3937
3938 // If the target can create a predicated operator for the reduction at no
3939 // extra cost in the loop (for example a predicated vadd), it can be
3940 // cheaper for the select to remain in the loop than be sunk out of it,
3941 // and so use the select value for the phi instead of the old
3942 // LoopExitValue.
3943 if (PreferPredicatedReductionSelect ||
3944 TTI->preferPredicatedReductionSelect(
3945 RdxDesc.getOpcode(), PhiTy,
3946 TargetTransformInfo::ReductionFlags())) {
3947 auto *VecRdxPhi =
3948 cast<PHINode>(State.get(PhiR, Part));
3949 VecRdxPhi->setIncomingValueForBlock(VectorLoopLatch, Sel);
3950 }
3951 }
3952 }
3953
3954 // If the vector reduction can be performed in a smaller type, we truncate
3955 // then extend the loop exit value to enable InstCombine to evaluate the
3956 // entire expression in the smaller type.
3957 if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
3958 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!")(static_cast <bool> (!PhiR->isInLoop() && "Unexpected truncated inloop reduction!"
) ? void (0) : __assert_fail ("!PhiR->isInLoop() && \"Unexpected truncated inloop reduction!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 3958, __extension__
__PRETTY_FUNCTION__))
;
3959 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
3960 Builder.SetInsertPoint(VectorLoopLatch->getTerminator());
3961 VectorParts RdxParts(UF);
3962 for (unsigned Part = 0; Part < UF; ++Part) {
3963 RdxParts[Part] = State.get(LoopExitInstDef, Part);
3964 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
3965 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
3966 : Builder.CreateZExt(Trunc, VecTy);
3967 for (User *U : llvm::make_early_inc_range(RdxParts[Part]->users()))
3968 if (U != Trunc) {
3969 U->replaceUsesOfWith(RdxParts[Part], Extnd);
3970 RdxParts[Part] = Extnd;
3971 }
3972 }
3973 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
3974 for (unsigned Part = 0; Part < UF; ++Part) {
3975 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
3976 State.reset(LoopExitInstDef, RdxParts[Part], Part);
3977 }
3978 }
3979
3980 // Reduce all of the unrolled parts into a single vector.
3981 Value *ReducedPartRdx = State.get(LoopExitInstDef, 0);
3982 unsigned Op = RecurrenceDescriptor::getOpcode(RK);
3983
3984 // The middle block terminator has already been assigned a DebugLoc here (the
3985 // OrigLoop's single latch terminator). We want the whole middle block to
3986 // appear to execute on this line because: (a) it is all compiler generated,
3987 // (b) these instructions are always executed after evaluating the latch
3988 // conditional branch, and (c) other passes may add new predecessors which
3989 // terminate on this line. This is the easiest way to ensure we don't
3990 // accidentally cause an extra step back into the loop while debugging.
3991 setDebugLocFromInst(LoopMiddleBlock->getTerminator());
3992 if (PhiR->isOrdered())
3993 ReducedPartRdx = State.get(LoopExitInstDef, UF - 1);
3994 else {
3995 // Floating-point operations should have some FMF to enable the reduction.
3996 IRBuilderBase::FastMathFlagGuard FMFG(Builder);
3997 Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
3998 for (unsigned Part = 1; Part < UF; ++Part) {
3999 Value *RdxPart = State.get(LoopExitInstDef, Part);
4000 if (Op != Instruction::ICmp && Op != Instruction::FCmp) {
4001 ReducedPartRdx = Builder.CreateBinOp(
4002 (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx");
4003 } else if (RecurrenceDescriptor::isSelectCmpRecurrenceKind(RK))
4004 ReducedPartRdx = createSelectCmpOp(Builder, ReductionStartValue, RK,
4005 ReducedPartRdx, RdxPart);
4006 else
4007 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
4008 }
4009 }
4010
4011 // Create the reduction after the loop. Note that inloop reductions create the
4012 // target reduction in the loop using a Reduction recipe.
4013 if (VF.isVector() && !PhiR->isInLoop()) {
4014 ReducedPartRdx =
4015 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx, OrigPhi);
4016 // If the reduction can be performed in a smaller type, we need to extend
4017 // the reduction to the wider type before we branch to the original loop.
4018 if (PhiTy != RdxDesc.getRecurrenceType())
4019 ReducedPartRdx = RdxDesc.isSigned()
4020 ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
4021 : Builder.CreateZExt(ReducedPartRdx, PhiTy);
4022 }
4023
4024 PHINode *ResumePhi =
4025 dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
4026
4027 // Create a phi node that merges control-flow from the backedge-taken check
4028 // block and the middle block.
4029 PHINode *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx",
4030 LoopScalarPreHeader->getTerminator());
4031
4032 // If we are fixing reductions in the epilogue loop then we should already
4033 // have created a bc.merge.rdx Phi after the main vector body. Ensure that
4034 // we carry over the incoming values correctly.
4035 for (auto *Incoming : predecessors(LoopScalarPreHeader)) {
4036 if (Incoming == LoopMiddleBlock)
4037 BCBlockPhi->addIncoming(ReducedPartRdx, Incoming);
4038 else if (ResumePhi && llvm::is_contained(ResumePhi->blocks(), Incoming))
4039 BCBlockPhi->addIncoming(ResumePhi->getIncomingValueForBlock(Incoming),
4040 Incoming);
4041 else
4042 BCBlockPhi->addIncoming(ReductionStartValue, Incoming);
4043 }
4044
4045 // Set the resume value for this reduction
4046 ReductionResumeValues.insert({&RdxDesc, BCBlockPhi});
4047
4048 // Now, we need to fix the users of the reduction variable
4049 // inside and outside of the scalar remainder loop.
4050
4051 // We know that the loop is in LCSSA form. We need to update the PHI nodes
4052 // in the exit blocks. See comment on analogous loop in
4053 // fixFirstOrderRecurrence for a more complete explaination of the logic.
4054 if (!Cost->requiresScalarEpilogue(VF))
4055 for (PHINode &LCSSAPhi : LoopExitBlock->phis())
4056 if (llvm::is_contained(LCSSAPhi.incoming_values(), LoopExitInst))
4057 LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);
4058
4059 // Fix the scalar loop reduction variable with the incoming reduction sum
4060 // from the vector body and from the backedge value.
4061 int IncomingEdgeBlockIdx =
4062 OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch());
4063 assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index")(static_cast <bool> (IncomingEdgeBlockIdx >= 0 &&
"Invalid block index") ? void (0) : __assert_fail ("IncomingEdgeBlockIdx >= 0 && \"Invalid block index\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4063, __extension__
__PRETTY_FUNCTION__))
;
4064 // Pick the other block.
4065 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
4066 OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
4067 OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
4068}
4069
4070void InnerLoopVectorizer::clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
4071 VPTransformState &State) {
4072 RecurKind RK = RdxDesc.getRecurrenceKind();
4073 if (RK != RecurKind::Add && RK != RecurKind::Mul)
4074 return;
4075
4076 Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
4077 assert(LoopExitInstr && "null loop exit instruction")(static_cast <bool> (LoopExitInstr && "null loop exit instruction"
) ? void (0) : __assert_fail ("LoopExitInstr && \"null loop exit instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4077, __extension__
__PRETTY_FUNCTION__))
;
4078 SmallVector<Instruction *, 8> Worklist;
4079 SmallPtrSet<Instruction *, 8> Visited;
4080 Worklist.push_back(LoopExitInstr);
4081 Visited.insert(LoopExitInstr);
4082
4083 while (!Worklist.empty()) {
4084 Instruction *Cur = Worklist.pop_back_val();
4085 if (isa<OverflowingBinaryOperator>(Cur))
4086 for (unsigned Part = 0; Part < UF; ++Part) {
4087 // FIXME: Should not rely on getVPValue at this point.
4088 Value *V = State.get(State.Plan->getVPValue(Cur, true), Part);
4089 cast<Instruction>(V)->dropPoisonGeneratingFlags();
4090 }
4091
4092 for (User *U : Cur->users()) {
4093 Instruction *UI = cast<Instruction>(U);
4094 if ((Cur != LoopExitInstr || OrigLoop->contains(UI->getParent())) &&
4095 Visited.insert(UI).second)
4096 Worklist.push_back(UI);
4097 }
4098 }
4099}
4100
4101void InnerLoopVectorizer::fixLCSSAPHIs(VPTransformState &State) {
4102 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
4103 if (LCSSAPhi.getBasicBlockIndex(LoopMiddleBlock) != -1)
4104 // Some phis were already hand updated by the reduction and recurrence
4105 // code above, leave them alone.
4106 continue;
4107
4108 auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
4109 // Non-instruction incoming values will have only one value.
4110
4111 VPLane Lane = VPLane::getFirstLane();
4112 if (isa<Instruction>(IncomingValue) &&
4113 !Cost->isUniformAfterVectorization(cast<Instruction>(IncomingValue),
4114 VF))
4115 Lane = VPLane::getLastLaneForVF(VF);
4116
4117 // Can be a loop invariant incoming value or the last scalar value to be
4118 // extracted from the vectorized loop.
4119 // FIXME: Should not rely on getVPValue at this point.
4120 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
4121 Value *lastIncomingValue =
4122 OrigLoop->isLoopInvariant(IncomingValue)
4123 ? IncomingValue
4124 : State.get(State.Plan->getVPValue(IncomingValue, true),
4125 VPIteration(UF - 1, Lane));
4126 LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock);
4127 }
4128}
4129
4130void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
4131 // The basic block and loop containing the predicated instruction.
4132 auto *PredBB = PredInst->getParent();
4133 auto *VectorLoop = LI->getLoopFor(PredBB);
4134
4135 // Initialize a worklist with the operands of the predicated instruction.
4136 SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());
4137
4138 // Holds instructions that we need to analyze again. An instruction may be
4139 // reanalyzed if we don't yet know if we can sink it or not.
4140 SmallVector<Instruction *, 8> InstsToReanalyze;
4141
4142 // Returns true if a given use occurs in the predicated block. Phi nodes use
4143 // their operands in their corresponding predecessor blocks.
4144 auto isBlockOfUsePredicated = [&](Use &U) -> bool {
4145 auto *I = cast<Instruction>(U.getUser());
4146 BasicBlock *BB = I->getParent();
4147 if (auto *Phi = dyn_cast<PHINode>(I))
4148 BB = Phi->getIncomingBlock(
4149 PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
4150 return BB == PredBB;
4151 };
4152
4153 // Iteratively sink the scalarized operands of the predicated instruction
4154 // into the block we created for it. When an instruction is sunk, it's
4155 // operands are then added to the worklist. The algorithm ends after one pass
4156 // through the worklist doesn't sink a single instruction.
4157 bool Changed;
4158 do {
4159 // Add the instructions that need to be reanalyzed to the worklist, and
4160 // reset the changed indicator.
4161 Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());
4162 InstsToReanalyze.clear();
4163 Changed = false;
4164
4165 while (!Worklist.empty()) {
4166 auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());
4167
4168 // We can't sink an instruction if it is a phi node, is not in the loop,
4169 // or may have side effects.
4170 if (!I || isa<PHINode>(I) || !VectorLoop->contains(I) ||
4171 I->mayHaveSideEffects())
4172 continue;
4173
4174 // If the instruction is already in PredBB, check if we can sink its
4175 // operands. In that case, VPlan's sinkScalarOperands() succeeded in
4176 // sinking the scalar instruction I, hence it appears in PredBB; but it
4177 // may have failed to sink I's operands (recursively), which we try
4178 // (again) here.
4179 if (I->getParent() == PredBB) {
4180 Worklist.insert(I->op_begin(), I->op_end());
4181 continue;
4182 }
4183
4184 // It's legal to sink the instruction if all its uses occur in the
4185 // predicated block. Otherwise, there's nothing to do yet, and we may
4186 // need to reanalyze the instruction.
4187 if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) {
4188 InstsToReanalyze.push_back(I);
4189 continue;
4190 }
4191
4192 // Move the instruction to the beginning of the predicated block, and add
4193 // it's operands to the worklist.
4194 I->moveBefore(&*PredBB->getFirstInsertionPt());
4195 Worklist.insert(I->op_begin(), I->op_end());
4196
4197 // The sinking may have enabled other instructions to be sunk, so we will
4198 // need to iterate.
4199 Changed = true;
4200 }
4201 } while (Changed);
4202}
4203
4204void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
4205 for (PHINode *OrigPhi : OrigPHIsToFix) {
4206 VPWidenPHIRecipe *VPPhi =
4207 cast<VPWidenPHIRecipe>(State.Plan->getVPValue(OrigPhi));
4208 PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, 0));
4209 // Make sure the builder has a valid insert point.
4210 Builder.SetInsertPoint(NewPhi);
4211 for (unsigned i = 0; i < VPPhi->getNumOperands(); ++i) {
4212 VPValue *Inc = VPPhi->getIncomingValue(i);
4213 VPBasicBlock *VPBB = VPPhi->getIncomingBlock(i);
4214 NewPhi->addIncoming(State.get(Inc, 0), State.CFG.VPBB2IRBB[VPBB]);
4215 }
4216 }
4217}
4218
4219bool InnerLoopVectorizer::useOrderedReductions(
4220 const RecurrenceDescriptor &RdxDesc) {
4221 return Cost->useOrderedReductions(RdxDesc);
4222}
4223
4224void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
4225 VPWidenPHIRecipe *PhiR,
4226 VPTransformState &State) {
4227 PHINode *P = cast<PHINode>(PN);
4228 if (EnableVPlanNativePath) {
4229 // Currently we enter here in the VPlan-native path for non-induction
4230 // PHIs where all control flow is uniform. We simply widen these PHIs.
4231 // Create a vector phi with no operands - the vector phi operands will be
4232 // set at the end of vector code generation.
4233 Type *VecTy = (State.VF.isScalar())
4234 ? PN->getType()
4235 : VectorType::get(PN->getType(), State.VF);
4236 Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
4237 State.set(PhiR, VecPhi, 0);
4238 OrigPHIsToFix.push_back(P);
4239
4240 return;
4241 }
4242
4243 assert(PN->getParent() == OrigLoop->getHeader() &&(static_cast <bool> (PN->getParent() == OrigLoop->
getHeader() && "Non-header phis should have been handled elsewhere"
) ? void (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4244, __extension__
__PRETTY_FUNCTION__))
4244 "Non-header phis should have been handled elsewhere")(static_cast <bool> (PN->getParent() == OrigLoop->
getHeader() && "Non-header phis should have been handled elsewhere"
) ? void (0) : __assert_fail ("PN->getParent() == OrigLoop->getHeader() && \"Non-header phis should have been handled elsewhere\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4244, __extension__
__PRETTY_FUNCTION__))
;
4245
4246 // In order to support recurrences we need to be able to vectorize Phi nodes.
4247 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
4248 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
4249 // this value when we vectorize all of the instructions that use the PHI.
4250
4251 assert(!Legal->isReductionVariable(P) &&(static_cast <bool> (!Legal->isReductionVariable(P) &&
"reductions should be handled elsewhere") ? void (0) : __assert_fail
("!Legal->isReductionVariable(P) && \"reductions should be handled elsewhere\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4252, __extension__
__PRETTY_FUNCTION__))
4252 "reductions should be handled elsewhere")(static_cast <bool> (!Legal->isReductionVariable(P) &&
"reductions should be handled elsewhere") ? void (0) : __assert_fail
("!Legal->isReductionVariable(P) && \"reductions should be handled elsewhere\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4252, __extension__
__PRETTY_FUNCTION__))
;
4253
4254 setDebugLocFromInst(P);
4255
4256 // This PHINode must be an induction variable.
4257 // Make sure that we know about it.
4258 assert(Legal->getInductionVars().count(P) && "Not an induction variable")(static_cast <bool> (Legal->getInductionVars().count
(P) && "Not an induction variable") ? void (0) : __assert_fail
("Legal->getInductionVars().count(P) && \"Not an induction variable\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4258, __extension__
__PRETTY_FUNCTION__))
;
4259
4260 InductionDescriptor II = Legal->getInductionVars().lookup(P);
4261 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
4262
4263 auto *IVR = PhiR->getParent()->getPlan()->getCanonicalIV();
4264 PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0));
4265
4266 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
4267 // which can be found from the original scalar operations.
4268 switch (II.getKind()) {
4269 case InductionDescriptor::IK_NoInduction:
4270 llvm_unreachable("Unknown induction")::llvm::llvm_unreachable_internal("Unknown induction", "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp"
, 4270)
;
4271 case InductionDescriptor::IK_IntInduction:
4272 case InductionDescriptor::IK_FpInduction:
4273 llvm_unreachable("Integer/fp induction is handled elsewhere.")::llvm::llvm_unreachable_internal("Integer/fp induction is handled elsewhere."
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4273)
;
4274 case InductionDescriptor::IK_PtrInduction: {
4275 // Handle the pointer induction variable case.
4276 assert(P->getType()->isPointerTy() && "Unexpected type.")(static_cast <bool> (P->getType()->isPointerTy() &&
"Unexpected type.") ? void (0) : __assert_fail ("P->getType()->isPointerTy() && \"Unexpected type.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4276, __extension__
__PRETTY_FUNCTION__))
;
4277
4278 if (all_of(PhiR->users(), [PhiR](const VPUser *U) {
4279 return cast<VPRecipeBase>(U)->usesScalars(PhiR);
4280 })) {
4281 // This is the normalized GEP that starts counting at zero.
4282 Value *PtrInd =
4283 Builder.CreateSExtOrTrunc(CanonicalIV, II.getStep()->getType());
4284 // Determine the number of scalars we need to generate for each unroll
4285 // iteration. If the instruction is uniform, we only need to generate the
4286 // first lane. Otherwise, we generate all VF values.
4287 bool IsUniform = vputils::onlyFirstLaneUsed(PhiR);
4288 assert((IsUniform || !State.VF.isScalable()) &&(static_cast <bool> ((IsUniform || !State.VF.isScalable
()) && "Cannot scalarize a scalable VF") ? void (0) :
__assert_fail ("(IsUniform || !State.VF.isScalable()) && \"Cannot scalarize a scalable VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4289, __extension__
__PRETTY_FUNCTION__))
4289 "Cannot scalarize a scalable VF")(static_cast <bool> ((IsUniform || !State.VF.isScalable
()) && "Cannot scalarize a scalable VF") ? void (0) :
__assert_fail ("(IsUniform || !State.VF.isScalable()) && \"Cannot scalarize a scalable VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4289, __extension__
__PRETTY_FUNCTION__))
;
4290 unsigned Lanes = IsUniform ? 1 : State.VF.getFixedValue();
4291
4292 for (unsigned Part = 0; Part < UF; ++Part) {
4293 Value *PartStart =
4294 createStepForVF(Builder, PtrInd->getType(), VF, Part);
4295
4296 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
4297 Value *Idx = Builder.CreateAdd(
4298 PartStart, ConstantInt::get(PtrInd->getType(), Lane));
4299 Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
4300
4301 Value *Step = CreateStepValue(II.getStep(), *PSE.getSE(),
4302 State.CFG.PrevBB->getTerminator());
4303 Value *SclrGep = emitTransformedIndex(Builder, GlobalIdx,
4304 II.getStartValue(), Step, II);
4305 SclrGep->setName("next.gep");
4306 State.set(PhiR, SclrGep, VPIteration(Part, Lane));
4307 }
4308 }
4309 return;
4310 }
4311 assert(isa<SCEVConstant>(II.getStep()) &&(static_cast <bool> (isa<SCEVConstant>(II.getStep
()) && "Induction step not a SCEV constant!") ? void (
0) : __assert_fail ("isa<SCEVConstant>(II.getStep()) && \"Induction step not a SCEV constant!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4312, __extension__
__PRETTY_FUNCTION__))
4312 "Induction step not a SCEV constant!")(static_cast <bool> (isa<SCEVConstant>(II.getStep
()) && "Induction step not a SCEV constant!") ? void (
0) : __assert_fail ("isa<SCEVConstant>(II.getStep()) && \"Induction step not a SCEV constant!\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4312, __extension__
__PRETTY_FUNCTION__))
;
4313 Type *PhiType = II.getStep()->getType();
4314
4315 // Build a pointer phi
4316 Value *ScalarStartValue = PhiR->getStartValue()->getLiveInIRValue();
4317 Type *ScStValueType = ScalarStartValue->getType();
4318 PHINode *NewPointerPhi =
4319 PHINode::Create(ScStValueType, 2, "pointer.phi", CanonicalIV);
4320 NewPointerPhi->addIncoming(ScalarStartValue, LoopVectorPreHeader);
4321
4322 // A pointer induction, performed by using a gep
4323 BasicBlock *LoopLatch = LI->getLoopFor(State.CFG.PrevBB)->getLoopLatch();
4324 Instruction *InductionLoc = LoopLatch->getTerminator();
4325 const SCEV *ScalarStep = II.getStep();
4326 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
4327 Value *ScalarStepValue =
4328 Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
4329 Value *RuntimeVF = getRuntimeVF(Builder, PhiType, VF);
4330 Value *NumUnrolledElems =
4331 Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF));
4332 Value *InductionGEP = GetElementPtrInst::Create(
4333 II.getElementType(), NewPointerPhi,
4334 Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
4335 InductionLoc);
4336 NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
4337
4338 // Create UF many actual address geps that use the pointer
4339 // phi as base and a vectorized version of the step value
4340 // (<step*0, ..., step*N>) as offset.
4341 for (unsigned Part = 0; Part < State.UF; ++Part) {
4342 Type *VecPhiType = VectorType::get(PhiType, State.VF);
4343 Value *StartOffsetScalar =
4344 Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, Part));
4345 Value *StartOffset =
4346 Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
4347 // Create a vector of consecutive numbers from zero to VF.
4348 StartOffset =
4349 Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType));
4350
4351 Value *GEP = Builder.CreateGEP(
4352 II.getElementType(), NewPointerPhi,
4353 Builder.CreateMul(
4354 StartOffset, Builder.CreateVectorSplat(State.VF, ScalarStepValue),
4355 "vector.gep"));
4356 State.set(PhiR, GEP, Part);
4357 }
4358 }
4359 }
4360}
4361
4362/// A helper function for checking whether an integer division-related
4363/// instruction may divide by zero (in which case it must be predicated if
4364/// executed conditionally in the scalar code).
4365/// TODO: It may be worthwhile to generalize and check isKnownNonZero().
4366/// Non-zero divisors that are non compile-time constants will not be
4367/// converted into multiplication, so we will still end up scalarizing
4368/// the division, but can do so w/o predication.
4369static bool mayDivideByZero(Instruction &I) {
4370 assert((I.getOpcode() == Instruction::UDiv ||(static_cast <bool> ((I.getOpcode() == Instruction::UDiv
|| I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction
::URem || I.getOpcode() == Instruction::SRem) && "Unexpected instruction"
) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4374, __extension__
__PRETTY_FUNCTION__))
4371 I.getOpcode() == Instruction::SDiv ||(static_cast <bool> ((I.getOpcode() == Instruction::UDiv
|| I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction
::URem || I.getOpcode() == Instruction::SRem) && "Unexpected instruction"
) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4374, __extension__
__PRETTY_FUNCTION__))
4372 I.getOpcode() == Instruction::URem ||(static_cast <bool> ((I.getOpcode() == Instruction::UDiv
|| I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction
::URem || I.getOpcode() == Instruction::SRem) && "Unexpected instruction"
) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4374, __extension__
__PRETTY_FUNCTION__))
4373 I.getOpcode() == Instruction::SRem) &&(static_cast <bool> ((I.getOpcode() == Instruction::UDiv
|| I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction
::URem || I.getOpcode() == Instruction::SRem) && "Unexpected instruction"
) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4374, __extension__
__PRETTY_FUNCTION__))
4374 "Unexpected instruction")(static_cast <bool> ((I.getOpcode() == Instruction::UDiv
|| I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction
::URem || I.getOpcode() == Instruction::SRem) && "Unexpected instruction"
) ? void (0) : __assert_fail ("(I.getOpcode() == Instruction::UDiv || I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::URem || I.getOpcode() == Instruction::SRem) && \"Unexpected instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4374, __extension__
__PRETTY_FUNCTION__))
;
4375 Value *Divisor = I.getOperand(1);
4376 auto *CInt = dyn_cast<ConstantInt>(Divisor);
4377 return !CInt || CInt->isZero();
4378}
4379
4380void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
4381 VPUser &ArgOperands,
4382 VPTransformState &State) {
4383 assert(!isa<DbgInfoIntrinsic>(I) &&(static_cast <bool> (!isa<DbgInfoIntrinsic>(I) &&
"DbgInfoIntrinsic should have been dropped during VPlan construction"
) ? void (0) : __assert_fail ("!isa<DbgInfoIntrinsic>(I) && \"DbgInfoIntrinsic should have been dropped during VPlan construction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4384, __extension__
__PRETTY_FUNCTION__))
4384 "DbgInfoIntrinsic should have been dropped during VPlan construction")(static_cast <bool> (!isa<DbgInfoIntrinsic>(I) &&
"DbgInfoIntrinsic should have been dropped during VPlan construction"
) ? void (0) : __assert_fail ("!isa<DbgInfoIntrinsic>(I) && \"DbgInfoIntrinsic should have been dropped during VPlan construction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4384, __extension__
__PRETTY_FUNCTION__))
;
4385 setDebugLocFromInst(&I);
4386
4387 Module *M = I.getParent()->getParent()->getParent();
4388 auto *CI = cast<CallInst>(&I);
4389
4390 SmallVector<Type *, 4> Tys;
4391 for (Value *ArgOperand : CI->args())
4392 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
4393
4394 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4395
4396 // The flag shows whether we use Intrinsic or a usual Call for vectorized
4397 // version of the instruction.
4398 // Is it beneficial to perform intrinsic call compared to lib call?
4399 bool NeedToScalarize = false;
4400 InstructionCost CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
4401 InstructionCost IntrinsicCost = ID ? Cost->getVectorIntrinsicCost(CI, VF) : 0;
4402 bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost;
4403 assert((UseVectorIntrinsic || !NeedToScalarize) &&(static_cast <bool> ((UseVectorIntrinsic || !NeedToScalarize
) && "Instruction should be scalarized elsewhere.") ?
void (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4404, __extension__
__PRETTY_FUNCTION__))
4404 "Instruction should be scalarized elsewhere.")(static_cast <bool> ((UseVectorIntrinsic || !NeedToScalarize
) && "Instruction should be scalarized elsewhere.") ?
void (0) : __assert_fail ("(UseVectorIntrinsic || !NeedToScalarize) && \"Instruction should be scalarized elsewhere.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4404, __extension__
__PRETTY_FUNCTION__))
;
4405 assert((IntrinsicCost.isValid() || CallCost.isValid()) &&(static_cast <bool> ((IntrinsicCost.isValid() || CallCost
.isValid()) && "Either the intrinsic cost or vector call cost must be valid"
) ? void (0) : __assert_fail ("(IntrinsicCost.isValid() || CallCost.isValid()) && \"Either the intrinsic cost or vector call cost must be valid\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4406, __extension__
__PRETTY_FUNCTION__))
4406 "Either the intrinsic cost or vector call cost must be valid")(static_cast <bool> ((IntrinsicCost.isValid() || CallCost
.isValid()) && "Either the intrinsic cost or vector call cost must be valid"
) ? void (0) : __assert_fail ("(IntrinsicCost.isValid() || CallCost.isValid()) && \"Either the intrinsic cost or vector call cost must be valid\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4406, __extension__
__PRETTY_FUNCTION__))
;
4407
4408 for (unsigned Part = 0; Part < UF; ++Part) {
4409 SmallVector<Type *, 2> TysForDecl = {CI->getType()};
4410 SmallVector<Value *, 4> Args;
4411 for (auto &I : enumerate(ArgOperands.operands())) {
4412 // Some intrinsics have a scalar argument - don't replace it with a
4413 // vector.
4414 Value *Arg;
4415 if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
4416 Arg = State.get(I.value(), Part);
4417 else {
4418 Arg = State.get(I.value(), VPIteration(0, 0));
4419 if (hasVectorInstrinsicOverloadedScalarOpd(ID, I.index()))
4420 TysForDecl.push_back(Arg->getType());
4421 }
4422 Args.push_back(Arg);
4423 }
4424
4425 Function *VectorF;
4426 if (UseVectorIntrinsic) {
4427 // Use vector version of the intrinsic.
4428 if (VF.isVector())
4429 TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
4430 VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
4431 assert(VectorF && "Can't retrieve vector intrinsic.")(static_cast <bool> (VectorF && "Can't retrieve vector intrinsic."
) ? void (0) : __assert_fail ("VectorF && \"Can't retrieve vector intrinsic.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4431, __extension__
__PRETTY_FUNCTION__))
;
4432 } else {
4433 // Use vector version of the function call.
4434 const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
4435#ifndef NDEBUG
4436 assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&(static_cast <bool> (VFDatabase(*CI).getVectorizedFunction
(Shape) != nullptr && "Can't create vector function."
) ? void (0) : __assert_fail ("VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && \"Can't create vector function.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4437, __extension__
__PRETTY_FUNCTION__))
4437 "Can't create vector function.")(static_cast <bool> (VFDatabase(*CI).getVectorizedFunction
(Shape) != nullptr && "Can't create vector function."
) ? void (0) : __assert_fail ("VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr && \"Can't create vector function.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4437, __extension__
__PRETTY_FUNCTION__))
;
4438#endif
4439 VectorF = VFDatabase(*CI).getVectorizedFunction(Shape);
4440 }
4441 SmallVector<OperandBundleDef, 1> OpBundles;
4442 CI->getOperandBundlesAsDefs(OpBundles);
4443 CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
4444
4445 if (isa<FPMathOperator>(V))
4446 V->copyFastMathFlags(CI);
4447
4448 State.set(Def, V, Part);
4449 addMetadata(V, &I);
4450 }
4451}
4452
4453void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
4454 // We should not collect Scalars more than once per VF. Right now, this
4455 // function is called from collectUniformsAndScalars(), which already does
4456 // this check. Collecting Scalars for VF=1 does not make any sense.
4457 assert(VF.isVector() && Scalars.find(VF) == Scalars.end() &&(static_cast <bool> (VF.isVector() && Scalars.find
(VF) == Scalars.end() && "This function should not be visited twice for the same VF"
) ? void (0) : __assert_fail ("VF.isVector() && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4458, __extension__
__PRETTY_FUNCTION__))
4458 "This function should not be visited twice for the same VF")(static_cast <bool> (VF.isVector() && Scalars.find
(VF) == Scalars.end() && "This function should not be visited twice for the same VF"
) ? void (0) : __assert_fail ("VF.isVector() && Scalars.find(VF) == Scalars.end() && \"This function should not be visited twice for the same VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4458, __extension__
__PRETTY_FUNCTION__))
;
4459
4460 // This avoids any chances of creating a REPLICATE recipe during planning
4461 // since that would result in generation of scalarized code during execution,
4462 // which is not supported for scalable vectors.
4463 if (VF.isScalable()) {
4464 Scalars[VF].insert(Uniforms[VF].begin(), Uniforms[VF].end());
4465 return;
4466 }
4467
4468 SmallSetVector<Instruction *, 8> Worklist;
4469
4470 // These sets are used to seed the analysis with pointers used by memory
4471 // accesses that will remain scalar.
4472 SmallSetVector<Instruction *, 8> ScalarPtrs;
4473 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
4474 auto *Latch = TheLoop->getLoopLatch();
4475
4476 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
4477 // The pointer operands of loads and stores will be scalar as long as the
4478 // memory access is not a gather or scatter operation. The value operand of a
4479 // store will remain scalar if the store is scalarized.
4480 auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
4481 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
4482 assert(WideningDecision != CM_Unknown &&(static_cast <bool> (WideningDecision != CM_Unknown &&
"Widening decision should be ready at this moment") ? void (
0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4483, __extension__
__PRETTY_FUNCTION__))
4483 "Widening decision should be ready at this moment")(static_cast <bool> (WideningDecision != CM_Unknown &&
"Widening decision should be ready at this moment") ? void (
0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4483, __extension__
__PRETTY_FUNCTION__))
;
4484 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
4485 if (Ptr == Store->getValueOperand())
4486 return WideningDecision == CM_Scalarize;
4487 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&(static_cast <bool> (Ptr == getLoadStorePointerOperand(
MemAccess) && "Ptr is neither a value or pointer operand"
) ? void (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4488, __extension__
__PRETTY_FUNCTION__))
4488 "Ptr is neither a value or pointer operand")(static_cast <bool> (Ptr == getLoadStorePointerOperand(
MemAccess) && "Ptr is neither a value or pointer operand"
) ? void (0) : __assert_fail ("Ptr == getLoadStorePointerOperand(MemAccess) && \"Ptr is neither a value or pointer operand\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4488, __extension__
__PRETTY_FUNCTION__))
;
4489 return WideningDecision != CM_GatherScatter;
4490 };
4491
4492 // A helper that returns true if the given value is a bitcast or
4493 // getelementptr instruction contained in the loop.
4494 auto isLoopVaryingBitCastOrGEP = [&](Value *V) {
4495 return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||
4496 isa<GetElementPtrInst>(V)) &&
4497 !TheLoop->isLoopInvariant(V);
4498 };
4499
4500 // A helper that evaluates a memory access's use of a pointer. If the use will
4501 // be a scalar use and the pointer is only used by memory accesses, we place
4502 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
4503 // PossibleNonScalarPtrs.
4504 auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
4505 // We only care about bitcast and getelementptr instructions contained in
4506 // the loop.
4507 if (!isLoopVaryingBitCastOrGEP(Ptr))
4508 return;
4509
4510 // If the pointer has already been identified as scalar (e.g., if it was
4511 // also identified as uniform), there's nothing to do.
4512 auto *I = cast<Instruction>(Ptr);
4513 if (Worklist.count(I))
4514 return;
4515
4516 // If the use of the pointer will be a scalar use, and all users of the
4517 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
4518 // place the pointer in PossibleNonScalarPtrs.
4519 if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
4520 return isa<LoadInst>(U) || isa<StoreInst>(U);
4521 }))
4522 ScalarPtrs.insert(I);
4523 else
4524 PossibleNonScalarPtrs.insert(I);
4525 };
4526
4527 // We seed the scalars analysis with three classes of instructions: (1)
4528 // instructions marked uniform-after-vectorization and (2) bitcast,
4529 // getelementptr and (pointer) phi instructions used by memory accesses
4530 // requiring a scalar use.
4531 //
4532 // (1) Add to the worklist all instructions that have been identified as
4533 // uniform-after-vectorization.
4534 Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());
4535
4536 // (2) Add to the worklist all bitcast and getelementptr instructions used by
4537 // memory accesses requiring a scalar use. The pointer operands of loads and
4538 // stores will be scalar as long as the memory accesses is not a gather or
4539 // scatter operation. The value operand of a store will remain scalar if the
4540 // store is scalarized.
4541 for (auto *BB : TheLoop->blocks())
4542 for (auto &I : *BB) {
4543 if (auto *Load = dyn_cast<LoadInst>(&I)) {
4544 evaluatePtrUse(Load, Load->getPointerOperand());
4545 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
4546 evaluatePtrUse(Store, Store->getPointerOperand());
4547 evaluatePtrUse(Store, Store->getValueOperand());
4548 }
4549 }
4550 for (auto *I : ScalarPtrs)
4551 if (!PossibleNonScalarPtrs.count(I)) {
4552 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *I << "\n"; } } while (false)
;
4553 Worklist.insert(I);
4554 }
4555
4556 // Insert the forced scalars.
4557 // FIXME: Currently widenPHIInstruction() often creates a dead vector
4558 // induction variable when the PHI user is scalarized.
4559 auto ForcedScalar = ForcedScalars.find(VF);
4560 if (ForcedScalar != ForcedScalars.end())
4561 for (auto *I : ForcedScalar->second)
4562 Worklist.insert(I);
4563
4564 // Expand the worklist by looking through any bitcasts and getelementptr
4565 // instructions we've already identified as scalar. This is similar to the
4566 // expansion step in collectLoopUniforms(); however, here we're only
4567 // expanding to include additional bitcasts and getelementptr instructions.
4568 unsigned Idx = 0;
4569 while (Idx != Worklist.size()) {
4570 Instruction *Dst = Worklist[Idx++];
4571 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
4572 continue;
4573 auto *Src = cast<Instruction>(Dst->getOperand(0));
4574 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
4575 auto *J = cast<Instruction>(U);
4576 return !TheLoop->contains(J) || Worklist.count(J) ||
4577 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
4578 isScalarUse(J, Src));
4579 })) {
4580 Worklist.insert(Src);
4581 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Src << "\n"; } } while (false)
;
4582 }
4583 }
4584
4585 // An induction variable will remain scalar if all users of the induction
4586 // variable and induction variable update remain scalar.
4587 for (auto &Induction : Legal->getInductionVars()) {
4588 auto *Ind = Induction.first;
4589 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4590
4591 // If tail-folding is applied, the primary induction variable will be used
4592 // to feed a vector compare.
4593 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
4594 continue;
4595
4596 // Returns true if \p Indvar is a pointer induction that is used directly by
4597 // load/store instruction \p I.
4598 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
4599 Instruction *I) {
4600 return Induction.second.getKind() ==
4601 InductionDescriptor::IK_PtrInduction &&
4602 (isa<LoadInst>(I) || isa<StoreInst>(I)) &&
4603 Indvar == getLoadStorePointerOperand(I) && isScalarUse(I, Indvar);
4604 };
4605
4606 // Determine if all users of the induction variable are scalar after
4607 // vectorization.
4608 auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
4609 auto *I = cast<Instruction>(U);
4610 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
4611 IsDirectLoadStoreFromPtrIndvar(Ind, I);
4612 });
4613 if (!ScalarInd)
4614 continue;
4615
4616 // Determine if all users of the induction variable update instruction are
4617 // scalar after vectorization.
4618 auto ScalarIndUpdate =
4619 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
4620 auto *I = cast<Instruction>(U);
4621 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
4622 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
4623 });
4624 if (!ScalarIndUpdate)
4625 continue;
4626
4627 // The induction variable and its update instruction will remain scalar.
4628 Worklist.insert(Ind);
4629 Worklist.insert(IndUpdate);
4630 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *Ind << "\n"; } } while (false)
;
4631 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdatedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
4632 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found scalar instruction: "
<< *IndUpdate << "\n"; } } while (false)
;
4633 }
4634
4635 Scalars[VF].insert(Worklist.begin(), Worklist.end());
4636}
4637
4638bool LoopVectorizationCostModel::isScalarWithPredication(
4639 Instruction *I, ElementCount VF) const {
4640 if (!blockNeedsPredicationForAnyReason(I->getParent()))
4641 return false;
4642 switch(I->getOpcode()) {
4643 default:
4644 break;
4645 case Instruction::Load:
4646 case Instruction::Store: {
4647 if (!Legal->isMaskRequired(I))
4648 return false;
4649 auto *Ptr = getLoadStorePointerOperand(I);
4650 auto *Ty = getLoadStoreType(I);
4651 Type *VTy = Ty;
4652 if (VF.isVector())
4653 VTy = VectorType::get(Ty, VF);
4654 const Align Alignment = getLoadStoreAlignment(I);
4655 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
4656 TTI.isLegalMaskedGather(VTy, Alignment))
4657 : !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
4658 TTI.isLegalMaskedScatter(VTy, Alignment));
4659 }
4660 case Instruction::UDiv:
4661 case Instruction::SDiv:
4662 case Instruction::SRem:
4663 case Instruction::URem:
4664 return mayDivideByZero(*I);
4665 }
4666 return false;
4667}
4668
4669bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
4670 Instruction *I, ElementCount VF) {
4671 assert(isAccessInterleaved(I) && "Expecting interleaved access.")(static_cast <bool> (isAccessInterleaved(I) && "Expecting interleaved access."
) ? void (0) : __assert_fail ("isAccessInterleaved(I) && \"Expecting interleaved access.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4671, __extension__
__PRETTY_FUNCTION__))
;
4672 assert(getWideningDecision(I, VF) == CM_Unknown &&(static_cast <bool> (getWideningDecision(I, VF) == CM_Unknown
&& "Decision should not be set yet.") ? void (0) : __assert_fail
("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4673, __extension__
__PRETTY_FUNCTION__))
4673 "Decision should not be set yet.")(static_cast <bool> (getWideningDecision(I, VF) == CM_Unknown
&& "Decision should not be set yet.") ? void (0) : __assert_fail
("getWideningDecision(I, VF) == CM_Unknown && \"Decision should not be set yet.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4673, __extension__
__PRETTY_FUNCTION__))
;
4674 auto *Group = getInterleavedAccessGroup(I);
4675 assert(Group && "Must have a group.")(static_cast <bool> (Group && "Must have a group."
) ? void (0) : __assert_fail ("Group && \"Must have a group.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4675, __extension__
__PRETTY_FUNCTION__))
;
4676
4677 // If the instruction's allocated size doesn't equal it's type size, it
4678 // requires padding and will be scalarized.
4679 auto &DL = I->getModule()->getDataLayout();
4680 auto *ScalarTy = getLoadStoreType(I);
4681 if (hasIrregularType(ScalarTy, DL))
4682 return false;
4683
4684 // Check if masking is required.
4685 // A Group may need masking for one of two reasons: it resides in a block that
4686 // needs predication, or it was decided to use masking to deal with gaps
4687 // (either a gap at the end of a load-access that may result in a speculative
4688 // load, or any gaps in a store-access).
4689 bool PredicatedAccessRequiresMasking =
4690 blockNeedsPredicationForAnyReason(I->getParent()) &&
4691 Legal->isMaskRequired(I);
4692 bool LoadAccessWithGapsRequiresEpilogMasking =
4693 isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
4694 !isScalarEpilogueAllowed();
4695 bool StoreAccessWithGapsRequiresMasking =
4696 isa<StoreInst>(I) && (Group->getNumMembers() < Group->getFactor());
4697 if (!PredicatedAccessRequiresMasking &&
4698 !LoadAccessWithGapsRequiresEpilogMasking &&
4699 !StoreAccessWithGapsRequiresMasking)
4700 return true;
4701
4702 // If masked interleaving is required, we expect that the user/target had
4703 // enabled it, because otherwise it either wouldn't have been created or
4704 // it should have been invalidated by the CostModel.
4705 assert(useMaskedInterleavedAccesses(TTI) &&(static_cast <bool> (useMaskedInterleavedAccesses(TTI) &&
"Masked interleave-groups for predicated accesses are not enabled."
) ? void (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4706, __extension__
__PRETTY_FUNCTION__))
4706 "Masked interleave-groups for predicated accesses are not enabled.")(static_cast <bool> (useMaskedInterleavedAccesses(TTI) &&
"Masked interleave-groups for predicated accesses are not enabled."
) ? void (0) : __assert_fail ("useMaskedInterleavedAccesses(TTI) && \"Masked interleave-groups for predicated accesses are not enabled.\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4706, __extension__
__PRETTY_FUNCTION__))
;
4707
4708 if (Group->isReverse())
4709 return false;
4710
4711 auto *Ty = getLoadStoreType(I);
4712 const Align Alignment = getLoadStoreAlignment(I);
4713 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
4714 : TTI.isLegalMaskedStore(Ty, Alignment);
4715}
4716
4717bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
4718 Instruction *I, ElementCount VF) {
4719 // Get and ensure we have a valid memory instruction.
4720 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction")(static_cast <bool> ((isa<LoadInst, StoreInst>(I)
) && "Invalid memory instruction") ? void (0) : __assert_fail
("(isa<LoadInst, StoreInst>(I)) && \"Invalid memory instruction\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4720, __extension__
__PRETTY_FUNCTION__))
;
4721
4722 auto *Ptr = getLoadStorePointerOperand(I);
4723 auto *ScalarTy = getLoadStoreType(I);
4724
4725 // In order to be widened, the pointer should be consecutive, first of all.
4726 if (!Legal->isConsecutivePtr(ScalarTy, Ptr))
4727 return false;
4728
4729 // If the instruction is a store located in a predicated block, it will be
4730 // scalarized.
4731 if (isScalarWithPredication(I, VF))
4732 return false;
4733
4734 // If the instruction's allocated size doesn't equal it's type size, it
4735 // requires padding and will be scalarized.
4736 auto &DL = I->getModule()->getDataLayout();
4737 if (hasIrregularType(ScalarTy, DL))
4738 return false;
4739
4740 return true;
4741}
4742
4743void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
4744 // We should not collect Uniforms more than once per VF. Right now,
4745 // this function is called from collectUniformsAndScalars(), which
4746 // already does this check. Collecting Uniforms for VF=1 does not make any
4747 // sense.
4748
4749 assert(VF.isVector() && Uniforms.find(VF) == Uniforms.end() &&(static_cast <bool> (VF.isVector() && Uniforms.
find(VF) == Uniforms.end() && "This function should not be visited twice for the same VF"
) ? void (0) : __assert_fail ("VF.isVector() && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4750, __extension__
__PRETTY_FUNCTION__))
4750 "This function should not be visited twice for the same VF")(static_cast <bool> (VF.isVector() && Uniforms.
find(VF) == Uniforms.end() && "This function should not be visited twice for the same VF"
) ? void (0) : __assert_fail ("VF.isVector() && Uniforms.find(VF) == Uniforms.end() && \"This function should not be visited twice for the same VF\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4750, __extension__
__PRETTY_FUNCTION__))
;
4751
4752 // Visit the list of Uniforms. If we'll not find any uniform value, we'll
4753 // not analyze again. Uniforms.count(VF) will return 1.
4754 Uniforms[VF].clear();
4755
4756 // We now know that the loop is vectorizable!
4757 // Collect instructions inside the loop that will remain uniform after
4758 // vectorization.
4759
4760 // Global values, params and instructions outside of current loop are out of
4761 // scope.
4762 auto isOutOfScope = [&](Value *V) -> bool {
4763 Instruction *I = dyn_cast<Instruction>(V);
4764 return (!I || !TheLoop->contains(I));
4765 };
4766
4767 // Worklist containing uniform instructions demanding lane 0.
4768 SetVector<Instruction *> Worklist;
4769 BasicBlock *Latch = TheLoop->getLoopLatch();
4770
4771 // Add uniform instructions demanding lane 0 to the worklist. Instructions
4772 // that are scalar with predication must not be considered uniform after
4773 // vectorization, because that would create an erroneous replicating region
4774 // where only a single instance out of VF should be formed.
4775 // TODO: optimize such seldom cases if found important, see PR40816.
4776 auto addToWorklistIfAllowed = [&](Instruction *I) -> void {
4777 if (isOutOfScope(I)) {
4778 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform due to scope: "
<< *I << "\n"; } } while (false)
4779 << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform due to scope: "
<< *I << "\n"; } } while (false)
;
4780 return;
4781 }
4782 if (isScalarWithPredication(I, VF)) {
4783 LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform being ScalarWithPredication: "
<< *I << "\n"; } } while (false)
4784 << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found not uniform being ScalarWithPredication: "
<< *I << "\n"; } } while (false)
;
4785 return;
4786 }
4787 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found uniform instruction: "
<< *I << "\n"; } } while (false)
;
4788 Worklist.insert(I);
4789 };
4790
4791 // Start with the conditional branch. If the branch condition is an
4792 // instruction contained in the loop that is only used by the branch, it is
4793 // uniform.
4794 auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));
4795 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
4796 addToWorklistIfAllowed(Cmp);
4797
4798 auto isUniformDecision = [&](Instruction *I, ElementCount VF) {
4799 InstWidening WideningDecision = getWideningDecision(I, VF);
4800 assert(WideningDecision != CM_Unknown &&(static_cast <bool> (WideningDecision != CM_Unknown &&
"Widening decision should be ready at this moment") ? void (
0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4801, __extension__
__PRETTY_FUNCTION__))
4801 "Widening decision should be ready at this moment")(static_cast <bool> (WideningDecision != CM_Unknown &&
"Widening decision should be ready at this moment") ? void (
0) : __assert_fail ("WideningDecision != CM_Unknown && \"Widening decision should be ready at this moment\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4801, __extension__
__PRETTY_FUNCTION__))
;
4802
4803 // A uniform memory op is itself uniform. We exclude uniform stores
4804 // here as they demand the last lane, not the first one.
4805 if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) {
4806 assert(WideningDecision == CM_Scalarize)(static_cast <bool> (WideningDecision == CM_Scalarize) ?
void (0) : __assert_fail ("WideningDecision == CM_Scalarize"
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4806, __extension__
__PRETTY_FUNCTION__))
;
4807 return true;
4808 }
4809
4810 return (WideningDecision == CM_Widen ||
4811 WideningDecision == CM_Widen_Reverse ||
4812 WideningDecision == CM_Interleave);
4813 };
4814
4815
4816 // Returns true if Ptr is the pointer operand of a memory access instruction
4817 // I, and I is known to not require scalarization.
4818 auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
4819 return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
4820 };
4821
4822 // Holds a list of values which are known to have at least one uniform use.
4823 // Note that there may be other uses which aren't uniform. A "uniform use"
4824 // here is something which only demands lane 0 of the unrolled iterations;
4825 // it does not imply that all lanes produce the same value (e.g. this is not
4826 // the usual meaning of uniform)
4827 SetVector<Value *> HasUniformUse;
4828
4829 // Scan the loop for instructions which are either a) known to have only
4830 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
4831 for (auto *BB : TheLoop->blocks())
4832 for (auto &I : *BB) {
4833 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I)) {
4834 switch (II->getIntrinsicID()) {
4835 case Intrinsic::sideeffect:
4836 case Intrinsic::experimental_noalias_scope_decl:
4837 case Intrinsic::assume:
4838 case Intrinsic::lifetime_start:
4839 case Intrinsic::lifetime_end:
4840 if (TheLoop->hasLoopInvariantOperands(&I))
4841 addToWorklistIfAllowed(&I);
4842 break;
4843 default:
4844 break;
4845 }
4846 }
4847
4848 // ExtractValue instructions must be uniform, because the operands are
4849 // known to be loop-invariant.
4850 if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
4851 assert(isOutOfScope(EVI->getAggregateOperand()) &&(static_cast <bool> (isOutOfScope(EVI->getAggregateOperand
()) && "Expected aggregate value to be loop invariant"
) ? void (0) : __assert_fail ("isOutOfScope(EVI->getAggregateOperand()) && \"Expected aggregate value to be loop invariant\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4852, __extension__
__PRETTY_FUNCTION__))
4852 "Expected aggregate value to be loop invariant")(static_cast <bool> (isOutOfScope(EVI->getAggregateOperand
()) && "Expected aggregate value to be loop invariant"
) ? void (0) : __assert_fail ("isOutOfScope(EVI->getAggregateOperand()) && \"Expected aggregate value to be loop invariant\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4852, __extension__
__PRETTY_FUNCTION__))
;
4853 addToWorklistIfAllowed(EVI);
4854 continue;
4855 }
4856
4857 // If there's no pointer operand, there's nothing to do.
4858 auto *Ptr = getLoadStorePointerOperand(&I);
4859 if (!Ptr)
4860 continue;
4861
4862 // A uniform memory op is itself uniform. We exclude uniform stores
4863 // here as they demand the last lane, not the first one.
4864 if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))
4865 addToWorklistIfAllowed(&I);
4866
4867 if (isUniformDecision(&I, VF)) {
4868 assert(isVectorizedMemAccessUse(&I, Ptr) && "consistency check")(static_cast <bool> (isVectorizedMemAccessUse(&I, Ptr
) && "consistency check") ? void (0) : __assert_fail (
"isVectorizedMemAccessUse(&I, Ptr) && \"consistency check\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 4868, __extension__
__PRETTY_FUNCTION__))
;
4869 HasUniformUse.insert(Ptr);
4870 }
4871 }
4872
4873 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
4874 // demanding) users. Since loops are assumed to be in LCSSA form, this
4875 // disallows uses outside the loop as well.
4876 for (auto *V : HasUniformUse) {
4877 if (isOutOfScope(V))
4878 continue;
4879 auto *I = cast<Instruction>(V);
4880 auto UsersAreMemAccesses =
4881 llvm::all_of(I->users(), [&](User *U) -> bool {
4882 return isVectorizedMemAccessUse(cast<Instruction>(U), V);
4883 });
4884 if (UsersAreMemAccesses)
4885 addToWorklistIfAllowed(I);
4886 }
4887
4888 // Expand Worklist in topological order: whenever a new instruction
4889 // is added , its users should be already inside Worklist. It ensures
4890 // a uniform instruction will only be used by uniform instructions.
4891 unsigned idx = 0;
4892 while (idx != Worklist.size()) {
4893 Instruction *I = Worklist[idx++];
4894
4895 for (auto OV : I->operand_values()) {
4896 // isOutOfScope operands cannot be uniform instructions.
4897 if (isOutOfScope(OV))
4898 continue;
4899 // First order recurrence Phi's should typically be considered
4900 // non-uniform.
4901 auto *OP = dyn_cast<PHINode>(OV);
4902 if (OP && Legal->isFirstOrderRecurrence(OP))
4903 continue;
4904 // If all the users of the operand are uniform, then add the
4905 // operand into the uniform worklist.
4906 auto *OI = cast<Instruction>(OV);
4907 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
4908 auto *J = cast<Instruction>(U);
4909 return Worklist.count(J) || isVectorizedMemAccessUse(J, OI);
4910 }))
4911 addToWorklistIfAllowed(OI);
4912 }
4913 }
4914
4915 // For an instruction to be added into Worklist above, all its users inside
4916 // the loop should also be in Worklist. However, this condition cannot be
4917 // true for phi nodes that form a cyclic dependence. We must process phi
4918 // nodes separately. An induction variable will remain uniform if all users
4919 // of the induction variable and induction variable update remain uniform.
4920 // The code below handles both pointer and non-pointer induction variables.
4921 for (auto &Induction : Legal->getInductionVars()) {
4922 auto *Ind = Induction.first;
4923 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
4924
4925 // Determine if all users of the induction variable are uniform after
4926 // vectorization.
4927 auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
4928 auto *I = cast<Instruction>(U);
4929 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
4930 isVectorizedMemAccessUse(I, Ind);
4931 });
4932 if (!UniformInd)
4933 continue;
4934
4935 // Determine if all users of the induction variable update instruction are
4936 // uniform after vectorization.
4937 auto UniformIndUpdate =
4938 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
4939 auto *I = cast<Instruction>(U);
4940 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
4941 isVectorizedMemAccessUse(I, IndUpdate);
4942 });
4943 if (!UniformIndUpdate)
4944 continue;
4945
4946 // The induction variable and its update instruction will remain uniform.
4947 addToWorklistIfAllowed(Ind);
4948 addToWorklistIfAllowed(IndUpdate);
4949 }
4950
4951 Uniforms[VF].insert(Worklist.begin(), Worklist.end());
4952}
4953
4954bool LoopVectorizationCostModel::runtimeChecksRequired() {
4955 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Performing code size checks.\n"
; } } while (false)
;
4956
4957 if (Legal->getRuntimePointerChecking()->Need) {
4958 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
4959 "runtime pointer checks needed. Enable vectorization of this "
4960 "loop with '#pragma clang loop vectorize(enable)' when "
4961 "compiling with -Os/-Oz",
4962 "CantVersionLoopWithOptForSize", ORE, TheLoop);
4963 return true;
4964 }
4965
4966 if (!PSE.getPredicate().isAlwaysTrue()) {
4967 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
4968 "runtime SCEV checks needed. Enable vectorization of this "
4969 "loop with '#pragma clang loop vectorize(enable)' when "
4970 "compiling with -Os/-Oz",
4971 "CantVersionLoopWithOptForSize", ORE, TheLoop);
4972 return true;
4973 }
4974
4975 // FIXME: Avoid specializing for stride==1 instead of bailing out.
4976 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
4977 reportVectorizationFailure("Runtime stride check for small trip count",
4978 "runtime stride == 1 checks needed. Enable vectorization of "
4979 "this loop without such check by compiling with -Os/-Oz",
4980 "CantVersionLoopWithOptForSize", ORE, TheLoop);
4981 return true;
4982 }
4983
4984 return false;
4985}
4986
4987ElementCount
4988LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
4989 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors)
4990 return ElementCount::getScalable(0);
4991
4992 if (Hints->isScalableVectorizationDisabled()) {
4993 reportVectorizationInfo("Scalable vectorization is explicitly disabled",
4994 "ScalableVectorizationDisabled", ORE, TheLoop);
4995 return ElementCount::getScalable(0);
4996 }
4997
4998 LLVM_DEBUG(dbgs() << "LV: Scalable vectorization is available\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Scalable vectorization is available\n"
; } } while (false)
;
4999
5000 auto MaxScalableVF = ElementCount::getScalable(
5001 std::numeric_limits<ElementCount::ScalarTy>::max());
5002
5003 // Test that the loop-vectorizer can legalize all operations for this MaxVF.
5004 // FIXME: While for scalable vectors this is currently sufficient, this should
5005 // be replaced by a more detailed mechanism that filters out specific VFs,
5006 // instead of invalidating vectorization for a whole set of VFs based on the
5007 // MaxVF.
5008
5009 // Disable scalable vectorization if the loop contains unsupported reductions.
5010 if (!canVectorizeReductions(MaxScalableVF)) {
5011 reportVectorizationInfo(
5012 "Scalable vectorization not supported for the reduction "
5013 "operations found in this loop.",
5014 "ScalableVFUnfeasible", ORE, TheLoop);
5015 return ElementCount::getScalable(0);
5016 }
5017
5018 // Disable scalable vectorization if the loop contains any instructions
5019 // with element types not supported for scalable vectors.
5020 if (any_of(ElementTypesInLoop, [&](Type *Ty) {
5021 return !Ty->isVoidTy() &&
5022 !this->TTI.isElementTypeLegalForScalableVector(Ty);
5023 })) {
5024 reportVectorizationInfo("Scalable vectorization is not supported "
5025 "for all element types found in this loop.",
5026 "ScalableVFUnfeasible", ORE, TheLoop);
5027 return ElementCount::getScalable(0);
5028 }
5029
5030 if (Legal->isSafeForAnyVectorWidth())
5031 return MaxScalableVF;
5032
5033 // Limit MaxScalableVF by the maximum safe dependence distance.
5034 Optional<unsigned> MaxVScale = TTI.getMaxVScale();
5035 if (!MaxVScale && TheFunction->hasFnAttribute(Attribute::VScaleRange))
5036 MaxVScale =
5037 TheFunction->getFnAttribute(Attribute::VScaleRange).getVScaleRangeMax();
5038 MaxScalableVF = ElementCount::getScalable(
5039 MaxVScale ? (MaxSafeElements / MaxVScale.getValue()) : 0);
5040 if (!MaxScalableVF)
5041 reportVectorizationInfo(
5042 "Max legal vector width too small, scalable vectorization "
5043 "unfeasible.",
5044 "ScalableVFUnfeasible", ORE, TheLoop);
5045
5046 return MaxScalableVF;
5047}
5048
5049FixedScalableVFPair LoopVectorizationCostModel::computeFeasibleMaxVF(
5050 unsigned ConstTripCount, ElementCount UserVF, bool FoldTailByMasking) {
5051 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
5052 unsigned SmallestType, WidestType;
5053 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
5054
5055 // Get the maximum safe dependence distance in bits computed by LAA.
5056 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
5057 // the memory accesses that is most restrictive (involved in the smallest
5058 // dependence distance).
5059 unsigned MaxSafeElements =
5060 PowerOf2Floor(Legal->getMaxSafeVectorWidthInBits() / WidestType);
5061
5062 auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElements);
5063 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElements);
5064
5065 LLVM_DEBUG(dbgs() << "LV: The max safe fixed VF is: " << MaxSafeFixedVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The max safe fixed VF is: "
<< MaxSafeFixedVF << ".\n"; } } while (false)
5066 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The max safe fixed VF is: "
<< MaxSafeFixedVF << ".\n"; } } while (false)
;
5067 LLVM_DEBUG(dbgs() << "LV: The max safe scalable VF is: " << MaxSafeScalableVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The max safe scalable VF is: "
<< MaxSafeScalableVF << ".\n"; } } while (false)
5068 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The max safe scalable VF is: "
<< MaxSafeScalableVF << ".\n"; } } while (false)
;
5069
5070 // First analyze the UserVF, fall back if the UserVF should be ignored.
5071 if (UserVF) {
5072 auto MaxSafeUserVF =
5073 UserVF.isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
5074
5075 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
5076 // If `VF=vscale x N` is safe, then so is `VF=N`
5077 if (UserVF.isScalable())
5078 return FixedScalableVFPair(
5079 ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
5080 else
5081 return UserVF;
5082 }
5083
5084 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF))(static_cast <bool> (ElementCount::isKnownGT(UserVF, MaxSafeUserVF
)) ? void (0) : __assert_fail ("ElementCount::isKnownGT(UserVF, MaxSafeUserVF)"
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5084, __extension__
__PRETTY_FUNCTION__))
;
5085
5086 // Only clamp if the UserVF is not scalable. If the UserVF is scalable, it
5087 // is better to ignore the hint and let the compiler choose a suitable VF.
5088 if (!UserVF.isScalable()) {
5089 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is unsafe, clamping to max safe VF=" <<
MaxSafeFixedVF << ".\n"; } } while (false)
5090 << " is unsafe, clamping to max safe VF="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is unsafe, clamping to max safe VF=" <<
MaxSafeFixedVF << ".\n"; } } while (false)
5091 << MaxSafeFixedVF << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is unsafe, clamping to max safe VF=" <<
MaxSafeFixedVF << ".\n"; } } while (false)
;
5092 ORE->emit([&]() {
5093 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
5094 TheLoop->getStartLoc(),
5095 TheLoop->getHeader())
5096 << "User-specified vectorization factor "
5097 << ore::NV("UserVectorizationFactor", UserVF)
5098 << " is unsafe, clamping to maximum safe vectorization factor "
5099 << ore::NV("VectorizationFactor", MaxSafeFixedVF);
5100 });
5101 return MaxSafeFixedVF;
5102 }
5103
5104 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors) {
5105 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is ignored because scalable vectors are not "
"available.\n"; } } while (false)
5106 << " is ignored because scalable vectors are not "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is ignored because scalable vectors are not "
"available.\n"; } } while (false)
5107 "available.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is ignored because scalable vectors are not "
"available.\n"; } } while (false)
;
5108 ORE->emit([&]() {
5109 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
5110 TheLoop->getStartLoc(),
5111 TheLoop->getHeader())
5112 << "User-specified vectorization factor "
5113 << ore::NV("UserVectorizationFactor", UserVF)
5114 << " is ignored because the target does not support scalable "
5115 "vectors. The compiler will pick a more suitable value.";
5116 });
5117 } else {
5118 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is unsafe. Ignoring scalable UserVF.\n"; }
} while (false)
5119 << " is unsafe. Ignoring scalable UserVF.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: User VF=" <<
UserVF << " is unsafe. Ignoring scalable UserVF.\n"; }
} while (false)
;
5120 ORE->emit([&]() {
5121 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
5122 TheLoop->getStartLoc(),
5123 TheLoop->getHeader())
5124 << "User-specified vectorization factor "
5125 << ore::NV("UserVectorizationFactor", UserVF)
5126 << " is unsafe. Ignoring the hint to let the compiler pick a "
5127 "more suitable value.";
5128 });
5129 }
5130 }
5131
5132 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestTypedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
<< SmallestType << " / " << WidestType <<
" bits.\n"; } } while (false)
5133 << " / " << WidestType << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Smallest and Widest types: "
<< SmallestType << " / " << WidestType <<
" bits.\n"; } } while (false)
;
5134
5135 FixedScalableVFPair Result(ElementCount::getFixed(1),
5136 ElementCount::getScalable(0));
5137 if (auto MaxVF =
5138 getMaximizedVFForTarget(ConstTripCount, SmallestType, WidestType,
5139 MaxSafeFixedVF, FoldTailByMasking))
5140 Result.FixedVF = MaxVF;
5141
5142 if (auto MaxVF =
5143 getMaximizedVFForTarget(ConstTripCount, SmallestType, WidestType,
5144 MaxSafeScalableVF, FoldTailByMasking))
5145 if (MaxVF.isScalable()) {
5146 Result.ScalableVF = MaxVF;
5147 LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found feasible scalable VF = "
<< MaxVF << "\n"; } } while (false)
5148 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found feasible scalable VF = "
<< MaxVF << "\n"; } } while (false)
;
5149 }
5150
5151 return Result;
5152}
5153
5154FixedScalableVFPair
5155LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
5156 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
5157 // TODO: It may by useful to do since it's still likely to be dynamically
5158 // uniform if the target can skip.
5159 reportVectorizationFailure(
5160 "Not inserting runtime ptr check for divergent target",
5161 "runtime pointer checks needed. Not enabled for divergent target",
5162 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
5163 return FixedScalableVFPair::getNone();
5164 }
5165
5166 unsigned TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
5167 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Found trip count: "
<< TC << '\n'; } } while (false)
;
5168 if (TC == 1) {
5169 reportVectorizationFailure("Single iteration (non) loop",
5170 "loop trip count is one, irrelevant for vectorization",
5171 "SingleIterationLoop", ORE, TheLoop);
5172 return FixedScalableVFPair::getNone();
5173 }
5174
5175 switch (ScalarEpilogueStatus) {
5176 case CM_ScalarEpilogueAllowed:
5177 return computeFeasibleMaxVF(TC, UserVF, false);
5178 case CM_ScalarEpilogueNotAllowedUsePredicate:
5179 LLVM_FALLTHROUGH[[gnu::fallthrough]];
5180 case CM_ScalarEpilogueNotNeededUsePredicate:
5181 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
5182 dbgs() << "LV: vector predicate hint/switch found.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
5183 << "LV: Not allowing scalar epilogue, creating predicated "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
5184 << "vector loop.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: vector predicate hint/switch found.\n"
<< "LV: Not allowing scalar epilogue, creating predicated "
<< "vector loop.\n"; } } while (false)
;
5185 break;
5186 case CM_ScalarEpilogueNotAllowedLowTripLoop:
5187 // fallthrough as a special case of OptForSize
5188 case CM_ScalarEpilogueNotAllowedOptSize:
5189 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
5190 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false)
5191 dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n"
; } } while (false)
;
5192 else
5193 LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to low trip "
<< "count.\n"; } } while (false)
5194 << "count.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Not allowing scalar epilogue due to low trip "
<< "count.\n"; } } while (false)
;
5195
5196 // Bail if runtime checks are required, which are not good when optimising
5197 // for size.
5198 if (runtimeChecksRequired())
5199 return FixedScalableVFPair::getNone();
5200
5201 break;
5202 }
5203
5204 // The only loops we can vectorize without a scalar epilogue, are loops with
5205 // a bottom-test and a single exiting block. We'd have to handle the fact
5206 // that not every instruction executes on the last iteration. This will
5207 // require a lane mask which varies through the vector loop body. (TODO)
5208 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
5209 // If there was a tail-folding hint/switch, but we can't fold the tail by
5210 // masking, fallback to a vectorization with a scalar epilogue.
5211 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
5212 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n"; } } while (false)
5213 "scalar epilogue instead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n"; } } while (false)
;
5214 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
5215 return computeFeasibleMaxVF(TC, UserVF, false);
5216 }
5217 return FixedScalableVFPair::getNone();
5218 }
5219
5220 // Now try the tail folding
5221
5222 // Invalidate interleave groups that require an epilogue if we can't mask
5223 // the interleave-group.
5224 if (!useMaskedInterleavedAccesses(TTI)) {
5225 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&(static_cast <bool> (WideningDecisions.empty() &&
Uniforms.empty() && Scalars.empty() && "No decisions should have been taken at this point"
) ? void (0) : __assert_fail ("WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && \"No decisions should have been taken at this point\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5226, __extension__
__PRETTY_FUNCTION__))
5226 "No decisions should have been taken at this point")(static_cast <bool> (WideningDecisions.empty() &&
Uniforms.empty() && Scalars.empty() && "No decisions should have been taken at this point"
) ? void (0) : __assert_fail ("WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() && \"No decisions should have been taken at this point\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5226, __extension__
__PRETTY_FUNCTION__))
;
5227 // Note: There is no need to invalidate any cost modeling decisions here, as
5228 // non where taken so far.
5229 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
5230 }
5231
5232 FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(TC, UserVF, true);
5233 // Avoid tail folding if the trip count is known to be a multiple of any VF
5234 // we chose.
5235 // FIXME: The condition below pessimises the case for fixed-width vectors,
5236 // when scalable VFs are also candidates for vectorization.
5237 if (MaxFactors.FixedVF.isVector() && !MaxFactors.ScalableVF) {
5238 ElementCount MaxFixedVF = MaxFactors.FixedVF;
5239 assert((UserVF.isNonZero() || isPowerOf2_32(MaxFixedVF.getFixedValue())) &&(static_cast <bool> ((UserVF.isNonZero() || isPowerOf2_32
(MaxFixedVF.getFixedValue())) && "MaxFixedVF must be a power of 2"
) ? void (0) : __assert_fail ("(UserVF.isNonZero() || isPowerOf2_32(MaxFixedVF.getFixedValue())) && \"MaxFixedVF must be a power of 2\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5240, __extension__
__PRETTY_FUNCTION__))
5240 "MaxFixedVF must be a power of 2")(static_cast <bool> ((UserVF.isNonZero() || isPowerOf2_32
(MaxFixedVF.getFixedValue())) && "MaxFixedVF must be a power of 2"
) ? void (0) : __assert_fail ("(UserVF.isNonZero() || isPowerOf2_32(MaxFixedVF.getFixedValue())) && \"MaxFixedVF must be a power of 2\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5240, __extension__
__PRETTY_FUNCTION__))
;
5241 unsigned MaxVFtimesIC = UserIC ? MaxFixedVF.getFixedValue() * UserIC
5242 : MaxFixedVF.getFixedValue();
5243 ScalarEvolution *SE = PSE.getSE();
5244 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
5245 const SCEV *ExitCount = SE->getAddExpr(
5246 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
5247 const SCEV *Rem = SE->getURemExpr(
5248 SE->applyLoopGuards(ExitCount, TheLoop),
5249 SE->getConstant(BackedgeTakenCount->getType(), MaxVFtimesIC));
5250 if (Rem->isZero()) {
5251 // Accept MaxFixedVF if we do not have a tail.
5252 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: No tail will remain for any chosen VF.\n"
; } } while (false)
;
5253 return MaxFactors;
5254 }
5255 }
5256
5257 // For scalable vectors don't use tail folding for low trip counts or
5258 // optimizing for code size. We only permit this if the user has explicitly
5259 // requested it.
5260 if (ScalarEpilogueStatus != CM_ScalarEpilogueNotNeededUsePredicate &&
5261 ScalarEpilogueStatus != CM_ScalarEpilogueNotAllowedUsePredicate &&
5262 MaxFactors.ScalableVF.isVector())
5263 MaxFactors.ScalableVF = ElementCount::getScalable(0);
5264
5265 // If we don't know the precise trip count, or if the trip count that we
5266 // found modulo the vectorization factor is not zero, try to fold the tail
5267 // by masking.
5268 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
5269 if (Legal->prepareToFoldTailByMasking()) {
5270 FoldTailByMasking = true;
5271 return MaxFactors;
5272 }
5273
5274 // If there was a tail-folding hint/switch, but we can't fold the tail by
5275 // masking, fallback to a vectorization with a scalar epilogue.
5276 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
5277 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n"; } } while (false)
5278 "scalar epilogue instead.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n"; } } while (false)
;
5279 ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
5280 return MaxFactors;
5281 }
5282
5283 if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
5284 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Can't fold tail by masking: don't vectorize\n"
; } } while (false)
;
5285 return FixedScalableVFPair::getNone();
5286 }
5287
5288 if (TC == 0) {
5289 reportVectorizationFailure(
5290 "Unable to calculate the loop count due to complex control flow",
5291 "unable to calculate the loop count due to complex control flow",
5292 "UnknownLoopCountComplexCFG", ORE, TheLoop);
5293 return FixedScalableVFPair::getNone();
5294 }
5295
5296 reportVectorizationFailure(
5297 "Cannot optimize for size and vectorize at the same time.",
5298 "cannot optimize for size and vectorize at the same time. "
5299 "Enable vectorization of this loop with '#pragma clang loop "
5300 "vectorize(enable)' when compiling with -Os/-Oz",
5301 "NoTailLoopWithOptForSize", ORE, TheLoop);
5302 return FixedScalableVFPair::getNone();
5303}
5304
5305ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget(
5306 unsigned ConstTripCount, unsigned SmallestType, unsigned WidestType,
5307 const ElementCount &MaxSafeVF, bool FoldTailByMasking) {
5308 bool ComputeScalableMaxVF = MaxSafeVF.isScalable();
5309 TypeSize WidestRegister = TTI.getRegisterBitWidth(
5310 ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector
5311 : TargetTransformInfo::RGK_FixedWidthVector);
5312
5313 // Convenience function to return the minimum of two ElementCounts.
5314 auto MinVF = [](const ElementCount &LHS, const ElementCount &RHS) {
5315 assert((LHS.isScalable() == RHS.isScalable()) &&(static_cast <bool> ((LHS.isScalable() == RHS.isScalable
()) && "Scalable flags must match") ? void (0) : __assert_fail
("(LHS.isScalable() == RHS.isScalable()) && \"Scalable flags must match\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5316, __extension__
__PRETTY_FUNCTION__))
5316 "Scalable flags must match")(static_cast <bool> ((LHS.isScalable() == RHS.isScalable
()) && "Scalable flags must match") ? void (0) : __assert_fail
("(LHS.isScalable() == RHS.isScalable()) && \"Scalable flags must match\""
, "llvm/lib/Transforms/Vectorize/LoopVectorize.cpp", 5316, __extension__
__PRETTY_FUNCTION__))
;
5317 return ElementCount::isKnownLT(LHS, RHS) ? LHS : RHS;
5318 };
5319
5320 // Ensure MaxVF is a power of 2; the dependence distance bound may not be.
5321 // Note that both WidestRegister and WidestType may not be a powers of 2.
5322 auto MaxVectorElementCount = ElementCount::get(
5323 PowerOf2Floor(WidestRegister.getKnownMinSize() / WidestType),
5324 ComputeScalableMaxVF);
5325 MaxVectorElementCount = MinVF(MaxVectorElementCount, MaxSafeVF);
5326 LLVM_DEBUG(dbgs() << "LV: The Widest register safe to use is: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register safe to use is: "
<< (MaxVectorElementCount * WidestType) << " bits.\n"
; } } while (false)
5327 << (MaxVectorElementCount * WidestType) << " bits.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The Widest register safe to use is: "
<< (MaxVectorElementCount * WidestType) << " bits.\n"
; } } while (false)
;
5328
5329 if (!MaxVectorElementCount) {
5330 LLVM_DEBUG(dbgs() << "LV: The target has no "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has no "
<< (ComputeScalableMaxVF ? "scalable" : "fixed") <<
" vector registers.\n"; } } while (false)
5331 << (ComputeScalableMaxVF ? "scalable" : "fixed")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has no "
<< (ComputeScalableMaxVF ? "scalable" : "fixed") <<
" vector registers.\n"; } } while (false)
5332 << " vector registers.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: The target has no "
<< (ComputeScalableMaxVF ? "scalable" : "fixed") <<
" vector registers.\n"; } } while (false)
;
5333 return ElementCount::getFixed(1);
5334 }
5335
5336 const auto TripCountEC = ElementCount::getFixed(ConstTripCount);
5337 if (ConstTripCount &&
5338 ElementCount::isKnownLE(TripCountEC, MaxVectorElementCount) &&
5339 (!FoldTailByMasking || isPowerOf2_32(ConstTripCount))) {
5340 // If loop trip count (TC) is known at compile time there is no point in
5341 // choosing VF greater than TC (as done in the loop below). Select maximum
5342 // power of two which doesn't exceed TC.
5343 // If MaxVectorElementCount is scalable, we only fall back on a fixed VF
5344 // when the TC is less than or equal to the known number of lanes.
5345 auto ClampedConstTripCount = PowerOf2Floor(ConstTripCount);
5346 LLVM_DEBUG(dbgs() << "LV: Clamping the MaxVF to maximum power of two not "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
"exceeding the constant trip count: " << ClampedConstTripCount
<< "\n"; } } while (false)
5347 "exceeding the constant trip count: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
"exceeding the constant trip count: " << ClampedConstTripCount
<< "\n"; } } while (false)
5348 << ClampedConstTripCount << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Clamping the MaxVF to maximum power of two not "
"exceeding the constant trip count: " << ClampedConstTripCount
<< "\n"; } } while (false)
;
5349 return ElementCount::getFixed(ClampedConstTripCount);
5350 }
5351
5352 ElementCount MaxVF = MaxVectorElementCount;
5353 if (TTI.shouldMaximizeVectorBandwidth() ||
5354 (MaximizeBandwidth && isScalarEpilogueAllowed())) {
5355 auto MaxVectorElementCountMaxBW = ElementCount::get(
5356 PowerOf2Floor(WidestRegister.getKnownMinSize() / SmallestType),
5357 ComputeScalableMaxVF);
5358 MaxVectorElementCountMaxBW = MinVF(MaxVectorElementCountMaxBW, MaxSafeVF);
5359
5360 // Collect all viable vectorization factors larger than the default MaxVF
5361 // (i.e. MaxVectorElementCount).
5362 SmallVector<ElementCount, 8> VFs;
5363 for (ElementCount VS = MaxVectorElementCount * 2;
5364 ElementCount::isKnownLE(VS, MaxVectorElementCountMaxBW); VS *= 2)
5365 VFs.push_back(VS);
5366
5367 // For each VF calculate its register usage.
5368 auto RUs = calculateRegisterUsage(VFs);
5369
5370 // Select the largest VF which doesn't require more registers than existing
5371 // ones.
5372 for (int i = RUs.size() - 1; i >= 0; --i) {
5373 bool Selected = true;
5374 for (auto &pair : RUs[i].MaxLocalUsers) {
5375 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(pair.first);
5376 if (pair.second > TargetNumRegisters)
5377 Selected = false;
5378 }
5379 if (Selected) {
5380 MaxVF = VFs[i];
5381 break;
5382 }
5383 }
5384 if (ElementCount MinVF =
5385 TTI.getMinimumVF(SmallestType, ComputeScalableMaxVF)) {
5386 if (ElementCount::isKnownLT(MaxVF, MinVF)) {
5387 LLVM_DEBUG(dbgs() << "LV: Overriding calculated MaxVF(" << MaxVFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Overriding calculated MaxVF("
<< MaxVF << ") with target's minimum: " <<
MinVF << '\n'; } } while (false)
5388 << ") with target's minimum: " << MinVF << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("loop-vectorize")) { dbgs() << "LV: Overriding calculated MaxVF("
<< MaxVF << ") with target's minimum: " <<
MinVF << '\n'; } } while (false)
;
5389 MaxVF = MinVF;
5390 }
5391 }
5392 }
5393 return MaxVF;
5394}
5395
5396Optional<unsigned> LoopVectorizationCostModel::getVScaleForTuning() const {
5397 if<