Bug Summary

File:llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Warning:line 5345, column 41
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LoopVectorize.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Transforms/Vectorize -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include -D NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-04-040900-46481-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/Proposal/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanHCFGBuilder.h"
61#include "VPlanPredicator.h"
62#include "VPlanTransforms.h"
63#include "llvm/ADT/APInt.h"
64#include "llvm/ADT/ArrayRef.h"
65#include "llvm/ADT/DenseMap.h"
66#include "llvm/ADT/DenseMapInfo.h"
67#include "llvm/ADT/Hashing.h"
68#include "llvm/ADT/MapVector.h"
69#include "llvm/ADT/None.h"
70#include "llvm/ADT/Optional.h"
71#include "llvm/ADT/STLExtras.h"
72#include "llvm/ADT/SmallPtrSet.h"
73#include "llvm/ADT/SmallSet.h"
74#include "llvm/ADT/SmallVector.h"
75#include "llvm/ADT/Statistic.h"
76#include "llvm/ADT/StringRef.h"
77#include "llvm/ADT/Twine.h"
78#include "llvm/ADT/iterator_range.h"
79#include "llvm/Analysis/AssumptionCache.h"
80#include "llvm/Analysis/BasicAliasAnalysis.h"
81#include "llvm/Analysis/BlockFrequencyInfo.h"
82#include "llvm/Analysis/CFG.h"
83#include "llvm/Analysis/CodeMetrics.h"
84#include "llvm/Analysis/DemandedBits.h"
85#include "llvm/Analysis/GlobalsModRef.h"
86#include "llvm/Analysis/LoopAccessAnalysis.h"
87#include "llvm/Analysis/LoopAnalysisManager.h"
88#include "llvm/Analysis/LoopInfo.h"
89#include "llvm/Analysis/LoopIterator.h"
90#include "llvm/Analysis/OptimizationRemarkEmitter.h"
91#include "llvm/Analysis/ProfileSummaryInfo.h"
92#include "llvm/Analysis/ScalarEvolution.h"
93#include "llvm/Analysis/ScalarEvolutionExpressions.h"
94#include "llvm/Analysis/TargetLibraryInfo.h"
95#include "llvm/Analysis/TargetTransformInfo.h"
96#include "llvm/Analysis/VectorUtils.h"
97#include "llvm/IR/Attributes.h"
98#include "llvm/IR/BasicBlock.h"
99#include "llvm/IR/CFG.h"
100#include "llvm/IR/Constant.h"
101#include "llvm/IR/Constants.h"
102#include "llvm/IR/DataLayout.h"
103#include "llvm/IR/DebugInfoMetadata.h"
104#include "llvm/IR/DebugLoc.h"
105#include "llvm/IR/DerivedTypes.h"
106#include "llvm/IR/DiagnosticInfo.h"
107#include "llvm/IR/Dominators.h"
108#include "llvm/IR/Function.h"
109#include "llvm/IR/IRBuilder.h"
110#include "llvm/IR/InstrTypes.h"
111#include "llvm/IR/Instruction.h"
112#include "llvm/IR/Instructions.h"
113#include "llvm/IR/IntrinsicInst.h"
114#include "llvm/IR/Intrinsics.h"
115#include "llvm/IR/LLVMContext.h"
116#include "llvm/IR/Metadata.h"
117#include "llvm/IR/Module.h"
118#include "llvm/IR/Operator.h"
119#include "llvm/IR/PatternMatch.h"
120#include "llvm/IR/Type.h"
121#include "llvm/IR/Use.h"
122#include "llvm/IR/User.h"
123#include "llvm/IR/Value.h"
124#include "llvm/IR/ValueHandle.h"
125#include "llvm/IR/Verifier.h"
126#include "llvm/InitializePasses.h"
127#include "llvm/Pass.h"
128#include "llvm/Support/Casting.h"
129#include "llvm/Support/CommandLine.h"
130#include "llvm/Support/Compiler.h"
131#include "llvm/Support/Debug.h"
132#include "llvm/Support/ErrorHandling.h"
133#include "llvm/Support/InstructionCost.h"
134#include "llvm/Support/MathExtras.h"
135#include "llvm/Support/raw_ostream.h"
136#include "llvm/Transforms/Utils/BasicBlockUtils.h"
137#include "llvm/Transforms/Utils/InjectTLIMappings.h"
138#include "llvm/Transforms/Utils/LoopSimplify.h"
139#include "llvm/Transforms/Utils/LoopUtils.h"
140#include "llvm/Transforms/Utils/LoopVersioning.h"
141#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
142#include "llvm/Transforms/Utils/SizeOpts.h"
143#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
144#include <algorithm>
145#include <cassert>
146#include <cstdint>
147#include <cstdlib>
148#include <functional>
149#include <iterator>
150#include <limits>
151#include <memory>
152#include <string>
153#include <tuple>
154#include <utility>
155
156using namespace llvm;
157
158#define LV_NAME"loop-vectorize" "loop-vectorize"
159#define DEBUG_TYPE"loop-vectorize" LV_NAME"loop-vectorize"
160
161#ifndef NDEBUG1
162const char VerboseDebug[] = DEBUG_TYPE"loop-vectorize" "-verbose";
163#endif
164
165/// @{
166/// Metadata attribute names
167const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all";
168const char LLVMLoopVectorizeFollowupVectorized[] =
169 "llvm.loop.vectorize.followup_vectorized";
170const char LLVMLoopVectorizeFollowupEpilogue[] =
171 "llvm.loop.vectorize.followup_epilogue";
172/// @}
173
174STATISTIC(LoopsVectorized, "Number of loops vectorized")static llvm::Statistic LoopsVectorized = {"loop-vectorize", "LoopsVectorized"
, "Number of loops vectorized"}
;
175STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization")static llvm::Statistic LoopsAnalyzed = {"loop-vectorize", "LoopsAnalyzed"
, "Number of loops analyzed for vectorization"}
;
176STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized")static llvm::Statistic LoopsEpilogueVectorized = {"loop-vectorize"
, "LoopsEpilogueVectorized", "Number of epilogues vectorized"
}
;
177
178static cl::opt<bool> EnableEpilogueVectorization(
179 "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
180 cl::desc("Enable vectorization of epilogue loops."));
181
182static cl::opt<unsigned> EpilogueVectorizationForceVF(
183 "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
184 cl::desc("When epilogue vectorization is enabled, and a value greater than "
185 "1 is specified, forces the given VF for all applicable epilogue "
186 "loops."));
187
188static cl::opt<unsigned> EpilogueVectorizationMinVF(
189 "epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden,
190 cl::desc("Only loops with vectorization factor equal to or larger than "
191 "the specified value are considered for epilogue vectorization."));
192
193/// Loops with a known constant trip count below this number are vectorized only
194/// if no scalar iteration overheads are incurred.
195static cl::opt<unsigned> TinyTripCountVectorThreshold(
196 "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
197 cl::desc("Loops with a constant trip count that is smaller than this "
198 "value are vectorized only if no scalar iteration overheads "
199 "are incurred."));
200
201static cl::opt<unsigned> PragmaVectorizeMemoryCheckThreshold(
202 "pragma-vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
203 cl::desc("The maximum allowed number of runtime memory checks with a "
204 "vectorize(enable) pragma."));
205
206// Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
207// that predication is preferred, and this lists all options. I.e., the
208// vectorizer will try to fold the tail-loop (epilogue) into the vector body
209// and predicate the instructions accordingly. If tail-folding fails, there are
210// different fallback strategies depending on these values:
211namespace PreferPredicateTy {
212 enum Option {
213 ScalarEpilogue = 0,
214 PredicateElseScalarEpilogue,
215 PredicateOrDontVectorize
216 };
217} // namespace PreferPredicateTy
218
219static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
220 "prefer-predicate-over-epilogue",
221 cl::init(PreferPredicateTy::ScalarEpilogue),
222 cl::Hidden,
223 cl::desc("Tail-folding and predication preferences over creating a scalar "
224 "epilogue loop."),
225 cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
226 "scalar-epilogue",llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
227 "Don't tail-predicate loops, create scalar epilogue")llvm::cl::OptionEnumValue { "scalar-epilogue", int(PreferPredicateTy
::ScalarEpilogue), "Don't tail-predicate loops, create scalar epilogue"
}
,
228 clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
229 "predicate-else-scalar-epilogue",llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
230 "prefer tail-folding, create scalar epilogue if tail "llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
231 "folding fails.")llvm::cl::OptionEnumValue { "predicate-else-scalar-epilogue",
int(PreferPredicateTy::PredicateElseScalarEpilogue), "prefer tail-folding, create scalar epilogue if tail "
"folding fails." }
,
232 clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
233 "predicate-dont-vectorize",llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
234 "prefers tail-folding, don't attempt vectorization if "llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
235 "tail-folding fails.")llvm::cl::OptionEnumValue { "predicate-dont-vectorize", int(PreferPredicateTy
::PredicateOrDontVectorize), "prefers tail-folding, don't attempt vectorization if "
"tail-folding fails." }
));
236
237static cl::opt<bool> MaximizeBandwidth(
238 "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
239 cl::desc("Maximize bandwidth when selecting vectorization factor which "
240 "will be determined by the smallest type in loop."));
241
242static cl::opt<bool> EnableInterleavedMemAccesses(
243 "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
244 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
245
246/// An interleave-group may need masking if it resides in a block that needs
247/// predication, or in order to mask away gaps.
248static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
249 "enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden,
250 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
251
252static cl::opt<unsigned> TinyTripCountInterleaveThreshold(
253 "tiny-trip-count-interleave-threshold", cl::init(128), cl::Hidden,
254 cl::desc("We don't interleave loops with a estimated constant trip count "
255 "below this number"));
256
257static cl::opt<unsigned> ForceTargetNumScalarRegs(
258 "force-target-num-scalar-regs", cl::init(0), cl::Hidden,
259 cl::desc("A flag that overrides the target's number of scalar registers."));
260
261static cl::opt<unsigned> ForceTargetNumVectorRegs(
262 "force-target-num-vector-regs", cl::init(0), cl::Hidden,
263 cl::desc("A flag that overrides the target's number of vector registers."));
264
265static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
266 "force-target-max-scalar-interleave", cl::init(0), cl::Hidden,
267 cl::desc("A flag that overrides the target's max interleave factor for "
268 "scalar loops."));
269
270static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
271 "force-target-max-vector-interleave", cl::init(0), cl::Hidden,
272 cl::desc("A flag that overrides the target's max interleave factor for "
273 "vectorized loops."));
274
275static cl::opt<unsigned> ForceTargetInstructionCost(
276 "force-target-instruction-cost", cl::init(0), cl::Hidden,
277 cl::desc("A flag that overrides the target's expected cost for "
278 "an instruction to a single constant value. Mostly "
279 "useful for getting consistent testing."));
280
281static cl::opt<bool> ForceTargetSupportsScalableVectors(
282 "force-target-supports-scalable-vectors", cl::init(false), cl::Hidden,
283 cl::desc(
284 "Pretend that scalable vectors are supported, even if the target does "
285 "not support them. This flag should only be used for testing."));
286
287static cl::opt<unsigned> SmallLoopCost(
288 "small-loop-cost", cl::init(20), cl::Hidden,
289 cl::desc(
290 "The cost of a loop that is considered 'small' by the interleaver."));
291
292static cl::opt<bool> LoopVectorizeWithBlockFrequency(
293 "loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden,
294 cl::desc("Enable the use of the block frequency analysis to access PGO "
295 "heuristics minimizing code growth in cold regions and being more "
296 "aggressive in hot regions."));
297
298// Runtime interleave loops for load/store throughput.
299static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
300 "enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden,
301 cl::desc(
302 "Enable runtime interleaving until load/store ports are saturated"));
303
304/// Interleave small loops with scalar reductions.
305static cl::opt<bool> InterleaveSmallLoopScalarReduction(
306 "interleave-small-loop-scalar-reduction", cl::init(false), cl::Hidden,
307 cl::desc("Enable interleaving for loops with small iteration counts that "
308 "contain scalar reductions to expose ILP."));
309
310/// The number of stores in a loop that are allowed to need predication.
311static cl::opt<unsigned> NumberOfStoresToPredicate(
312 "vectorize-num-stores-pred", cl::init(1), cl::Hidden,
313 cl::desc("Max number of stores to be predicated behind an if."));
314
315static cl::opt<bool> EnableIndVarRegisterHeur(
316 "enable-ind-var-reg-heur", cl::init(true), cl::Hidden,
317 cl::desc("Count the induction variable only once when interleaving"));
318
319static cl::opt<bool> EnableCondStoresVectorization(
320 "enable-cond-stores-vec", cl::init(true), cl::Hidden,
321 cl::desc("Enable if predication of stores during vectorization."));
322
323static cl::opt<unsigned> MaxNestedScalarReductionIC(
324 "max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden,
325 cl::desc("The maximum interleave count to use when interleaving a scalar "
326 "reduction in a nested loop."));
327
328static cl::opt<bool>
329 PreferInLoopReductions("prefer-inloop-reductions", cl::init(false),
330 cl::Hidden,
331 cl::desc("Prefer in-loop vector reductions, "
332 "overriding the targets preference."));
333
334static cl::opt<bool> ForceOrderedReductions(
335 "force-ordered-reductions", cl::init(false), cl::Hidden,
336 cl::desc("Enable the vectorisation of loops with in-order (strict) "
337 "FP reductions"));
338
339static cl::opt<bool> PreferPredicatedReductionSelect(
340 "prefer-predicated-reduction-select", cl::init(false), cl::Hidden,
341 cl::desc(
342 "Prefer predicating a reduction operation over an after loop select."));
343
344cl::opt<bool> EnableVPlanNativePath(
345 "enable-vplan-native-path", cl::init(false), cl::Hidden,
346 cl::desc("Enable VPlan-native vectorization path with "
347 "support for outer loop vectorization."));
348
349// FIXME: Remove this switch once we have divergence analysis. Currently we
350// assume divergent non-backedge branches when this switch is true.
351cl::opt<bool> EnableVPlanPredication(
352 "enable-vplan-predication", cl::init(false), cl::Hidden,
353 cl::desc("Enable VPlan-native vectorization path predicator with "
354 "support for outer loop vectorization."));
355
356// This flag enables the stress testing of the VPlan H-CFG construction in the
357// VPlan-native vectorization path. It must be used in conjuction with
358// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
359// verification of the H-CFGs built.
360static cl::opt<bool> VPlanBuildStressTest(
361 "vplan-build-stress-test", cl::init(false), cl::Hidden,
362 cl::desc(
363 "Build VPlan for every supported loop nest in the function and bail "
364 "out right after the build (stress test the VPlan H-CFG construction "
365 "in the VPlan-native vectorization path)."));
366
367cl::opt<bool> llvm::EnableLoopInterleaving(
368 "interleave-loops", cl::init(true), cl::Hidden,
369 cl::desc("Enable loop interleaving in Loop vectorization passes"));
370cl::opt<bool> llvm::EnableLoopVectorization(
371 "vectorize-loops", cl::init(true), cl::Hidden,
372 cl::desc("Run the Loop vectorization passes"));
373
374cl::opt<bool> PrintVPlansInDotFormat(
375 "vplan-print-in-dot-format", cl::init(false), cl::Hidden,
376 cl::desc("Use dot format instead of plain text when dumping VPlans"));
377
378/// A helper function that returns true if the given type is irregular. The
379/// type is irregular if its allocated size doesn't equal the store size of an
380/// element of the corresponding vector type.
381static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
382 // Determine if an array of N elements of type Ty is "bitcast compatible"
383 // with a <N x Ty> vector.
384 // This is only true if there is no padding between the array elements.
385 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
386}
387
388/// A helper function that returns the reciprocal of the block probability of
389/// predicated blocks. If we return X, we are assuming the predicated block
390/// will execute once for every X iterations of the loop header.
391///
392/// TODO: We should use actual block probability here, if available. Currently,
393/// we always assume predicated blocks have a 50% chance of executing.
394static unsigned getReciprocalPredBlockProb() { return 2; }
395
396/// A helper function that returns an integer or floating-point constant with
397/// value C.
398static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
399 return Ty->isIntegerTy() ? ConstantInt::getSigned(Ty, C)
400 : ConstantFP::get(Ty, C);
401}
402
403/// Returns "best known" trip count for the specified loop \p L as defined by
404/// the following procedure:
405/// 1) Returns exact trip count if it is known.
406/// 2) Returns expected trip count according to profile data if any.
407/// 3) Returns upper bound estimate if it is known.
408/// 4) Returns None if all of the above failed.
409static Optional<unsigned> getSmallBestKnownTC(ScalarEvolution &SE, Loop *L) {
410 // Check if exact trip count is known.
411 if (unsigned ExpectedTC = SE.getSmallConstantTripCount(L))
412 return ExpectedTC;
413
414 // Check if there is an expected trip count available from profile data.
415 if (LoopVectorizeWithBlockFrequency)
416 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
417 return EstimatedTC;
418
419 // Check if upper bound estimate is known.
420 if (unsigned ExpectedTC = SE.getSmallConstantMaxTripCount(L))
421 return ExpectedTC;
422
423 return None;
424}
425
426// Forward declare GeneratedRTChecks.
427class GeneratedRTChecks;
428
429namespace llvm {
430
431/// InnerLoopVectorizer vectorizes loops which contain only one basic
432/// block to a specified vectorization factor (VF).
433/// This class performs the widening of scalars into vectors, or multiple
434/// scalars. This class also implements the following features:
435/// * It inserts an epilogue loop for handling loops that don't have iteration
436/// counts that are known to be a multiple of the vectorization factor.
437/// * It handles the code generation for reduction variables.
438/// * Scalarization (implementation using scalars) of un-vectorizable
439/// instructions.
440/// InnerLoopVectorizer does not perform any vectorization-legality
441/// checks, and relies on the caller to check for the different legality
442/// aspects. The InnerLoopVectorizer relies on the
443/// LoopVectorizationLegality class to provide information about the induction
444/// and reduction variables that were found to a given vectorization factor.
445class InnerLoopVectorizer {
446public:
447 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
448 LoopInfo *LI, DominatorTree *DT,
449 const TargetLibraryInfo *TLI,
450 const TargetTransformInfo *TTI, AssumptionCache *AC,
451 OptimizationRemarkEmitter *ORE, ElementCount VecWidth,
452 unsigned UnrollFactor, LoopVectorizationLegality *LVL,
453 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
454 ProfileSummaryInfo *PSI, GeneratedRTChecks &RTChecks)
455 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
456 AC(AC), ORE(ORE), VF(VecWidth), UF(UnrollFactor),
457 Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI),
458 PSI(PSI), RTChecks(RTChecks) {
459 // Query this against the original loop and save it here because the profile
460 // of the original loop header may change as the transformation happens.
461 OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize(
462 OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass);
463 }
464
465 virtual ~InnerLoopVectorizer() = default;
466
467 /// Create a new empty loop that will contain vectorized instructions later
468 /// on, while the old loop will be used as the scalar remainder. Control flow
469 /// is generated around the vectorized (and scalar epilogue) loops consisting
470 /// of various checks and bypasses. Return the pre-header block of the new
471 /// loop.
472 /// In the case of epilogue vectorization, this function is overriden to
473 /// handle the more complex control flow around the loops.
474 virtual BasicBlock *createVectorizedLoopSkeleton();
475
476 /// Widen a single instruction within the innermost loop.
477 void widenInstruction(Instruction &I, VPValue *Def, VPUser &Operands,
478 VPTransformState &State);
479
480 /// Widen a single call instruction within the innermost loop.
481 void widenCallInstruction(CallInst &I, VPValue *Def, VPUser &ArgOperands,
482 VPTransformState &State);
483
484 /// Widen a single select instruction within the innermost loop.
485 void widenSelectInstruction(SelectInst &I, VPValue *VPDef, VPUser &Operands,
486 bool InvariantCond, VPTransformState &State);
487
488 /// Fix the vectorized code, taking care of header phi's, live-outs, and more.
489 void fixVectorizedLoop(VPTransformState &State);
490
491 // Return true if any runtime check is added.
492 bool areSafetyChecksAdded() { return AddedSafetyChecks; }
493
494 /// A type for vectorized values in the new loop. Each value from the
495 /// original loop, when vectorized, is represented by UF vector values in the
496 /// new unrolled loop, where UF is the unroll factor.
497 using VectorParts = SmallVector<Value *, 2>;
498
499 /// Vectorize a single GetElementPtrInst based on information gathered and
500 /// decisions taken during planning.
501 void widenGEP(GetElementPtrInst *GEP, VPValue *VPDef, VPUser &Indices,
502 unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
503 SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
504
505 /// Vectorize a single first-order recurrence or pointer induction PHINode in
506 /// a block. This method handles the induction variable canonicalization. It
507 /// supports both VF = 1 for unrolled loops and arbitrary length vectors.
508 void widenPHIInstruction(Instruction *PN, VPWidenPHIRecipe *PhiR,
509 VPTransformState &State);
510
511 /// A helper function to scalarize a single Instruction in the innermost loop.
512 /// Generates a sequence of scalar instances for each lane between \p MinLane
513 /// and \p MaxLane, times each part between \p MinPart and \p MaxPart,
514 /// inclusive. Uses the VPValue operands from \p Operands instead of \p
515 /// Instr's operands.
516 void scalarizeInstruction(Instruction *Instr, VPValue *Def, VPUser &Operands,
517 const VPIteration &Instance, bool IfPredicateInstr,
518 VPTransformState &State);
519
520 /// Widen an integer or floating-point induction variable \p IV. If \p Trunc
521 /// is provided, the integer induction variable will first be truncated to
522 /// the corresponding type.
523 void widenIntOrFpInduction(PHINode *IV, Value *Start, TruncInst *Trunc,
524 VPValue *Def, VPValue *CastDef,
525 VPTransformState &State);
526
527 /// Construct the vector value of a scalarized value \p V one lane at a time.
528 void packScalarIntoVectorValue(VPValue *Def, const VPIteration &Instance,
529 VPTransformState &State);
530
531 /// Try to vectorize interleaved access group \p Group with the base address
532 /// given in \p Addr, optionally masking the vector operations if \p
533 /// BlockInMask is non-null. Use \p State to translate given VPValues to IR
534 /// values in the vectorized loop.
535 void vectorizeInterleaveGroup(const InterleaveGroup<Instruction> *Group,
536 ArrayRef<VPValue *> VPDefs,
537 VPTransformState &State, VPValue *Addr,
538 ArrayRef<VPValue *> StoredValues,
539 VPValue *BlockInMask = nullptr);
540
541 /// Vectorize Load and Store instructions with the base address given in \p
542 /// Addr, optionally masking the vector operations if \p BlockInMask is
543 /// non-null. Use \p State to translate given VPValues to IR values in the
544 /// vectorized loop.
545 void vectorizeMemoryInstruction(Instruction *Instr, VPTransformState &State,
546 VPValue *Def, VPValue *Addr,
547 VPValue *StoredValue, VPValue *BlockInMask);
548
549 /// Set the debug location in the builder \p Ptr using the debug location in
550 /// \p V. If \p Ptr is None then it uses the class member's Builder.
551 void setDebugLocFromInst(const Value *V,
552 Optional<IRBuilder<> *> CustomBuilder = None);
553
554 /// Fix the non-induction PHIs in the OrigPHIsToFix vector.
555 void fixNonInductionPHIs(VPTransformState &State);
556
557 /// Returns true if the reordering of FP operations is not allowed, but we are
558 /// able to vectorize with strict in-order reductions for the given RdxDesc.
559 bool useOrderedReductions(RecurrenceDescriptor &RdxDesc);
560
561 /// Create a broadcast instruction. This method generates a broadcast
562 /// instruction (shuffle) for loop invariant values and for the induction
563 /// value. If this is the induction variable then we extend it to N, N+1, ...
564 /// this is needed because each iteration in the loop corresponds to a SIMD
565 /// element.
566 virtual Value *getBroadcastInstrs(Value *V);
567
568protected:
569 friend class LoopVectorizationPlanner;
570
571 /// A small list of PHINodes.
572 using PhiVector = SmallVector<PHINode *, 4>;
573
574 /// A type for scalarized values in the new loop. Each value from the
575 /// original loop, when scalarized, is represented by UF x VF scalar values
576 /// in the new unrolled loop, where UF is the unroll factor and VF is the
577 /// vectorization factor.
578 using ScalarParts = SmallVector<SmallVector<Value *, 4>, 2>;
579
580 /// Set up the values of the IVs correctly when exiting the vector loop.
581 void fixupIVUsers(PHINode *OrigPhi, const InductionDescriptor &II,
582 Value *CountRoundDown, Value *EndValue,
583 BasicBlock *MiddleBlock);
584
585 /// Create a new induction variable inside L.
586 PHINode *createInductionVariable(Loop *L, Value *Start, Value *End,
587 Value *Step, Instruction *DL);
588
589 /// Handle all cross-iteration phis in the header.
590 void fixCrossIterationPHIs(VPTransformState &State);
591
592 /// Create the exit value of first order recurrences in the middle block and
593 /// update their users.
594 void fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, VPTransformState &State);
595
596 /// Create code for the loop exit value of the reduction.
597 void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
598
599 /// Clear NSW/NUW flags from reduction instructions if necessary.
600 void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
601 VPTransformState &State);
602
603 /// Fixup the LCSSA phi nodes in the unique exit block. This simply
604 /// means we need to add the appropriate incoming value from the middle
605 /// block as exiting edges from the scalar epilogue loop (if present) are
606 /// already in place, and we exit the vector loop exclusively to the middle
607 /// block.
608 void fixLCSSAPHIs(VPTransformState &State);
609
610 /// Iteratively sink the scalarized operands of a predicated instruction into
611 /// the block that was created for it.
612 void sinkScalarOperands(Instruction *PredInst);
613
614 /// Shrinks vector element sizes to the smallest bitwidth they can be legally
615 /// represented as.
616 void truncateToMinimalBitwidths(VPTransformState &State);
617
618 /// This function adds
619 /// (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step, ...)
620 /// to each vector element of Val. The sequence starts at StartIndex.
621 /// \p Opcode is relevant for FP induction variable.
622 virtual Value *getStepVector(Value *Val, int StartIdx, Value *Step,
623 Instruction::BinaryOps Opcode =
624 Instruction::BinaryOpsEnd);
625
626 /// Compute scalar induction steps. \p ScalarIV is the scalar induction
627 /// variable on which to base the steps, \p Step is the size of the step, and
628 /// \p EntryVal is the value from the original loop that maps to the steps.
629 /// Note that \p EntryVal doesn't have to be an induction variable - it
630 /// can also be a truncate instruction.
631 void buildScalarSteps(Value *ScalarIV, Value *Step, Instruction *EntryVal,
632 const InductionDescriptor &ID, VPValue *Def,
633 VPValue *CastDef, VPTransformState &State);
634
635 /// Create a vector induction phi node based on an existing scalar one. \p
636 /// EntryVal is the value from the original loop that maps to the vector phi
637 /// node, and \p Step is the loop-invariant step. If \p EntryVal is a
638 /// truncate instruction, instead of widening the original IV, we widen a
639 /// version of the IV truncated to \p EntryVal's type.
640 void createVectorIntOrFpInductionPHI(const InductionDescriptor &II,
641 Value *Step, Value *Start,
642 Instruction *EntryVal, VPValue *Def,
643 VPValue *CastDef,
644 VPTransformState &State);
645
646 /// Returns true if an instruction \p I should be scalarized instead of
647 /// vectorized for the chosen vectorization factor.
648 bool shouldScalarizeInstruction(Instruction *I) const;
649
650 /// Returns true if we should generate a scalar version of \p IV.
651 bool needsScalarInduction(Instruction *IV) const;
652
653 /// If there is a cast involved in the induction variable \p ID, which should
654 /// be ignored in the vectorized loop body, this function records the
655 /// VectorLoopValue of the respective Phi also as the VectorLoopValue of the
656 /// cast. We had already proved that the casted Phi is equal to the uncasted
657 /// Phi in the vectorized loop (under a runtime guard), and therefore
658 /// there is no need to vectorize the cast - the same value can be used in the
659 /// vector loop for both the Phi and the cast.
660 /// If \p VectorLoopValue is a scalarized value, \p Lane is also specified,
661 /// Otherwise, \p VectorLoopValue is a widened/vectorized value.
662 ///
663 /// \p EntryVal is the value from the original loop that maps to the vector
664 /// phi node and is used to distinguish what is the IV currently being
665 /// processed - original one (if \p EntryVal is a phi corresponding to the
666 /// original IV) or the "newly-created" one based on the proof mentioned above
667 /// (see also buildScalarSteps() and createVectorIntOrFPInductionPHI()). In the
668 /// latter case \p EntryVal is a TruncInst and we must not record anything for
669 /// that IV, but it's error-prone to expect callers of this routine to care
670 /// about that, hence this explicit parameter.
671 void recordVectorLoopValueForInductionCast(
672 const InductionDescriptor &ID, const Instruction *EntryVal,
673 Value *VectorLoopValue, VPValue *CastDef, VPTransformState &State,
674 unsigned Part, unsigned Lane = UINT_MAX(2147483647 *2U +1U));
675
676 /// Generate a shuffle sequence that will reverse the vector Vec.
677 virtual Value *reverseVector(Value *Vec);
678
679 /// Returns (and creates if needed) the original loop trip count.
680 Value *getOrCreateTripCount(Loop *NewLoop);
681
682 /// Returns (and creates if needed) the trip count of the widened loop.
683 Value *getOrCreateVectorTripCount(Loop *NewLoop);
684
685 /// Returns a bitcasted value to the requested vector type.
686 /// Also handles bitcasts of vector<float> <-> vector<pointer> types.
687 Value *createBitOrPointerCast(Value *V, VectorType *DstVTy,
688 const DataLayout &DL);
689
690 /// Emit a bypass check to see if the vector trip count is zero, including if
691 /// it overflows.
692 void emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass);
693
694 /// Emit a bypass check to see if all of the SCEV assumptions we've
695 /// had to make are correct. Returns the block containing the checks or
696 /// nullptr if no checks have been added.
697 BasicBlock *emitSCEVChecks(Loop *L, BasicBlock *Bypass);
698
699 /// Emit bypass checks to check any memory assumptions we may have made.
700 /// Returns the block containing the checks or nullptr if no checks have been
701 /// added.
702 BasicBlock *emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass);
703
704 /// Compute the transformed value of Index at offset StartValue using step
705 /// StepValue.
706 /// For integer induction, returns StartValue + Index * StepValue.
707 /// For pointer induction, returns StartValue[Index * StepValue].
708 /// FIXME: The newly created binary instructions should contain nsw/nuw
709 /// flags, which can be found from the original scalar operations.
710 Value *emitTransformedIndex(IRBuilder<> &B, Value *Index, ScalarEvolution *SE,
711 const DataLayout &DL,
712 const InductionDescriptor &ID) const;
713
714 /// Emit basic blocks (prefixed with \p Prefix) for the iteration check,
715 /// vector loop preheader, middle block and scalar preheader. Also
716 /// allocate a loop object for the new vector loop and return it.
717 Loop *createVectorLoopSkeleton(StringRef Prefix);
718
719 /// Create new phi nodes for the induction variables to resume iteration count
720 /// in the scalar epilogue, from where the vectorized loop left off (given by
721 /// \p VectorTripCount).
722 /// In cases where the loop skeleton is more complicated (eg. epilogue
723 /// vectorization) and the resume values can come from an additional bypass
724 /// block, the \p AdditionalBypass pair provides information about the bypass
725 /// block and the end value on the edge from bypass to this loop.
726 void createInductionResumeValues(
727 Loop *L, Value *VectorTripCount,
728 std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
729
730 /// Complete the loop skeleton by adding debug MDs, creating appropriate
731 /// conditional branches in the middle block, preparing the builder and
732 /// running the verifier. Take in the vector loop \p L as argument, and return
733 /// the preheader of the completed vector loop.
734 BasicBlock *completeLoopSkeleton(Loop *L, MDNode *OrigLoopID);
735
736 /// Add additional metadata to \p To that was not present on \p Orig.
737 ///
738 /// Currently this is used to add the noalias annotations based on the
739 /// inserted memchecks. Use this for instructions that are *cloned* into the
740 /// vector loop.
741 void addNewMetadata(Instruction *To, const Instruction *Orig);
742
743 /// Add metadata from one instruction to another.
744 ///
745 /// This includes both the original MDs from \p From and additional ones (\see
746 /// addNewMetadata). Use this for *newly created* instructions in the vector
747 /// loop.
748 void addMetadata(Instruction *To, Instruction *From);
749
750 /// Similar to the previous function but it adds the metadata to a
751 /// vector of instructions.
752 void addMetadata(ArrayRef<Value *> To, Instruction *From);
753
754 /// Allow subclasses to override and print debug traces before/after vplan
755 /// execution, when trace information is requested.
756 virtual void printDebugTracesAtStart(){};
757 virtual void printDebugTracesAtEnd(){};
758
759 /// The original loop.
760 Loop *OrigLoop;
761
762 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
763 /// dynamic knowledge to simplify SCEV expressions and converts them to a
764 /// more usable form.
765 PredicatedScalarEvolution &PSE;
766
767 /// Loop Info.
768 LoopInfo *LI;
769
770 /// Dominator Tree.
771 DominatorTree *DT;
772
773 /// Alias Analysis.
774 AAResults *AA;
775
776 /// Target Library Info.
777 const TargetLibraryInfo *TLI;
778
779 /// Target Transform Info.
780 const TargetTransformInfo *TTI;
781
782 /// Assumption Cache.
783 AssumptionCache *AC;
784
785 /// Interface to emit optimization remarks.
786 OptimizationRemarkEmitter *ORE;
787
788 /// LoopVersioning. It's only set up (non-null) if memchecks were
789 /// used.
790 ///
791 /// This is currently only used to add no-alias metadata based on the
792 /// memchecks. The actually versioning is performed manually.
793 std::unique_ptr<LoopVersioning> LVer;
794
795 /// The vectorization SIMD factor to use. Each vector will have this many
796 /// vector elements.
797 ElementCount VF;
798
799 /// The vectorization unroll factor to use. Each scalar is vectorized to this
800 /// many different vector instructions.
801 unsigned UF;
802
803 /// The builder that we use
804 IRBuilder<> Builder;
805
806 // --- Vectorization state ---
807
808 /// The vector-loop preheader.
809 BasicBlock *LoopVectorPreHeader;
810
811 /// The scalar-loop preheader.
812 BasicBlock *LoopScalarPreHeader;
813
814 /// Middle Block between the vector and the scalar.
815 BasicBlock *LoopMiddleBlock;
816
817 /// The unique ExitBlock of the scalar loop if one exists. Note that
818 /// there can be multiple exiting edges reaching this block.
819 BasicBlock *LoopExitBlock;
820
821 /// The vector loop body.
822 BasicBlock *LoopVectorBody;
823
824 /// The scalar loop body.
825 BasicBlock *LoopScalarBody;
826
827 /// A list of all bypass blocks. The first block is the entry of the loop.
828 SmallVector<BasicBlock *, 4> LoopBypassBlocks;
829
830 /// The new Induction variable which was added to the new block.
831 PHINode *Induction = nullptr;
832
833 /// The induction variable of the old basic block.
834 PHINode *OldInduction = nullptr;
835
836 /// Store instructions that were predicated.
837 SmallVector<Instruction *, 4> PredicatedInstructions;
838
839 /// Trip count of the original loop.
840 Value *TripCount = nullptr;
841
842 /// Trip count of the widened loop (TripCount - TripCount % (VF*UF))
843 Value *VectorTripCount = nullptr;
844
845 /// The legality analysis.
846 LoopVectorizationLegality *Legal;
847
848 /// The profitablity analysis.
849 LoopVectorizationCostModel *Cost;
850
851 // Record whether runtime checks are added.
852 bool AddedSafetyChecks = false;
853
854 // Holds the end values for each induction variable. We save the end values
855 // so we can later fix-up the external users of the induction variables.
856 DenseMap<PHINode *, Value *> IVEndValues;
857
858 // Vector of original scalar PHIs whose corresponding widened PHIs need to be
859 // fixed up at the end of vector code generation.
860 SmallVector<PHINode *, 8> OrigPHIsToFix;
861
862 /// BFI and PSI are used to check for profile guided size optimizations.
863 BlockFrequencyInfo *BFI;
864 ProfileSummaryInfo *PSI;
865
866 // Whether this loop should be optimized for size based on profile guided size
867 // optimizatios.
868 bool OptForSizeBasedOnProfile;
869
870 /// Structure to hold information about generated runtime checks, responsible
871 /// for cleaning the checks, if vectorization turns out unprofitable.
872 GeneratedRTChecks &RTChecks;
873};
874
875class InnerLoopUnroller : public InnerLoopVectorizer {
876public:
877 InnerLoopUnroller(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
878 LoopInfo *LI, DominatorTree *DT,
879 const TargetLibraryInfo *TLI,
880 const TargetTransformInfo *TTI, AssumptionCache *AC,
881 OptimizationRemarkEmitter *ORE, unsigned UnrollFactor,
882 LoopVectorizationLegality *LVL,
883 LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
884 ProfileSummaryInfo *PSI, GeneratedRTChecks &Check)
885 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
886 ElementCount::getFixed(1), UnrollFactor, LVL, CM,
887 BFI, PSI, Check) {}
888
889private:
890 Value *getBroadcastInstrs(Value *V) override;
891 Value *getStepVector(Value *Val, int StartIdx, Value *Step,
892 Instruction::BinaryOps Opcode =
893 Instruction::BinaryOpsEnd) override;
894 Value *reverseVector(Value *Vec) override;
895};
896
897/// Encapsulate information regarding vectorization of a loop and its epilogue.
898/// This information is meant to be updated and used across two stages of
899/// epilogue vectorization.
900struct EpilogueLoopVectorizationInfo {
901 ElementCount MainLoopVF = ElementCount::getFixed(0);
902 unsigned MainLoopUF = 0;
903 ElementCount EpilogueVF = ElementCount::getFixed(0);
904 unsigned EpilogueUF = 0;
905 BasicBlock *MainLoopIterationCountCheck = nullptr;
906 BasicBlock *EpilogueIterationCountCheck = nullptr;
907 BasicBlock *SCEVSafetyCheck = nullptr;
908 BasicBlock *MemSafetyCheck = nullptr;
909 Value *TripCount = nullptr;
910 Value *VectorTripCount = nullptr;
911
912 EpilogueLoopVectorizationInfo(unsigned MVF, unsigned MUF, unsigned EVF,
913 unsigned EUF)
914 : MainLoopVF(ElementCount::getFixed(MVF)), MainLoopUF(MUF),
915 EpilogueVF(ElementCount::getFixed(EVF)), EpilogueUF(EUF) {
916 assert(EUF == 1 &&(static_cast<void> (0))
917 "A high UF for the epilogue loop is likely not beneficial.")(static_cast<void> (0));
918 }
919};
920
921/// An extension of the inner loop vectorizer that creates a skeleton for a
922/// vectorized loop that has its epilogue (residual) also vectorized.
923/// The idea is to run the vplan on a given loop twice, firstly to setup the
924/// skeleton and vectorize the main loop, and secondly to complete the skeleton
925/// from the first step and vectorize the epilogue. This is achieved by
926/// deriving two concrete strategy classes from this base class and invoking
927/// them in succession from the loop vectorizer planner.
928class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
929public:
930 InnerLoopAndEpilogueVectorizer(
931 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
932 DominatorTree *DT, const TargetLibraryInfo *TLI,
933 const TargetTransformInfo *TTI, AssumptionCache *AC,
934 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
935 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
936 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
937 GeneratedRTChecks &Checks)
938 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
939 EPI.MainLoopVF, EPI.MainLoopUF, LVL, CM, BFI, PSI,
940 Checks),
941 EPI(EPI) {}
942
943 // Override this function to handle the more complex control flow around the
944 // three loops.
945 BasicBlock *createVectorizedLoopSkeleton() final override {
946 return createEpilogueVectorizedLoopSkeleton();
947 }
948
949 /// The interface for creating a vectorized skeleton using one of two
950 /// different strategies, each corresponding to one execution of the vplan
951 /// as described above.
952 virtual BasicBlock *createEpilogueVectorizedLoopSkeleton() = 0;
953
954 /// Holds and updates state information required to vectorize the main loop
955 /// and its epilogue in two separate passes. This setup helps us avoid
956 /// regenerating and recomputing runtime safety checks. It also helps us to
957 /// shorten the iteration-count-check path length for the cases where the
958 /// iteration count of the loop is so small that the main vector loop is
959 /// completely skipped.
960 EpilogueLoopVectorizationInfo &EPI;
961};
962
963/// A specialized derived class of inner loop vectorizer that performs
964/// vectorization of *main* loops in the process of vectorizing loops and their
965/// epilogues.
966class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
967public:
968 EpilogueVectorizerMainLoop(
969 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
970 DominatorTree *DT, const TargetLibraryInfo *TLI,
971 const TargetTransformInfo *TTI, AssumptionCache *AC,
972 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
973 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
974 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
975 GeneratedRTChecks &Check)
976 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
977 EPI, LVL, CM, BFI, PSI, Check) {}
978 /// Implements the interface for creating a vectorized skeleton using the
979 /// *main loop* strategy (ie the first pass of vplan execution).
980 BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;
981
982protected:
983 /// Emits an iteration count bypass check once for the main loop (when \p
984 /// ForEpilogue is false) and once for the epilogue loop (when \p
985 /// ForEpilogue is true).
986 BasicBlock *emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass,
987 bool ForEpilogue);
988 void printDebugTracesAtStart() override;
989 void printDebugTracesAtEnd() override;
990};
991
992// A specialized derived class of inner loop vectorizer that performs
993// vectorization of *epilogue* loops in the process of vectorizing loops and
994// their epilogues.
995class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
996public:
997 EpilogueVectorizerEpilogueLoop(
998 Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
999 DominatorTree *DT, const TargetLibraryInfo *TLI,
1000 const TargetTransformInfo *TTI, AssumptionCache *AC,
1001 OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
1002 LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
1003 BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
1004 GeneratedRTChecks &Checks)
1005 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
1006 EPI, LVL, CM, BFI, PSI, Checks) {}
1007 /// Implements the interface for creating a vectorized skeleton using the
1008 /// *epilogue loop* strategy (ie the second pass of vplan execution).
1009 BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;
1010
1011protected:
1012 /// Emits an iteration count bypass check after the main vector loop has
1013 /// finished to see if there are any iterations left to execute by either
1014 /// the vector epilogue or the scalar epilogue.
1015 BasicBlock *emitMinimumVectorEpilogueIterCountCheck(Loop *L,
1016 BasicBlock *Bypass,
1017 BasicBlock *Insert);
1018 void printDebugTracesAtStart() override;
1019 void printDebugTracesAtEnd() override;
1020};
1021} // end namespace llvm
1022
1023/// Look for a meaningful debug location on the instruction or it's
1024/// operands.
1025static Instruction *getDebugLocFromInstOrOperands(Instruction *I) {
1026 if (!I)
1027 return I;
1028
1029 DebugLoc Empty;
1030 if (I->getDebugLoc() != Empty)
1031 return I;
1032
1033 for (Use &Op : I->operands()) {
1034 if (Instruction *OpInst = dyn_cast<Instruction>(Op))
1035 if (OpInst->getDebugLoc() != Empty)
1036 return OpInst;
1037 }
1038
1039 return I;
1040}
1041
1042void InnerLoopVectorizer::setDebugLocFromInst(
1043 const Value *V, Optional<IRBuilder<> *> CustomBuilder) {
1044 IRBuilder<> *B = (CustomBuilder == None) ? &Builder : *CustomBuilder;
1045 if (const Instruction *Inst = dyn_cast_or_null<Instruction>(V)) {
1046 const DILocation *DIL = Inst->getDebugLoc();
1047
1048 // When a FSDiscriminator is enabled, we don't need to add the multiply
1049 // factors to the discriminators.
1050 if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
1051 !isa<DbgInfoIntrinsic>(Inst) && !EnableFSDiscriminator) {
1052 // FIXME: For scalable vectors, assume vscale=1.
1053 auto NewDIL =
1054 DIL->cloneByMultiplyingDuplicationFactor(UF * VF.getKnownMinValue());
1055 if (NewDIL)
1056 B->SetCurrentDebugLocation(NewDIL.getValue());
1057 else
1058 LLVM_DEBUG(dbgs()do { } while (false)
1059 << "Failed to create new discriminator: "do { } while (false)
1060 << DIL->getFilename() << " Line: " << DIL->getLine())do { } while (false);
1061 } else
1062 B->SetCurrentDebugLocation(DIL);
1063 } else
1064 B->SetCurrentDebugLocation(DebugLoc());
1065}
1066
1067/// Write a \p DebugMsg about vectorization to the debug output stream. If \p I
1068/// is passed, the message relates to that particular instruction.
1069#ifndef NDEBUG1
1070static void debugVectorizationMessage(const StringRef Prefix,
1071 const StringRef DebugMsg,
1072 Instruction *I) {
1073 dbgs() << "LV: " << Prefix << DebugMsg;
1074 if (I != nullptr)
1075 dbgs() << " " << *I;
1076 else
1077 dbgs() << '.';
1078 dbgs() << '\n';
1079}
1080#endif
1081
1082/// Create an analysis remark that explains why vectorization failed
1083///
1084/// \p PassName is the name of the pass (e.g. can be AlwaysPrint). \p
1085/// RemarkName is the identifier for the remark. If \p I is passed it is an
1086/// instruction that prevents vectorization. Otherwise \p TheLoop is used for
1087/// the location of the remark. \return the remark object that can be
1088/// streamed to.
1089static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
1090 StringRef RemarkName, Loop *TheLoop, Instruction *I) {
1091 Value *CodeRegion = TheLoop->getHeader();
1092 DebugLoc DL = TheLoop->getStartLoc();
1093
1094 if (I) {
1095 CodeRegion = I->getParent();
1096 // If there is no debug location attached to the instruction, revert back to
1097 // using the loop's.
1098 if (I->getDebugLoc())
1099 DL = I->getDebugLoc();
1100 }
1101
1102 return OptimizationRemarkAnalysis(PassName, RemarkName, DL, CodeRegion);
1103}
1104
1105/// Return a value for Step multiplied by VF.
1106static Value *createStepForVF(IRBuilder<> &B, Constant *Step, ElementCount VF) {
1107 assert(isa<ConstantInt>(Step) && "Expected an integer step")(static_cast<void> (0));
1108 Constant *StepVal = ConstantInt::get(
1109 Step->getType(),
1110 cast<ConstantInt>(Step)->getSExtValue() * VF.getKnownMinValue());
1111 return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal;
1112}
1113
1114namespace llvm {
1115
1116/// Return the runtime value for VF.
1117Value *getRuntimeVF(IRBuilder<> &B, Type *Ty, ElementCount VF) {
1118 Constant *EC = ConstantInt::get(Ty, VF.getKnownMinValue());
1119 return VF.isScalable() ? B.CreateVScale(EC) : EC;
1120}
1121
1122void reportVectorizationFailure(const StringRef DebugMsg,
1123 const StringRef OREMsg, const StringRef ORETag,
1124 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
1125 Instruction *I) {
1126 LLVM_DEBUG(debugVectorizationMessage("Not vectorizing: ", DebugMsg, I))do { } while (false);
1127 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
1128 ORE->emit(
1129 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
1130 << "loop not vectorized: " << OREMsg);
1131}
1132
1133void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
1134 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
1135 Instruction *I) {
1136 LLVM_DEBUG(debugVectorizationMessage("", Msg, I))do { } while (false);
1137 LoopVectorizeHints Hints(TheLoop, true /* doesn't matter */, *ORE);
1138 ORE->emit(
1139 createLVAnalysis(Hints.vectorizeAnalysisPassName(), ORETag, TheLoop, I)
1140 << Msg);
1141}
1142
1143} // end namespace llvm
1144
1145#ifndef NDEBUG1
1146/// \return string containing a file name and a line # for the given loop.
1147static std::string getDebugLocString(const Loop *L) {
1148 std::string Result;
1149 if (L) {
1150 raw_string_ostream OS(Result);
1151 if (const DebugLoc LoopDbgLoc = L->getStartLoc())
1152 LoopDbgLoc.print(OS);
1153 else
1154 // Just print the module name.
1155 OS << L->getHeader()->getParent()->getParent()->getModuleIdentifier();
1156 OS.flush();
1157 }
1158 return Result;
1159}
1160#endif
1161
1162void InnerLoopVectorizer::addNewMetadata(Instruction *To,
1163 const Instruction *Orig) {
1164 // If the loop was versioned with memchecks, add the corresponding no-alias
1165 // metadata.
1166 if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
1167 LVer->annotateInstWithNoAlias(To, Orig);
1168}
1169
1170void InnerLoopVectorizer::addMetadata(Instruction *To,
1171 Instruction *From) {
1172 propagateMetadata(To, From);
1173 addNewMetadata(To, From);
1174}
1175
1176void InnerLoopVectorizer::addMetadata(ArrayRef<Value *> To,
1177 Instruction *From) {
1178 for (Value *V : To) {
1179 if (Instruction *I = dyn_cast<Instruction>(V))
1180 addMetadata(I, From);
1181 }
1182}
1183
1184namespace llvm {
1185
1186// Loop vectorization cost-model hints how the scalar epilogue loop should be
1187// lowered.
1188enum ScalarEpilogueLowering {
1189
1190 // The default: allowing scalar epilogues.
1191 CM_ScalarEpilogueAllowed,
1192
1193 // Vectorization with OptForSize: don't allow epilogues.
1194 CM_ScalarEpilogueNotAllowedOptSize,
1195
1196 // A special case of vectorisation with OptForSize: loops with a very small
1197 // trip count are considered for vectorization under OptForSize, thereby
1198 // making sure the cost of their loop body is dominant, free of runtime
1199 // guards and scalar iteration overheads.
1200 CM_ScalarEpilogueNotAllowedLowTripLoop,
1201
1202 // Loop hint predicate indicating an epilogue is undesired.
1203 CM_ScalarEpilogueNotNeededUsePredicate,
1204
1205 // Directive indicating we must either tail fold or not vectorize
1206 CM_ScalarEpilogueNotAllowedUsePredicate
1207};
1208
1209/// ElementCountComparator creates a total ordering for ElementCount
1210/// for the purposes of using it in a set structure.
1211struct ElementCountComparator {
1212 bool operator()(const ElementCount &LHS, const ElementCount &RHS) const {
1213 return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) <
1214 std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue());
1215 }
1216};
1217using ElementCountSet = SmallSet<ElementCount, 16, ElementCountComparator>;
1218
1219/// LoopVectorizationCostModel - estimates the expected speedups due to
1220/// vectorization.
1221/// In many cases vectorization is not profitable. This can happen because of
1222/// a number of reasons. In this class we mainly attempt to predict the
1223/// expected speedup/slowdowns due to the supported instruction set. We use the
1224/// TargetTransformInfo to query the different backends for the cost of
1225/// different operations.
1226class LoopVectorizationCostModel {
1227public:
1228 LoopVectorizationCostModel(ScalarEpilogueLowering SEL, Loop *L,
1229 PredicatedScalarEvolution &PSE, LoopInfo *LI,
1230 LoopVectorizationLegality *Legal,
1231 const TargetTransformInfo &TTI,
1232 const TargetLibraryInfo *TLI, DemandedBits *DB,
1233 AssumptionCache *AC,
1234 OptimizationRemarkEmitter *ORE, const Function *F,
1235 const LoopVectorizeHints *Hints,
1236 InterleavedAccessInfo &IAI)
1237 : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal),
1238 TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F),
1239 Hints(Hints), InterleaveInfo(IAI) {}
1240
1241 /// \return An upper bound for the vectorization factors (both fixed and
1242 /// scalable). If the factors are 0, vectorization and interleaving should be
1243 /// avoided up front.
1244 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
1245
1246 /// \return True if runtime checks are required for vectorization, and false
1247 /// otherwise.
1248 bool runtimeChecksRequired();
1249
1250 /// \return The most profitable vectorization factor and the cost of that VF.
1251 /// This method checks every VF in \p CandidateVFs. If UserVF is not ZERO
1252 /// then this vectorization factor will be selected if vectorization is
1253 /// possible.
1254 VectorizationFactor
1255 selectVectorizationFactor(const ElementCountSet &CandidateVFs);
1256
1257 VectorizationFactor
1258 selectEpilogueVectorizationFactor(const ElementCount MaxVF,
1259 const LoopVectorizationPlanner &LVP);
1260
1261 /// Setup cost-based decisions for user vectorization factor.
1262 /// \return true if the UserVF is a feasible VF to be chosen.
1263 bool selectUserVectorizationFactor(ElementCount UserVF) {
1264 collectUniformsAndScalars(UserVF);
1
Calling 'LoopVectorizationCostModel::collectUniformsAndScalars'
1265 collectInstsToScalarize(UserVF);
1266 return expectedCost(UserVF).first.isValid();
1267 }
1268
1269 /// \return The size (in bits) of the smallest and widest types in the code
1270 /// that needs to be vectorized. We ignore values that remain scalar such as
1271 /// 64 bit loop indices.
1272 std::pair<unsigned, unsigned> getSmallestAndWidestTypes();
1273
1274 /// \return The desired interleave count.
1275 /// If interleave count has been specified by metadata it will be returned.
1276 /// Otherwise, the interleave count is computed and returned. VF and LoopCost
1277 /// are the selected vectorization factor and the cost of the selected VF.
1278 unsigned selectInterleaveCount(ElementCount VF, unsigned LoopCost);
1279
1280 /// Memory access instruction may be vectorized in more than one way.
1281 /// Form of instruction after vectorization depends on cost.
1282 /// This function takes cost-based decisions for Load/Store instructions
1283 /// and collects them in a map. This decisions map is used for building
1284 /// the lists of loop-uniform and loop-scalar instructions.
1285 /// The calculated cost is saved with widening decision in order to
1286 /// avoid redundant calculations.
1287 void setCostBasedWideningDecision(ElementCount VF);
1288
1289 /// A struct that represents some properties of the register usage
1290 /// of a loop.
1291 struct RegisterUsage {
1292 /// Holds the number of loop invariant values that are used in the loop.
1293 /// The key is ClassID of target-provided register class.
1294 SmallMapVector<unsigned, unsigned, 4> LoopInvariantRegs;
1295 /// Holds the maximum number of concurrent live intervals in the loop.
1296 /// The key is ClassID of target-provided register class.
1297 SmallMapVector<unsigned, unsigned, 4> MaxLocalUsers;
1298 };
1299
1300 /// \return Returns information about the register usages of the loop for the
1301 /// given vectorization factors.
1302 SmallVector<RegisterUsage, 8>
1303 calculateRegisterUsage(ArrayRef<ElementCount> VFs);
1304
1305 /// Collect values we want to ignore in the cost model.
1306 void collectValuesToIgnore();
1307
1308 /// Collect all element types in the loop for which widening is needed.
1309 void collectElementTypesForWidening();
1310
1311 /// Split reductions into those that happen in the loop, and those that happen
1312 /// outside. In loop reductions are collected into InLoopReductionChains.
1313 void collectInLoopReductions();
1314
1315 /// Returns true if we should use strict in-order reductions for the given
1316 /// RdxDesc. This is true if the -enable-strict-reductions flag is passed,
1317 /// the IsOrdered flag of RdxDesc is set and we do not allow reordering
1318 /// of FP operations.
1319 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) {
1320 return !Hints->allowReordering() && RdxDesc.isOrdered();
1321 }
1322
1323 /// \returns The smallest bitwidth each instruction can be represented with.
1324 /// The vector equivalents of these instructions should be truncated to this
1325 /// type.
1326 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
1327 return MinBWs;
1328 }
1329
1330 /// \returns True if it is more profitable to scalarize instruction \p I for
1331 /// vectorization factor \p VF.
1332 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
1333 assert(VF.isVector() &&(static_cast<void> (0))
1334 "Profitable to scalarize relevant only for VF > 1.")(static_cast<void> (0));
1335
1336 // Cost model is not run in the VPlan-native path - return conservative
1337 // result until this changes.
1338 if (EnableVPlanNativePath)
1339 return false;
1340
1341 auto Scalars = InstsToScalarize.find(VF);
1342 assert(Scalars != InstsToScalarize.end() &&(static_cast<void> (0))
1343 "VF not yet analyzed for scalarization profitability")(static_cast<void> (0));
1344 return Scalars->second.find(I) != Scalars->second.end();
1345 }
1346
1347 /// Returns true if \p I is known to be uniform after vectorization.
1348 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
1349 if (VF.isScalar())
1350 return true;
1351
1352 // Cost model is not run in the VPlan-native path - return conservative
1353 // result until this changes.
1354 if (EnableVPlanNativePath)
1355 return false;
1356
1357 auto UniformsPerVF = Uniforms.find(VF);
1358 assert(UniformsPerVF != Uniforms.end() &&(static_cast<void> (0))
1359 "VF not yet analyzed for uniformity")(static_cast<void> (0));
1360 return UniformsPerVF->second.count(I);
1361 }
1362
1363 /// Returns true if \p I is known to be scalar after vectorization.
1364 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
1365 if (VF.isScalar())
1366 return true;
1367
1368 // Cost model is not run in the VPlan-native path - return conservative
1369 // result until this changes.
1370 if (EnableVPlanNativePath)
1371 return false;
1372
1373 auto ScalarsPerVF = Scalars.find(VF);
1374 assert(ScalarsPerVF != Scalars.end() &&(static_cast<void> (0))
1375 "Scalar values are not calculated for VF")(static_cast<void> (0));
1376 return ScalarsPerVF->second.count(I);
1377 }
1378
1379 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
1380 /// for vectorization factor \p VF.
1381 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
1382 return VF.isVector() && MinBWs.find(I) != MinBWs.end() &&
1383 !isProfitableToScalarize(I, VF) &&
1384 !isScalarAfterVectorization(I, VF);
1385 }
1386
1387 /// Decision that was taken during cost calculation for memory instruction.
1388 enum InstWidening {
1389 CM_Unknown,
1390 CM_Widen, // For consecutive accesses with stride +1.
1391 CM_Widen_Reverse, // For consecutive accesses with stride -1.
1392 CM_Interleave,
1393 CM_GatherScatter,
1394 CM_Scalarize
1395 };
1396
1397 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1398 /// instruction \p I and vector width \p VF.
1399 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
1400 InstructionCost Cost) {
1401 assert(VF.isVector() && "Expected VF >=2")(static_cast<void> (0));
1402 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1403 }
1404
1405 /// Save vectorization decision \p W and \p Cost taken by the cost model for
1406 /// interleaving group \p Grp and vector width \p VF.
1407 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
1408 ElementCount VF, InstWidening W,
1409 InstructionCost Cost) {
1410 assert(VF.isVector() && "Expected VF >=2")(static_cast<void> (0));
1411 /// Broadcast this decicion to all instructions inside the group.
1412 /// But the cost will be assigned to one instruction only.
1413 for (unsigned i = 0; i < Grp->getFactor(); ++i) {
1414 if (auto *I = Grp->getMember(i)) {
1415 if (Grp->getInsertPos() == I)
1416 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, Cost);
1417 else
1418 WideningDecisions[std::make_pair(I, VF)] = std::make_pair(W, 0);
1419 }
1420 }
1421 }
1422
1423 /// Return the cost model decision for the given instruction \p I and vector
1424 /// width \p VF. Return CM_Unknown if this instruction did not pass
1425 /// through the cost modeling.
1426 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
1427 assert(VF.isVector() && "Expected VF to be a vector VF")(static_cast<void> (0));
1428 // Cost model is not run in the VPlan-native path - return conservative
1429 // result until this changes.
1430 if (EnableVPlanNativePath)
1431 return CM_GatherScatter;
1432
1433 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1434 auto Itr = WideningDecisions.find(InstOnVF);
1435 if (Itr == WideningDecisions.end())
1436 return CM_Unknown;
1437 return Itr->second.first;
1438 }
1439
1440 /// Return the vectorization cost for the given instruction \p I and vector
1441 /// width \p VF.
1442 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
1443 assert(VF.isVector() && "Expected VF >=2")(static_cast<void> (0));
1444 std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
1445 assert(WideningDecisions.find(InstOnVF) != WideningDecisions.end() &&(static_cast<void> (0))
1446 "The cost is not calculated")(static_cast<void> (0));
1447 return WideningDecisions[InstOnVF].second;
1448 }
1449
1450 /// Return True if instruction \p I is an optimizable truncate whose operand
1451 /// is an induction variable. Such a truncate will be removed by adding a new
1452 /// induction variable with the destination type.
1453 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
1454 // If the instruction is not a truncate, return false.
1455 auto *Trunc = dyn_cast<TruncInst>(I);
1456 if (!Trunc)
1457 return false;
1458
1459 // Get the source and destination types of the truncate.
1460 Type *SrcTy = ToVectorTy(cast<CastInst>(I)->getSrcTy(), VF);
1461 Type *DestTy = ToVectorTy(cast<CastInst>(I)->getDestTy(), VF);
1462
1463 // If the truncate is free for the given types, return false. Replacing a
1464 // free truncate with an induction variable would add an induction variable
1465 // update instruction to each iteration of the loop. We exclude from this
1466 // check the primary induction variable since it will need an update
1467 // instruction regardless.
1468 Value *Op = Trunc->getOperand(0);
1469 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(SrcTy, DestTy))
1470 return false;
1471
1472 // If the truncated value is not an induction variable, return false.
1473 return Legal->isInductionPhi(Op);
1474 }
1475
1476 /// Collects the instructions to scalarize for each predicated instruction in
1477 /// the loop.
1478 void collectInstsToScalarize(ElementCount VF);
1479
1480 /// Collect Uniform and Scalar values for the given \p VF.
1481 /// The sets depend on CM decision for Load/Store instructions
1482 /// that may be vectorized as interleave, gather-scatter or scalarized.
1483 void collectUniformsAndScalars(ElementCount VF) {
1484 // Do the analysis once.
1485 if (VF.isScalar() || Uniforms.find(VF) != Uniforms.end())
2
Taking false branch
1486 return;
1487 setCostBasedWideningDecision(VF);
3
Calling 'LoopVectorizationCostModel::setCostBasedWideningDecision'
1488 collectLoopUniforms(VF);
1489 collectLoopScalars(VF);
1490 }
1491
1492 /// Returns true if the target machine supports masked store operation
1493 /// for the given \p DataType and kind of access to \p Ptr.
1494 bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const {
1495 return Legal->isConsecutivePtr(Ptr) &&
1496 TTI.isLegalMaskedStore(DataType, Alignment);
1497 }
1498
1499 /// Returns true if the target machine supports masked load operation
1500 /// for the given \p DataType and kind of access to \p Ptr.
1501 bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const {
1502 return Legal->isConsecutivePtr(Ptr) &&
1503 TTI.isLegalMaskedLoad(DataType, Alignment);
1504 }
1505
1506 /// Returns true if the target machine can represent \p V as a masked gather
1507 /// or scatter operation.
1508 bool isLegalGatherOrScatter(Value *V) {
1509 bool LI = isa<LoadInst>(V);
1510 bool SI = isa<StoreInst>(V);
1511 if (!LI && !SI)
1512 return false;
1513 auto *Ty = getLoadStoreType(V);
1514 Align Align = getLoadStoreAlignment(V);
1515 return (LI && TTI.isLegalMaskedGather(Ty, Align)) ||
1516 (SI && TTI.isLegalMaskedScatter(Ty, Align));
1517 }
1518
1519 /// Returns true if the target machine supports all of the reduction
1520 /// variables found for the given VF.
1521 bool canVectorizeReductions(ElementCount VF) const {
1522 return (all_of(Legal->getReductionVars(), [&](auto &Reduction) -> bool {
1523 const RecurrenceDescriptor &RdxDesc = Reduction.second;
1524 return TTI.isLegalToVectorizeReduction(RdxDesc, VF);
1525 }));
1526 }
1527
1528 /// Returns true if \p I is an instruction that will be scalarized with
1529 /// predication. Such instructions include conditional stores and
1530 /// instructions that may divide by zero.
1531 /// If a non-zero VF has been calculated, we check if I will be scalarized
1532 /// predication for that VF.
1533 bool isScalarWithPredication(Instruction *I) const;
1534
1535 // Returns true if \p I is an instruction that will be predicated either
1536 // through scalar predication or masked load/store or masked gather/scatter.
1537 // Superset of instructions that return true for isScalarWithPredication.
1538 bool isPredicatedInst(Instruction *I) {
1539 if (!blockNeedsPredication(I->getParent()))
1540 return false;
1541 // Loads and stores that need some form of masked operation are predicated
1542 // instructions.
1543 if (isa<LoadInst>(I) || isa<StoreInst>(I))
1544 return Legal->isMaskRequired(I);
1545 return isScalarWithPredication(I);
1546 }
1547
1548 /// Returns true if \p I is a memory instruction with consecutive memory
1549 /// access that can be widened.
1550 bool
1551 memoryInstructionCanBeWidened(Instruction *I,
1552 ElementCount VF = ElementCount::getFixed(1));
1553
1554 /// Returns true if \p I is a memory instruction in an interleaved-group
1555 /// of memory accesses that can be vectorized with wide vector loads/stores
1556 /// and shuffles.
1557 bool
1558 interleavedAccessCanBeWidened(Instruction *I,
1559 ElementCount VF = ElementCount::getFixed(1));
1560
1561 /// Check if \p Instr belongs to any interleaved access group.
1562 bool isAccessInterleaved(Instruction *Instr) {
1563 return InterleaveInfo.isInterleaved(Instr);
1564 }
1565
1566 /// Get the interleaved access group that \p Instr belongs to.
1567 const InterleaveGroup<Instruction> *
1568 getInterleavedAccessGroup(Instruction *Instr) {
1569 return InterleaveInfo.getInterleaveGroup(Instr);
1570 }
1571
1572 /// Returns true if we're required to use a scalar epilogue for at least
1573 /// the final iteration of the original loop.
1574 bool requiresScalarEpilogue(ElementCount VF) const {
1575 if (!isScalarEpilogueAllowed())
1576 return false;
1577 // If we might exit from anywhere but the latch, must run the exiting
1578 // iteration in scalar form.
1579 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch())
1580 return true;
1581 return VF.isVector() && InterleaveInfo.requiresScalarEpilogue();
1582 }
1583
1584 /// Returns true if a scalar epilogue is not allowed due to optsize or a
1585 /// loop hint annotation.
1586 bool isScalarEpilogueAllowed() const {
1587 return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
1588 }
1589
1590 /// Returns true if all loop blocks should be masked to fold tail loop.
1591 bool foldTailByMasking() const { return FoldTailByMasking; }
1592
1593 bool blockNeedsPredication(BasicBlock *BB) const {
1594 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
17
Assuming the condition is false
18
Returning value, which participates in a condition later
1595 }
1596
1597 /// A SmallMapVector to store the InLoop reduction op chains, mapping phi
1598 /// nodes to the chain of instructions representing the reductions. Uses a
1599 /// MapVector to ensure deterministic iteration order.
1600 using ReductionChainMap =
1601 SmallMapVector<PHINode *, SmallVector<Instruction *, 4>, 4>;
1602
1603 /// Return the chain of instructions representing an inloop reduction.
1604 const ReductionChainMap &getInLoopReductionChains() const {
1605 return InLoopReductionChains;
1606 }
1607
1608 /// Returns true if the Phi is part of an inloop reduction.
1609 bool isInLoopReduction(PHINode *Phi) const {
1610 return InLoopReductionChains.count(Phi);
1611 }
1612
1613 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1614 /// with factor VF. Return the cost of the instruction, including
1615 /// scalarization overhead if it's needed.
1616 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1617
1618 /// Estimate cost of a call instruction CI if it were vectorized with factor
1619 /// VF. Return the cost of the instruction, including scalarization overhead
1620 /// if it's needed. The flag NeedToScalarize shows if the call needs to be
1621 /// scalarized -
1622 /// i.e. either vector version isn't available, or is too expensive.
1623 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF,
1624 bool &NeedToScalarize) const;
1625
1626 /// Returns true if the per-lane cost of VectorizationFactor A is lower than
1627 /// that of B.
1628 bool isMoreProfitable(const VectorizationFactor &A,
1629 const VectorizationFactor &B) const;
1630
1631 /// Invalidates decisions already taken by the cost model.
1632 void invalidateCostModelingDecisions() {
1633 WideningDecisions.clear();
1634 Uniforms.clear();
1635 Scalars.clear();
1636 }
1637
1638private:
1639 unsigned NumPredStores = 0;
1640
1641 /// \return An upper bound for the vectorization factors for both
1642 /// fixed and scalable vectorization, where the minimum-known number of
1643 /// elements is a power-of-2 larger than zero. If scalable vectorization is
1644 /// disabled or unsupported, then the scalable part will be equal to
1645 /// ElementCount::getScalable(0).
1646 FixedScalableVFPair computeFeasibleMaxVF(unsigned ConstTripCount,
1647 ElementCount UserVF);
1648
1649 /// \return the maximized element count based on the targets vector
1650 /// registers and the loop trip-count, but limited to a maximum safe VF.
1651 /// This is a helper function of computeFeasibleMaxVF.
1652 /// FIXME: MaxSafeVF is currently passed by reference to avoid some obscure
1653 /// issue that occurred on one of the buildbots which cannot be reproduced
1654 /// without having access to the properietary compiler (see comments on
1655 /// D98509). The issue is currently under investigation and this workaround
1656 /// will be removed as soon as possible.
1657 ElementCount getMaximizedVFForTarget(unsigned ConstTripCount,
1658 unsigned SmallestType,
1659 unsigned WidestType,
1660 const ElementCount &MaxSafeVF);
1661
1662 /// \return the maximum legal scalable VF, based on the safe max number
1663 /// of elements.
1664 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
1665
1666 /// The vectorization cost is a combination of the cost itself and a boolean
1667 /// indicating whether any of the contributing operations will actually
1668 /// operate on vector values after type legalization in the backend. If this
1669 /// latter value is false, then all operations will be scalarized (i.e. no
1670 /// vectorization has actually taken place).
1671 using VectorizationCostTy = std::pair<InstructionCost, bool>;
1672
1673 /// Returns the expected execution cost. The unit of the cost does
1674 /// not matter because we use the 'cost' units to compare different
1675 /// vector widths. The cost that is returned is *not* normalized by
1676 /// the factor width. If \p Invalid is not nullptr, this function
1677 /// will add a pair(Instruction*, ElementCount) to \p Invalid for
1678 /// each instruction that has an Invalid cost for the given VF.
1679 using InstructionVFPair = std::pair<Instruction *, ElementCount>;
1680 VectorizationCostTy
1681 expectedCost(ElementCount VF,
1682 SmallVectorImpl<InstructionVFPair> *Invalid = nullptr);
1683
1684 /// Returns the execution time cost of an instruction for a given vector
1685 /// width. Vector width of one means scalar.
1686 VectorizationCostTy getInstructionCost(Instruction *I, ElementCount VF);
1687
1688 /// The cost-computation logic from getInstructionCost which provides
1689 /// the vector type as an output parameter.
1690 InstructionCost getInstructionCost(Instruction *I, ElementCount VF,
1691 Type *&VectorTy);
1692
1693 /// Return the cost of instructions in an inloop reduction pattern, if I is
1694 /// part of that pattern.
1695 Optional<InstructionCost>
1696 getReductionPatternCost(Instruction *I, ElementCount VF, Type *VectorTy,
1697 TTI::TargetCostKind CostKind);
1698
1699 /// Calculate vectorization cost of memory instruction \p I.
1700 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1701
1702 /// The cost computation for scalarized memory instruction.
1703 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1704
1705 /// The cost computation for interleaving group of memory instructions.
1706 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1707
1708 /// The cost computation for Gather/Scatter instruction.
1709 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1710
1711 /// The cost computation for widening instruction \p I with consecutive
1712 /// memory access.
1713 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
1714
1715 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1716 /// Load: scalar load + broadcast.
1717 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1718 /// element)
1719 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1720
1721 /// Estimate the overhead of scalarizing an instruction. This is a
1722 /// convenience wrapper for the type-based getScalarizationOverhead API.
1723 InstructionCost getScalarizationOverhead(Instruction *I,
1724 ElementCount VF) const;
1725
1726 /// Returns whether the instruction is a load or store and will be a emitted
1727 /// as a vector operation.
1728 bool isConsecutiveLoadOrStore(Instruction *I);
1729
1730 /// Returns true if an artificially high cost for emulated masked memrefs
1731 /// should be used.
1732 bool useEmulatedMaskMemRefHack(Instruction *I);
1733
1734 /// Map of scalar integer values to the smallest bitwidth they can be legally
1735 /// represented as. The vector equivalents of these values should be truncated
1736 /// to this type.
1737 MapVector<Instruction *, uint64_t> MinBWs;
1738
1739 /// A type representing the costs for instructions if they were to be
1740 /// scalarized rather than vectorized. The entries are Instruction-Cost
1741 /// pairs.
1742 using ScalarCostsTy = DenseMap<Instruction *, InstructionCost>;
1743
1744 /// A set containing all BasicBlocks that are known to present after
1745 /// vectorization as a predicated block.
1746 SmallPtrSet<BasicBlock *, 4> PredicatedBBsAfterVectorization;
1747
1748 /// Records whether it is allowed to have the original scalar loop execute at
1749 /// least once. This may be needed as a fallback loop in case runtime
1750 /// aliasing/dependence checks fail, or to handle the tail/remainder
1751 /// iterations when the trip count is unknown or doesn't divide by the VF,
1752 /// or as a peel-loop to handle gaps in interleave-groups.
1753 /// Under optsize and when the trip count is very small we don't allow any
1754 /// iterations to execute in the scalar loop.
1755 ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
1756
1757 /// All blocks of loop are to be masked to fold tail of scalar iterations.
1758 bool FoldTailByMasking = false;
1759
1760 /// A map holding scalar costs for different vectorization factors. The
1761 /// presence of a cost for an instruction in the mapping indicates that the
1762 /// instruction will be scalarized when vectorizing with the associated
1763 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1764 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
1765
1766 /// Holds the instructions known to be uniform after vectorization.
1767 /// The data is collected per VF.
1768 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1769
1770 /// Holds the instructions known to be scalar after vectorization.
1771 /// The data is collected per VF.
1772 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1773
1774 /// Holds the instructions (address computations) that are forced to be
1775 /// scalarized.
1776 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1777
1778 /// PHINodes of the reductions that should be expanded in-loop along with
1779 /// their associated chains of reduction operations, in program order from top
1780 /// (PHI) to bottom
1781 ReductionChainMap InLoopReductionChains;
1782
1783 /// A Map of inloop reduction operations and their immediate chain operand.
1784 /// FIXME: This can be removed once reductions can be costed correctly in
1785 /// vplan. This was added to allow quick lookup to the inloop operations,
1786 /// without having to loop through InLoopReductionChains.
1787 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
1788
1789 /// Returns the expected difference in cost from scalarizing the expression
1790 /// feeding a predicated instruction \p PredInst. The instructions to
1791 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1792 /// non-negative return value implies the expression will be scalarized.
1793 /// Currently, only single-use chains are considered for scalarization.
1794 int computePredInstDiscount(Instruction *PredInst, ScalarCostsTy &ScalarCosts,
1795 ElementCount VF);
1796
1797 /// Collect the instructions that are uniform after vectorization. An
1798 /// instruction is uniform if we represent it with a single scalar value in
1799 /// the vectorized loop corresponding to each vector iteration. Examples of
1800 /// uniform instructions include pointer operands of consecutive or
1801 /// interleaved memory accesses. Note that although uniformity implies an
1802 /// instruction will be scalar, the reverse is not true. In general, a
1803 /// scalarized instruction will be represented by VF scalar values in the
1804 /// vectorized loop, each corresponding to an iteration of the original
1805 /// scalar loop.
1806 void collectLoopUniforms(ElementCount VF);
1807
1808 /// Collect the instructions that are scalar after vectorization. An
1809 /// instruction is scalar if it is known to be uniform or will be scalarized
1810 /// during vectorization. Non-uniform scalarized instructions will be
1811 /// represented by VF values in the vectorized loop, each corresponding to an
1812 /// iteration of the original scalar loop.
1813 void collectLoopScalars(ElementCount VF);
1814
1815 /// Keeps cost model vectorization decision and cost for instructions.
1816 /// Right now it is used for memory instructions only.
1817 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1818 std::pair<InstWidening, InstructionCost>>;
1819
1820 DecisionList WideningDecisions;
1821
1822 /// Returns true if \p V is expected to be vectorized and it needs to be
1823 /// extracted.
1824 bool needsExtract(Value *V, ElementCount VF) const {
1825 Instruction *I = dyn_cast<Instruction>(V);
1826 if (VF.isScalar() || !I || !TheLoop->contains(I) ||
1827 TheLoop->isLoopInvariant(I))
1828 return false;
1829
1830 // Assume we can vectorize V (and hence we need extraction) if the
1831 // scalars are not computed yet. This can happen, because it is called
1832 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1833 // the scalars are collected. That should be a safe assumption in most
1834 // cases, because we check if the operands have vectorizable types
1835 // beforehand in LoopVectorizationLegality.
1836 return Scalars.find(VF) == Scalars.end() ||
1837 !isScalarAfterVectorization(I, VF);
1838 };
1839
1840 /// Returns a range containing only operands needing to be extracted.
1841 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1842 ElementCount VF) const {
1843 return SmallVector<Value *, 4>(make_filter_range(
1844 Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
1845 }
1846
1847 /// Determines if we have the infrastructure to vectorize loop \p L and its
1848 /// epilogue, assuming the main loop is vectorized by \p VF.
1849 bool isCandidateForEpilogueVectorization(const Loop &L,
1850 const ElementCount VF) const;
1851
1852 /// Returns true if epilogue vectorization is considered profitable, and
1853 /// false otherwise.
1854 /// \p VF is the vectorization factor chosen for the original loop.
1855 bool isEpilogueVectorizationProfitable(const ElementCount VF) const;
1856
1857public:
1858 /// The loop that we evaluate.
1859 Loop *TheLoop;
1860
1861 /// Predicated scalar evolution analysis.
1862 PredicatedScalarEvolution &PSE;
1863
1864 /// Loop Info analysis.
1865 LoopInfo *LI;
1866
1867 /// Vectorization legality.
1868 LoopVectorizationLegality *Legal;
1869
1870 /// Vector target information.
1871 const TargetTransformInfo &TTI;
1872
1873 /// Target Library Info.
1874 const TargetLibraryInfo *TLI;
1875
1876 /// Demanded bits analysis.
1877 DemandedBits *DB;
1878
1879 /// Assumption cache.
1880 AssumptionCache *AC;
1881
1882 /// Interface to emit optimization remarks.
1883 OptimizationRemarkEmitter *ORE;
1884
1885 const Function *TheFunction;
1886
1887 /// Loop Vectorize Hint.
1888 const LoopVectorizeHints *Hints;
1889
1890 /// The interleave access information contains groups of interleaved accesses
1891 /// with the same stride and close to each other.
1892 InterleavedAccessInfo &InterleaveInfo;
1893
1894 /// Values to ignore in the cost model.
1895 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1896
1897 /// Values to ignore in the cost model when VF > 1.
1898 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1899
1900 /// All element types found in the loop.
1901 SmallPtrSet<Type *, 16> ElementTypesInLoop;
1902
1903 /// Profitable vector factors.
1904 SmallVector<VectorizationFactor, 8> ProfitableVFs;
1905};
1906} // end namespace llvm
1907
1908/// Helper struct to manage generating runtime checks for vectorization.
1909///
1910/// The runtime checks are created up-front in temporary blocks to allow better
1911/// estimating the cost and un-linked from the existing IR. After deciding to
1912/// vectorize, the checks are moved back. If deciding not to vectorize, the
1913/// temporary blocks are completely removed.
1914class GeneratedRTChecks {
1915 /// Basic block which contains the generated SCEV checks, if any.
1916 BasicBlock *SCEVCheckBlock = nullptr;
1917
1918 /// The value representing the result of the generated SCEV checks. If it is
1919 /// nullptr, either no SCEV checks have been generated or they have been used.
1920 Value *SCEVCheckCond = nullptr;
1921
1922 /// Basic block which contains the generated memory runtime checks, if any.
1923 BasicBlock *MemCheckBlock = nullptr;
1924
1925 /// The value representing the result of the generated memory runtime checks.
1926 /// If it is nullptr, either no memory runtime checks have been generated or
1927 /// they have been used.
1928 Instruction *MemRuntimeCheckCond = nullptr;
1929
1930 DominatorTree *DT;
1931 LoopInfo *LI;
1932
1933 SCEVExpander SCEVExp;
1934 SCEVExpander MemCheckExp;
1935
1936public:
1937 GeneratedRTChecks(ScalarEvolution &SE, DominatorTree *DT, LoopInfo *LI,
1938 const DataLayout &DL)
1939 : DT(DT), LI(LI), SCEVExp(SE, DL, "scev.check"),
1940 MemCheckExp(SE, DL, "scev.check") {}
1941
1942 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1943 /// accurately estimate the cost of the runtime checks. The blocks are
1944 /// un-linked from the IR and is added back during vector code generation. If
1945 /// there is no vector code generation, the check blocks are removed
1946 /// completely.
1947 void Create(Loop *L, const LoopAccessInfo &LAI,
1948 const SCEVUnionPredicate &UnionPred) {
1949
1950 BasicBlock *LoopHeader = L->getHeader();
1951 BasicBlock *Preheader = L->getLoopPreheader();
1952
1953 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1954 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1955 // may be used by SCEVExpander. The blocks will be un-linked from their
1956 // predecessors and removed from LI & DT at the end of the function.
1957 if (!UnionPred.isAlwaysTrue()) {
1958 SCEVCheckBlock = SplitBlock(Preheader, Preheader->getTerminator(), DT, LI,
1959 nullptr, "vector.scevcheck");
1960
1961 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1962 &UnionPred, SCEVCheckBlock->getTerminator());
1963 }
1964
1965 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1966 if (RtPtrChecking.Need) {
1967 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1968 MemCheckBlock = SplitBlock(Pred, Pred->getTerminator(), DT, LI, nullptr,
1969 "vector.memcheck");
1970
1971 std::tie(std::ignore, MemRuntimeCheckCond) =
1972 addRuntimeChecks(MemCheckBlock->getTerminator(), L,
1973 RtPtrChecking.getChecks(), MemCheckExp);
1974 assert(MemRuntimeCheckCond &&(static_cast<void> (0))
1975 "no RT checks generated although RtPtrChecking "(static_cast<void> (0))
1976 "claimed checks are required")(static_cast<void> (0));
1977 }
1978
1979 if (!MemCheckBlock && !SCEVCheckBlock)
1980 return;
1981
1982 // Unhook the temporary block with the checks, update various places
1983 // accordingly.
1984 if (SCEVCheckBlock)
1985 SCEVCheckBlock->replaceAllUsesWith(Preheader);
1986 if (MemCheckBlock)
1987 MemCheckBlock->replaceAllUsesWith(Preheader);
1988
1989 if (SCEVCheckBlock) {
1990 SCEVCheckBlock->getTerminator()->moveBefore(Preheader->getTerminator());
1991 new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1992 Preheader->getTerminator()->eraseFromParent();
1993 }
1994 if (MemCheckBlock) {
1995 MemCheckBlock->getTerminator()->moveBefore(Preheader->getTerminator());
1996 new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1997 Preheader->getTerminator()->eraseFromParent();
1998 }
1999
2000 DT->changeImmediateDominator(LoopHeader, Preheader);
2001 if (MemCheckBlock) {
2002 DT->eraseNode(MemCheckBlock);
2003 LI->removeBlock(MemCheckBlock);
2004 }
2005 if (SCEVCheckBlock) {
2006 DT->eraseNode(SCEVCheckBlock);
2007 LI->removeBlock(SCEVCheckBlock);
2008 }
2009 }
2010
2011 /// Remove the created SCEV & memory runtime check blocks & instructions, if
2012 /// unused.
2013 ~GeneratedRTChecks() {
2014 SCEVExpanderCleaner SCEVCleaner(SCEVExp, *DT);
2015 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp, *DT);
2016 if (!SCEVCheckCond)
2017 SCEVCleaner.markResultUsed();
2018
2019 if (!MemRuntimeCheckCond)
2020 MemCheckCleaner.markResultUsed();
2021
2022 if (MemRuntimeCheckCond) {
2023 auto &SE = *MemCheckExp.getSE();
2024 // Memory runtime check generation creates compares that use expanded
2025 // values. Remove them before running the SCEVExpanderCleaners.
2026 for (auto &I : make_early_inc_range(reverse(*MemCheckBlock))) {
2027 if (MemCheckExp.isInsertedInstruction(&I))
2028 continue;
2029 SE.forgetValue(&I);
2030 SE.eraseValueFromMap(&I);
2031 I.eraseFromParent();
2032 }
2033 }
2034 MemCheckCleaner.cleanup();
2035 SCEVCleaner.cleanup();
2036
2037 if (SCEVCheckCond)
2038 SCEVCheckBlock->eraseFromParent();
2039 if (MemRuntimeCheckCond)
2040 MemCheckBlock->eraseFromParent();
2041 }
2042
2043 /// Adds the generated SCEVCheckBlock before \p LoopVectorPreHeader and
2044 /// adjusts the branches to branch to the vector preheader or \p Bypass,
2045 /// depending on the generated condition.
2046 BasicBlock *emitSCEVChecks(Loop *L, BasicBlock *Bypass,
2047 BasicBlock *LoopVectorPreHeader,
2048 BasicBlock *LoopExitBlock) {
2049 if (!SCEVCheckCond)
2050 return nullptr;
2051 if (auto *C = dyn_cast<ConstantInt>(SCEVCheckCond))
2052 if (C->isZero())
2053 return nullptr;
2054
2055 auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
2056
2057 BranchInst::Create(LoopVectorPreHeader, SCEVCheckBlock);
2058 // Create new preheader for vector loop.
2059 if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
2060 PL->addBasicBlockToLoop(SCEVCheckBlock, *LI);
2061
2062 SCEVCheckBlock->getTerminator()->eraseFromParent();
2063 SCEVCheckBlock->moveBefore(LoopVectorPreHeader);
2064 Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
2065 SCEVCheckBlock);
2066
2067 DT->addNewBlock(SCEVCheckBlock, Pred);
2068 DT->changeImmediateDominator(LoopVectorPreHeader, SCEVCheckBlock);
2069
2070 ReplaceInstWithInst(
2071 SCEVCheckBlock->getTerminator(),
2072 BranchInst::Create(Bypass, LoopVectorPreHeader, SCEVCheckCond));
2073 // Mark the check as used, to prevent it from being removed during cleanup.
2074 SCEVCheckCond = nullptr;
2075 return SCEVCheckBlock;
2076 }
2077
2078 /// Adds the generated MemCheckBlock before \p LoopVectorPreHeader and adjusts
2079 /// the branches to branch to the vector preheader or \p Bypass, depending on
2080 /// the generated condition.
2081 BasicBlock *emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass,
2082 BasicBlock *LoopVectorPreHeader) {
2083 // Check if we generated code that checks in runtime if arrays overlap.
2084 if (!MemRuntimeCheckCond)
2085 return nullptr;
2086
2087 auto *Pred = LoopVectorPreHeader->getSinglePredecessor();
2088 Pred->getTerminator()->replaceSuccessorWith(LoopVectorPreHeader,
2089 MemCheckBlock);
2090
2091 DT->addNewBlock(MemCheckBlock, Pred);
2092 DT->changeImmediateDominator(LoopVectorPreHeader, MemCheckBlock);
2093 MemCheckBlock->moveBefore(LoopVectorPreHeader);
2094
2095 if (auto *PL = LI->getLoopFor(LoopVectorPreHeader))
2096 PL->addBasicBlockToLoop(MemCheckBlock, *LI);
2097
2098 ReplaceInstWithInst(
2099 MemCheckBlock->getTerminator(),
2100 BranchInst::Create(Bypass, LoopVectorPreHeader, MemRuntimeCheckCond));
2101 MemCheckBlock->getTerminator()->setDebugLoc(
2102 Pred->getTerminator()->getDebugLoc());
2103
2104 // Mark the check as used, to prevent it from being removed during cleanup.
2105 MemRuntimeCheckCond = nullptr;
2106 return MemCheckBlock;
2107 }
2108};
2109
2110// Return true if \p OuterLp is an outer loop annotated with hints for explicit
2111// vectorization. The loop needs to be annotated with #pragma omp simd
2112// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
2113// vector length information is not provided, vectorization is not considered
2114// explicit. Interleave hints are not allowed either. These limitations will be
2115// relaxed in the future.
2116// Please, note that we are currently forced to abuse the pragma 'clang
2117// vectorize' semantics. This pragma provides *auto-vectorization hints*
2118// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
2119// provides *explicit vectorization hints* (LV can bypass legal checks and
2120// assume that vectorization is legal). However, both hints are implemented
2121// using the same metadata (llvm.loop.vectorize, processed by
2122// LoopVectorizeHints). This will be fixed in the future when the native IR
2123// representation for pragma 'omp simd' is introduced.
2124static bool isExplicitVecOuterLoop(Loop *OuterLp,
2125 OptimizationRemarkEmitter *ORE) {
2126 assert(!OuterLp->isInnermost() && "This is not an outer loop")(static_cast<void> (0));
2127 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
2128
2129 // Only outer loops with an explicit vectorization hint are supported.
2130 // Unannotated outer loops are ignored.
2131 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
2132 return false;
2133
2134 Function *Fn = OuterLp->getHeader()->getParent();
2135 if (!Hints.allowVectorization(Fn, OuterLp,
2136 true /*VectorizeOnlyWhenForced*/)) {
2137 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n")do { } while (false);
2138 return false;
2139 }
2140
2141 if (Hints.getInterleave() > 1) {
2142 // TODO: Interleave support is future work.
2143 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "do { } while (false)
2144 "outer loops.\n")do { } while (false);
2145 Hints.emitRemarkWithHints();
2146 return false;
2147 }
2148
2149 return true;
2150}
2151
2152static void collectSupportedLoops(Loop &L, LoopInfo *LI,
2153 OptimizationRemarkEmitter *ORE,
2154 SmallVectorImpl<Loop *> &V) {
2155 // Collect inner loops and outer loops without irreducible control flow. For
2156 // now, only collect outer loops that have explicit vectorization hints. If we
2157 // are stress testing the VPlan H-CFG construction, we collect the outermost
2158 // loop of every loop nest.
2159 if (L.isInnermost() || VPlanBuildStressTest ||
2160 (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
2161 LoopBlocksRPO RPOT(&L);
2162 RPOT.perform(LI);
2163 if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
2164 V.push_back(&L);
2165 // TODO: Collect inner loops inside marked outer loops in case
2166 // vectorization fails for the outer loop. Do not invoke
2167 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
2168 // already known to be reducible. We can use an inherited attribute for
2169 // that.
2170 return;
2171 }
2172 }
2173 for (Loop *InnerL : L)
2174 collectSupportedLoops(*InnerL, LI, ORE, V);
2175}
2176
2177namespace {
2178
2179/// The LoopVectorize Pass.
2180struct LoopVectorize : public FunctionPass {
2181 /// Pass identification, replacement for typeid
2182 static char ID;
2183
2184 LoopVectorizePass Impl;
2185
2186 explicit LoopVectorize(bool InterleaveOnlyWhenForced = false,
2187 bool VectorizeOnlyWhenForced = false)
2188 : FunctionPass(ID),
2189 Impl({InterleaveOnlyWhenForced, VectorizeOnlyWhenForced}) {
2190 initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
2191 }
2192
2193 bool runOnFunction(Function &F) override {
2194 if (skipFunction(F))
2195 return false;
2196
2197 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
2198 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
2199 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2200 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
2201 auto *BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
2202 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
2203 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
2204 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2205 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
2206 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
2207 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
2208 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
2209 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
2210
2211 std::function<const LoopAccessInfo &(Loop &)> GetLAA =
2212 [&](Loop &L) -> const LoopAccessInfo & { return LAA->getInfo(&L); };
2213
2214 return Impl.runImpl(F, *SE, *LI, *TTI, *DT, *BFI, TLI, *DB, *AA, *AC,
2215 GetLAA, *ORE, PSI).MadeAnyChange;
2216 }
2217
2218 void getAnalysisUsage(AnalysisUsage &AU) const override {
2219 AU.addRequired<AssumptionCacheTracker>();
2220 AU.addRequired<BlockFrequencyInfoWrapperPass>();
2221 AU.addRequired<DominatorTreeWrapperPass>();
2222 AU.addRequired<LoopInfoWrapperPass>();
2223 AU.addRequired<ScalarEvolutionWrapperPass>();
2224 AU.addRequired<TargetTransformInfoWrapperPass>();
2225 AU.addRequired<AAResultsWrapperPass>();
2226 AU.addRequired<LoopAccessLegacyAnalysis>();
2227 AU.addRequired<DemandedBitsWrapperPass>();
2228 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
2229 AU.addRequired<InjectTLIMappingsLegacy>();
2230
2231 // We currently do not preserve loopinfo/dominator analyses with outer loop
2232 // vectorization. Until this is addressed, mark these analyses as preserved
2233 // only for non-VPlan-native path.
2234 // TODO: Preserve Loop and Dominator analyses for VPlan-native path.
2235 if (!EnableVPlanNativePath) {
2236 AU.addPreserved<LoopInfoWrapperPass>();
2237 AU.addPreserved<DominatorTreeWrapperPass>();
2238 }
2239
2240 AU.addPreserved<BasicAAWrapperPass>();
2241 AU.addPreserved<GlobalsAAWrapperPass>();
2242 AU.addRequired<ProfileSummaryInfoWrapperPass>();
2243 }
2244};
2245
2246} // end anonymous namespace
2247
2248//===----------------------------------------------------------------------===//
2249// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
2250// LoopVectorizationCostModel and LoopVectorizationPlanner.
2251//===----------------------------------------------------------------------===//
2252
2253Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
2254 // We need to place the broadcast of invariant variables outside the loop,
2255 // but only if it's proven safe to do so. Else, broadcast will be inside
2256 // vector loop body.
2257 Instruction *Instr = dyn_cast<Instruction>(V);
2258 bool SafeToHoist = OrigLoop->isLoopInvariant(V) &&
2259 (!Instr ||
2260 DT->dominates(Instr->getParent(), LoopVectorPreHeader));
2261 // Place the code for broadcasting invariant variables in the new preheader.
2262 IRBuilder<>::InsertPointGuard Guard(Builder);
2263 if (SafeToHoist)
2264 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
2265
2266 // Broadcast the scalar into all locations in the vector.
2267 Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
2268
2269 return Shuf;
2270}
2271
2272void InnerLoopVectorizer::createVectorIntOrFpInductionPHI(
2273 const InductionDescriptor &II, Value *Step, Value *Start,
2274 Instruction *EntryVal, VPValue *Def, VPValue *CastDef,
2275 VPTransformState &State) {
2276 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(static_cast<void> (0))
2277 "Expected either an induction phi-node or a truncate of it!")(static_cast<void> (0));
2278
2279 // Construct the initial value of the vector IV in the vector loop preheader
2280 auto CurrIP = Builder.saveIP();
2281 Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
2282 if (isa<TruncInst>(EntryVal)) {
2283 assert(Start->getType()->isIntegerTy() &&(static_cast<void> (0))
2284 "Truncation requires an integer type")(static_cast<void> (0));
2285 auto *TruncType = cast<IntegerType>(EntryVal->getType());
2286 Step = Builder.CreateTrunc(Step, TruncType);
2287 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
2288 }
2289 Value *SplatStart = Builder.CreateVectorSplat(VF, Start);
2290 Value *SteppedStart =
2291 getStepVector(SplatStart, 0, Step, II.getInductionOpcode());
2292
2293 // We create vector phi nodes for both integer and floating-point induction
2294 // variables. Here, we determine the kind of arithmetic we will perform.
2295 Instruction::BinaryOps AddOp;
2296 Instruction::BinaryOps MulOp;
2297 if (Step->getType()->isIntegerTy()) {
2298 AddOp = Instruction::Add;
2299 MulOp = Instruction::Mul;
2300 } else {
2301 AddOp = II.getInductionOpcode();
2302 MulOp = Instruction::FMul;
2303 }
2304
2305 // Multiply the vectorization factor by the step using integer or
2306 // floating-point arithmetic as appropriate.
2307 Type *StepType = Step->getType();
2308 if (Step->getType()->isFloatingPointTy())
2309 StepType = IntegerType::get(StepType->getContext(),
2310 StepType->getScalarSizeInBits());
2311 Value *RuntimeVF = getRuntimeVF(Builder, StepType, VF);
2312 if (Step->getType()->isFloatingPointTy())
2313 RuntimeVF = Builder.CreateSIToFP(RuntimeVF, Step->getType());
2314 Value *Mul = Builder.CreateBinOp(MulOp, Step, RuntimeVF);
2315
2316 // Create a vector splat to use in the induction update.
2317 //
2318 // FIXME: If the step is non-constant, we create the vector splat with
2319 // IRBuilder. IRBuilder can constant-fold the multiply, but it doesn't
2320 // handle a constant vector splat.
2321 Value *SplatVF = isa<Constant>(Mul)
2322 ? ConstantVector::getSplat(VF, cast<Constant>(Mul))
2323 : Builder.CreateVectorSplat(VF, Mul);
2324 Builder.restoreIP(CurrIP);
2325
2326 // We may need to add the step a number of times, depending on the unroll
2327 // factor. The last of those goes into the PHI.
2328 PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind",
2329 &*LoopVectorBody->getFirstInsertionPt());
2330 VecInd->setDebugLoc(EntryVal->getDebugLoc());
2331 Instruction *LastInduction = VecInd;
2332 for (unsigned Part = 0; Part < UF; ++Part) {
2333 State.set(Def, LastInduction, Part);
2334
2335 if (isa<TruncInst>(EntryVal))
2336 addMetadata(LastInduction, EntryVal);
2337 recordVectorLoopValueForInductionCast(II, EntryVal, LastInduction, CastDef,
2338 State, Part);
2339
2340 LastInduction = cast<Instruction>(
2341 Builder.CreateBinOp(AddOp, LastInduction, SplatVF, "step.add"));
2342 LastInduction->setDebugLoc(EntryVal->getDebugLoc());
2343 }
2344
2345 // Move the last step to the end of the latch block. This ensures consistent
2346 // placement of all induction updates.
2347 auto *LoopVectorLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
2348 auto *Br = cast<BranchInst>(LoopVectorLatch->getTerminator());
2349 auto *ICmp = cast<Instruction>(Br->getCondition());
2350 LastInduction->moveBefore(ICmp);
2351 LastInduction->setName("vec.ind.next");
2352
2353 VecInd->addIncoming(SteppedStart, LoopVectorPreHeader);
2354 VecInd->addIncoming(LastInduction, LoopVectorLatch);
2355}
2356
2357bool InnerLoopVectorizer::shouldScalarizeInstruction(Instruction *I) const {
2358 return Cost->isScalarAfterVectorization(I, VF) ||
2359 Cost->isProfitableToScalarize(I, VF);
2360}
2361
2362bool InnerLoopVectorizer::needsScalarInduction(Instruction *IV) const {
2363 if (shouldScalarizeInstruction(IV))
2364 return true;
2365 auto isScalarInst = [&](User *U) -> bool {
2366 auto *I = cast<Instruction>(U);
2367 return (OrigLoop->contains(I) && shouldScalarizeInstruction(I));
2368 };
2369 return llvm::any_of(IV->users(), isScalarInst);
2370}
2371
2372void InnerLoopVectorizer::recordVectorLoopValueForInductionCast(
2373 const InductionDescriptor &ID, const Instruction *EntryVal,
2374 Value *VectorLoopVal, VPValue *CastDef, VPTransformState &State,
2375 unsigned Part, unsigned Lane) {
2376 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&(static_cast<void> (0))
2377 "Expected either an induction phi-node or a truncate of it!")(static_cast<void> (0));
2378
2379 // This induction variable is not the phi from the original loop but the
2380 // newly-created IV based on the proof that casted Phi is equal to the
2381 // uncasted Phi in the vectorized loop (under a runtime guard possibly). It
2382 // re-uses the same InductionDescriptor that original IV uses but we don't
2383 // have to do any recording in this case - that is done when original IV is
2384 // processed.
2385 if (isa<TruncInst>(EntryVal))
2386 return;
2387
2388 const SmallVectorImpl<Instruction *> &Casts = ID.getCastInsts();
2389 if (Casts.empty())
2390 return;
2391 // Only the first Cast instruction in the Casts vector is of interest.
2392 // The rest of the Casts (if exist) have no uses outside the
2393 // induction update chain itself.
2394 if (Lane < UINT_MAX(2147483647 *2U +1U))
2395 State.set(CastDef, VectorLoopVal, VPIteration(Part, Lane));
2396 else
2397 State.set(CastDef, VectorLoopVal, Part);
2398}
2399
2400void InnerLoopVectorizer::widenIntOrFpInduction(PHINode *IV, Value *Start,
2401 TruncInst *Trunc, VPValue *Def,
2402 VPValue *CastDef,
2403 VPTransformState &State) {
2404 assert((IV->getType()->isIntegerTy() || IV != OldInduction) &&(static_cast<void> (0))
2405 "Primary induction variable must have an integer type")(static_cast<void> (0));
2406
2407 auto II = Legal->getInductionVars().find(IV);
2408 assert(II != Legal->getInductionVars().end() && "IV is not an induction")(static_cast<void> (0));
2409
2410 auto ID = II->second;
2411 assert(IV->getType() == ID.getStartValue()->getType() && "Types must match")(static_cast<void> (0));
2412
2413 // The value from the original loop to which we are mapping the new induction
2414 // variable.
2415 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
2416
2417 auto &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
2418
2419 // Generate code for the induction step. Note that induction steps are
2420 // required to be loop-invariant
2421 auto CreateStepValue = [&](const SCEV *Step) -> Value * {
2422 assert(PSE.getSE()->isLoopInvariant(Step, OrigLoop) &&(static_cast<void> (0))
2423 "Induction step should be loop invariant")(static_cast<void> (0));
2424 if (PSE.getSE()->isSCEVable(IV->getType())) {
2425 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
2426 return Exp.expandCodeFor(Step, Step->getType(),
2427 LoopVectorPreHeader->getTerminator());
2428 }
2429 return cast<SCEVUnknown>(Step)->getValue();
2430 };
2431
2432 // The scalar value to broadcast. This is derived from the canonical
2433 // induction variable. If a truncation type is given, truncate the canonical
2434 // induction variable and step. Otherwise, derive these values from the
2435 // induction descriptor.
2436 auto CreateScalarIV = [&](Value *&Step) -> Value * {
2437 Value *ScalarIV = Induction;
2438 if (IV != OldInduction) {
2439 ScalarIV = IV->getType()->isIntegerTy()
2440 ? Builder.CreateSExtOrTrunc(Induction, IV->getType())
2441 : Builder.CreateCast(Instruction::SIToFP, Induction,
2442 IV->getType());
2443 ScalarIV = emitTransformedIndex(Builder, ScalarIV, PSE.getSE(), DL, ID);
2444 ScalarIV->setName("offset.idx");
2445 }
2446 if (Trunc) {
2447 auto *TruncType = cast<IntegerType>(Trunc->getType());
2448 assert(Step->getType()->isIntegerTy() &&(static_cast<void> (0))
2449 "Truncation requires an integer step")(static_cast<void> (0));
2450 ScalarIV = Builder.CreateTrunc(ScalarIV, TruncType);
2451 Step = Builder.CreateTrunc(Step, TruncType);
2452 }
2453 return ScalarIV;
2454 };
2455
2456 // Create the vector values from the scalar IV, in the absence of creating a
2457 // vector IV.
2458 auto CreateSplatIV = [&](Value *ScalarIV, Value *Step) {
2459 Value *Broadcasted = getBroadcastInstrs(ScalarIV);
2460 for (unsigned Part = 0; Part < UF; ++Part) {
2461 assert(!VF.isScalable() && "scalable vectors not yet supported.")(static_cast<void> (0));
2462 Value *EntryPart =
2463 getStepVector(Broadcasted, VF.getKnownMinValue() * Part, Step,
2464 ID.getInductionOpcode());
2465 State.set(Def, EntryPart, Part);
2466 if (Trunc)
2467 addMetadata(EntryPart, Trunc);
2468 recordVectorLoopValueForInductionCast(ID, EntryVal, EntryPart, CastDef,
2469 State, Part);
2470 }
2471 };
2472
2473 // Fast-math-flags propagate from the original induction instruction.
2474 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
2475 if (ID.getInductionBinOp() && isa<FPMathOperator>(ID.getInductionBinOp()))
2476 Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());
2477
2478 // Now do the actual transformations, and start with creating the step value.
2479 Value *Step = CreateStepValue(ID.getStep());
2480 if (VF.isZero() || VF.isScalar()) {
2481 Value *ScalarIV = CreateScalarIV(Step);
2482 CreateSplatIV(ScalarIV, Step);
2483 return;
2484 }
2485
2486 // Determine if we want a scalar version of the induction variable. This is
2487 // true if the induction variable itself is not widened, or if it has at
2488 // least one user in the loop that is not widened.
2489 auto NeedsScalarIV = needsScalarInduction(EntryVal);
2490 if (!NeedsScalarIV) {
2491 createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef,
2492 State);
2493 return;
2494 }
2495
2496 // Try to create a new independent vector induction variable. If we can't
2497 // create the phi node, we will splat the scalar induction variable in each
2498 // loop iteration.
2499 if (!shouldScalarizeInstruction(EntryVal)) {
2500 createVectorIntOrFpInductionPHI(ID, Step, Start, EntryVal, Def, CastDef,
2501 State);
2502 Value *ScalarIV = CreateScalarIV(Step);
2503 // Create scalar steps that can be used by instructions we will later
2504 // scalarize. Note that the addition of the scalar steps will not increase
2505 // the number of instructions in the loop in the common case prior to
2506 // InstCombine. We will be trading one vector extract for each scalar step.
2507 buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, CastDef, State);
2508 return;
2509 }
2510
2511 // All IV users are scalar instructions, so only emit a scalar IV, not a
2512 // vectorised IV. Except when we tail-fold, then the splat IV feeds the
2513 // predicate used by the masked loads/stores.
2514 Value *ScalarIV = CreateScalarIV(Step);
2515 if (!Cost->isScalarEpilogueAllowed())
2516 CreateSplatIV(ScalarIV, Step);
2517 buildScalarSteps(ScalarIV, Step, EntryVal, ID, Def, CastDef, State);
2518}
2519
2520Value *InnerLoopVectorizer::getStepVector(Value *Val, int StartIdx, Value *Step,
2521 Instruction::BinaryOps BinOp) {
2522 // Create and check the types.
2523 auto *ValVTy = cast<VectorType>(Val->getType());
2524 ElementCount VLen = ValVTy->getElementCount();
2525
2526 Type *STy = Val->getType()->getScalarType();
2527 assert((STy->isIntegerTy() || STy->isFloatingPointTy()) &&(static_cast<void> (0))
2528 "Induction Step must be an integer or FP")(static_cast<void> (0));
2529 assert(Step->getType() == STy && "Step has wrong type")(static_cast<void> (0));
2530
2531 SmallVector<Constant *, 8> Indices;
2532
2533 // Create a vector of consecutive numbers from zero to VF.
2534 VectorType *InitVecValVTy = ValVTy;
2535 Type *InitVecValSTy = STy;
2536 if (STy->isFloatingPointTy()) {
2537 InitVecValSTy =
2538 IntegerType::get(STy->getContext(), STy->getScalarSizeInBits());
2539 InitVecValVTy = VectorType::get(InitVecValSTy, VLen);
2540 }
2541 Value *InitVec = Builder.CreateStepVector(InitVecValVTy);
2542
2543 // Add on StartIdx
2544 Value *StartIdxSplat = Builder.CreateVectorSplat(
2545 VLen, ConstantInt::get(InitVecValSTy, StartIdx));
2546 InitVec = Builder.CreateAdd(InitVec, StartIdxSplat);
2547
2548 if (STy->isIntegerTy()) {
2549 Step = Builder.CreateVectorSplat(VLen, Step);
2550 assert(Step->getType() == Val->getType() && "Invalid step vec")(static_cast<void> (0));
2551 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
2552 // which can be found from the original scalar operations.
2553 Step = Builder.CreateMul(InitVec, Step);
2554 return Builder.CreateAdd(Val, Step, "induction");
2555 }
2556
2557 // Floating point induction.
2558 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&(static_cast<void> (0))
2559 "Binary Opcode should be specified for FP induction")(static_cast<void> (0));
2560 InitVec = Builder.CreateUIToFP(InitVec, ValVTy);
2561 Step = Builder.CreateVectorSplat(VLen, Step);
2562 Value *MulOp = Builder.CreateFMul(InitVec, Step);
2563 return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");
2564}
2565
2566void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
2567 Instruction *EntryVal,
2568 const InductionDescriptor &ID,
2569 VPValue *Def, VPValue *CastDef,
2570 VPTransformState &State) {
2571 // We shouldn't have to build scalar steps if we aren't vectorizing.
2572 assert(VF.isVector() && "VF should be greater than one")(static_cast<void> (0));
2573 // Get the value type and ensure it and the step have the same integer type.
2574 Type *ScalarIVTy = ScalarIV->getType()->getScalarType();
2575 assert(ScalarIVTy == Step->getType() &&(static_cast<void> (0))
2576 "Val and Step should have the same type")(static_cast<void> (0));
2577
2578 // We build scalar steps for both integer and floating-point induction
2579 // variables. Here, we determine the kind of arithmetic we will perform.
2580 Instruction::BinaryOps AddOp;
2581 Instruction::BinaryOps MulOp;
2582 if (ScalarIVTy->isIntegerTy()) {
2583 AddOp = Instruction::Add;
2584 MulOp = Instruction::Mul;
2585 } else {
2586 AddOp = ID.getInductionOpcode();
2587 MulOp = Instruction::FMul;
2588 }
2589
2590 // Determine the number of scalars we need to generate for each unroll
2591 // iteration. If EntryVal is uniform, we only need to generate the first
2592 // lane. Otherwise, we generate all VF values.
2593 bool IsUniform =
2594 Cost->isUniformAfterVectorization(cast<Instruction>(EntryVal), VF);
2595 unsigned Lanes = IsUniform ? 1 : VF.getKnownMinValue();
2596 // Compute the scalar steps and save the results in State.
2597 Type *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
2598 ScalarIVTy->getScalarSizeInBits());
2599 Type *VecIVTy = nullptr;
2600 Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;
2601 if (!IsUniform && VF.isScalable()) {
2602 VecIVTy = VectorType::get(ScalarIVTy, VF);
2603 UnitStepVec = Builder.CreateStepVector(VectorType::get(IntStepTy, VF));
2604 SplatStep = Builder.CreateVectorSplat(VF, Step);
2605 SplatIV = Builder.CreateVectorSplat(VF, ScalarIV);
2606 }
2607
2608 for (unsigned Part = 0; Part < UF; ++Part) {
2609 Value *StartIdx0 =
2610 createStepForVF(Builder, ConstantInt::get(IntStepTy, Part), VF);
2611
2612 if (!IsUniform && VF.isScalable()) {
2613 auto *SplatStartIdx = Builder.CreateVectorSplat(VF, StartIdx0);
2614 auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
2615 if (ScalarIVTy->isFloatingPointTy())
2616 InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
2617 auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
2618 auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
2619 State.set(Def, Add, Part);
2620 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, CastDef, State,
2621 Part);
2622 // It's useful to record the lane values too for the known minimum number
2623 // of elements so we do those below. This improves the code quality when
2624 // trying to extract the first element, for example.
2625 }
2626
2627 if (ScalarIVTy->isFloatingPointTy())
2628 StartIdx0 = Builder.CreateSIToFP(StartIdx0, ScalarIVTy);
2629
2630 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
2631 Value *StartIdx = Builder.CreateBinOp(
2632 AddOp, StartIdx0, getSignedIntOrFpConstant(ScalarIVTy, Lane));
2633 // The step returned by `createStepForVF` is a runtime-evaluated value
2634 // when VF is scalable. Otherwise, it should be folded into a Constant.
2635 assert((VF.isScalable() || isa<Constant>(StartIdx)) &&(static_cast<void> (0))
2636 "Expected StartIdx to be folded to a constant when VF is not "(static_cast<void> (0))
2637 "scalable")(static_cast<void> (0));
2638 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
2639 auto *Add = Builder.CreateBinOp(AddOp, ScalarIV, Mul);
2640 State.set(Def, Add, VPIteration(Part, Lane));
2641 recordVectorLoopValueForInductionCast(ID, EntryVal, Add, CastDef, State,
2642 Part, Lane);
2643 }
2644 }
2645}
2646
2647void InnerLoopVectorizer::packScalarIntoVectorValue(VPValue *Def,
2648 const VPIteration &Instance,
2649 VPTransformState &State) {
2650 Value *ScalarInst = State.get(Def, Instance);
2651 Value *VectorValue = State.get(Def, Instance.Part);
2652 VectorValue = Builder.CreateInsertElement(
2653 VectorValue, ScalarInst,
2654 Instance.Lane.getAsRuntimeExpr(State.Builder, VF));
2655 State.set(Def, VectorValue, Instance.Part);
2656}
2657
2658Value *InnerLoopVectorizer::reverseVector(Value *Vec) {
2659 assert(Vec->getType()->isVectorTy() && "Invalid type")(static_cast<void> (0));
2660 return Builder.CreateVectorReverse(Vec, "reverse");
2661}
2662
2663// Return whether we allow using masked interleave-groups (for dealing with
2664// strided loads/stores that reside in predicated blocks, or for dealing
2665// with gaps).
2666static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
2667 // If an override option has been passed in for interleaved accesses, use it.
2668 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
2669 return EnableMaskedInterleavedMemAccesses;
2670
2671 return TTI.enableMaskedInterleavedAccessVectorization();
2672}
2673
2674// Try to vectorize the interleave group that \p Instr belongs to.
2675//
2676// E.g. Translate following interleaved load group (factor = 3):
2677// for (i = 0; i < N; i+=3) {
2678// R = Pic[i]; // Member of index 0
2679// G = Pic[i+1]; // Member of index 1
2680// B = Pic[i+2]; // Member of index 2
2681// ... // do something to R, G, B
2682// }
2683// To:
2684// %wide.vec = load <12 x i32> ; Read 4 tuples of R,G,B
2685// %R.vec = shuffle %wide.vec, poison, <0, 3, 6, 9> ; R elements
2686// %G.vec = shuffle %wide.vec, poison, <1, 4, 7, 10> ; G elements
2687// %B.vec = shuffle %wide.vec, poison, <2, 5, 8, 11> ; B elements
2688//
2689// Or translate following interleaved store group (factor = 3):
2690// for (i = 0; i < N; i+=3) {
2691// ... do something to R, G, B
2692// Pic[i] = R; // Member of index 0
2693// Pic[i+1] = G; // Member of index 1
2694// Pic[i+2] = B; // Member of index 2
2695// }
2696// To:
2697// %R_G.vec = shuffle %R.vec, %G.vec, <0, 1, 2, ..., 7>
2698// %B_U.vec = shuffle %B.vec, poison, <0, 1, 2, 3, u, u, u, u>
2699// %interleaved.vec = shuffle %R_G.vec, %B_U.vec,
2700// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> ; Interleave R,G,B elements
2701// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
2702void InnerLoopVectorizer::vectorizeInterleaveGroup(
2703 const InterleaveGroup<Instruction> *Group, ArrayRef<VPValue *> VPDefs,
2704 VPTransformState &State, VPValue *Addr, ArrayRef<VPValue *> StoredValues,
2705 VPValue *BlockInMask) {
2706 Instruction *Instr = Group->getInsertPos();
2707 const DataLayout &DL = Instr->getModule()->getDataLayout();
2708
2709 // Prepare for the vector type of the interleaved load/store.
2710 Type *ScalarTy = getLoadStoreType(Instr);
2711 unsigned InterleaveFactor = Group->getFactor();
2712 assert(!VF.isScalable() && "scalable vectors not yet supported.")(static_cast<void> (0));
2713 auto *VecTy = VectorType::get(ScalarTy, VF * InterleaveFactor);
2714
2715 // Prepare for the new pointers.
2716 SmallVector<Value *, 2> AddrParts;
2717 unsigned Index = Group->getIndex(Instr);
2718
2719 // TODO: extend the masked interleaved-group support to reversed access.
2720 assert((!BlockInMask || !Group->isReverse()) &&(static_cast<void> (0))
2721 "Reversed masked interleave-group not supported.")(static_cast<void> (0));
2722
2723 // If the group is reverse, adjust the index to refer to the last vector lane
2724 // instead of the first. We adjust the index from the first vector lane,
2725 // rather than directly getting the pointer for lane VF - 1, because the
2726 // pointer operand of the interleaved access is supposed to be uniform. For
2727 // uniform instructions, we're only required to generate a value for the
2728 // first vector lane in each unroll iteration.
2729 if (Group->isReverse())
2730 Index += (VF.getKnownMinValue() - 1) * Group->getFactor();
2731
2732 for (unsigned Part = 0; Part < UF; Part++) {
2733 Value *AddrPart = State.get(Addr, VPIteration(Part, 0));
2734 setDebugLocFromInst(AddrPart);
2735
2736 // Notice current instruction could be any index. Need to adjust the address
2737 // to the member of index 0.
2738 //
2739 // E.g. a = A[i+1]; // Member of index 1 (Current instruction)
2740 // b = A[i]; // Member of index 0
2741 // Current pointer is pointed to A[i+1], adjust it to A[i].
2742 //
2743 // E.g. A[i+1] = a; // Member of index 1
2744 // A[i] = b; // Member of index 0
2745 // A[i+2] = c; // Member of index 2 (Current instruction)
2746 // Current pointer is pointed to A[i+2], adjust it to A[i].
2747
2748 bool InBounds = false;
2749 if (auto *gep = dyn_cast<GetElementPtrInst>(AddrPart->stripPointerCasts()))
2750 InBounds = gep->isInBounds();
2751 AddrPart = Builder.CreateGEP(ScalarTy, AddrPart, Builder.getInt32(-Index));
2752 cast<GetElementPtrInst>(AddrPart)->setIsInBounds(InBounds);
2753
2754 // Cast to the vector pointer type.
2755 unsigned AddressSpace = AddrPart->getType()->getPointerAddressSpace();
2756 Type *PtrTy = VecTy->getPointerTo(AddressSpace);
2757 AddrParts.push_back(Builder.CreateBitCast(AddrPart, PtrTy));
2758 }
2759
2760 setDebugLocFromInst(Instr);
2761 Value *PoisonVec = PoisonValue::get(VecTy);
2762
2763 Value *MaskForGaps = nullptr;
2764 if (Group->requiresScalarEpilogue() && !Cost->isScalarEpilogueAllowed()) {
2765 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
2766 assert(MaskForGaps && "Mask for Gaps is required but it is null")(static_cast<void> (0));
2767 }
2768
2769 // Vectorize the interleaved load group.
2770 if (isa<LoadInst>(Instr)) {
2771 // For each unroll part, create a wide load for the group.
2772 SmallVector<Value *, 2> NewLoads;
2773 for (unsigned Part = 0; Part < UF; Part++) {
2774 Instruction *NewLoad;
2775 if (BlockInMask || MaskForGaps) {
2776 assert(useMaskedInterleavedAccesses(*TTI) &&(static_cast<void> (0))
2777 "masked interleaved groups are not allowed.")(static_cast<void> (0));
2778 Value *GroupMask = MaskForGaps;
2779 if (BlockInMask) {
2780 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2781 Value *ShuffledMask = Builder.CreateShuffleVector(
2782 BlockInMaskPart,
2783 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2784 "interleaved.mask");
2785 GroupMask = MaskForGaps
2786 ? Builder.CreateBinOp(Instruction::And, ShuffledMask,
2787 MaskForGaps)
2788 : ShuffledMask;
2789 }
2790 NewLoad =
2791 Builder.CreateMaskedLoad(VecTy, AddrParts[Part], Group->getAlign(),
2792 GroupMask, PoisonVec, "wide.masked.vec");
2793 }
2794 else
2795 NewLoad = Builder.CreateAlignedLoad(VecTy, AddrParts[Part],
2796 Group->getAlign(), "wide.vec");
2797 Group->addMetadata(NewLoad);
2798 NewLoads.push_back(NewLoad);
2799 }
2800
2801 // For each member in the group, shuffle out the appropriate data from the
2802 // wide loads.
2803 unsigned J = 0;
2804 for (unsigned I = 0; I < InterleaveFactor; ++I) {
2805 Instruction *Member = Group->getMember(I);
2806
2807 // Skip the gaps in the group.
2808 if (!Member)
2809 continue;
2810
2811 auto StrideMask =
2812 createStrideMask(I, InterleaveFactor, VF.getKnownMinValue());
2813 for (unsigned Part = 0; Part < UF; Part++) {
2814 Value *StridedVec = Builder.CreateShuffleVector(
2815 NewLoads[Part], StrideMask, "strided.vec");
2816
2817 // If this member has different type, cast the result type.
2818 if (Member->getType() != ScalarTy) {
2819 assert(!VF.isScalable() && "VF is assumed to be non scalable.")(static_cast<void> (0));
2820 VectorType *OtherVTy = VectorType::get(Member->getType(), VF);
2821 StridedVec = createBitOrPointerCast(StridedVec, OtherVTy, DL);
2822 }
2823
2824 if (Group->isReverse())
2825 StridedVec = reverseVector(StridedVec);
2826
2827 State.set(VPDefs[J], StridedVec, Part);
2828 }
2829 ++J;
2830 }
2831 return;
2832 }
2833
2834 // The sub vector type for current instruction.
2835 auto *SubVT = VectorType::get(ScalarTy, VF);
2836
2837 // Vectorize the interleaved store group.
2838 MaskForGaps = createBitMaskForGaps(Builder, VF.getKnownMinValue(), *Group);
2839 assert((!MaskForGaps || useMaskedInterleavedAccesses(*TTI)) &&(static_cast<void> (0))
2840 "masked interleaved groups are not allowed.")(static_cast<void> (0));
2841 assert((!MaskForGaps || !VF.isScalable()) &&(static_cast<void> (0))
2842 "masking gaps for scalable vectors is not yet supported.")(static_cast<void> (0));
2843 for (unsigned Part = 0; Part < UF; Part++) {
2844 // Collect the stored vector from each member.
2845 SmallVector<Value *, 4> StoredVecs;
2846 for (unsigned i = 0; i < InterleaveFactor; i++) {
2847 assert((Group->getMember(i) || MaskForGaps) &&(static_cast<void> (0))
2848 "Fail to get a member from an interleaved store group")(static_cast<void> (0));
2849 Instruction *Member = Group->getMember(i);
2850
2851 // Skip the gaps in the group.
2852 if (!Member) {
2853 Value *Undef = PoisonValue::get(SubVT);
2854 StoredVecs.push_back(Undef);
2855 continue;
2856 }
2857
2858 Value *StoredVec = State.get(StoredValues[i], Part);
2859
2860 if (Group->isReverse())
2861 StoredVec = reverseVector(StoredVec);
2862
2863 // If this member has different type, cast it to a unified type.
2864
2865 if (StoredVec->getType() != SubVT)
2866 StoredVec = createBitOrPointerCast(StoredVec, SubVT, DL);
2867
2868 StoredVecs.push_back(StoredVec);
2869 }
2870
2871 // Concatenate all vectors into a wide vector.
2872 Value *WideVec = concatenateVectors(Builder, StoredVecs);
2873
2874 // Interleave the elements in the wide vector.
2875 Value *IVec = Builder.CreateShuffleVector(
2876 WideVec, createInterleaveMask(VF.getKnownMinValue(), InterleaveFactor),
2877 "interleaved.vec");
2878
2879 Instruction *NewStoreInstr;
2880 if (BlockInMask || MaskForGaps) {
2881 Value *GroupMask = MaskForGaps;
2882 if (BlockInMask) {
2883 Value *BlockInMaskPart = State.get(BlockInMask, Part);
2884 Value *ShuffledMask = Builder.CreateShuffleVector(
2885 BlockInMaskPart,
2886 createReplicatedMask(InterleaveFactor, VF.getKnownMinValue()),
2887 "interleaved.mask");
2888 GroupMask = MaskForGaps ? Builder.CreateBinOp(Instruction::And,
2889 ShuffledMask, MaskForGaps)
2890 : ShuffledMask;
2891 }
2892 NewStoreInstr = Builder.CreateMaskedStore(IVec, AddrParts[Part],
2893 Group->getAlign(), GroupMask);
2894 } else
2895 NewStoreInstr =
2896 Builder.CreateAlignedStore(IVec, AddrParts[Part], Group->getAlign());
2897
2898 Group->addMetadata(NewStoreInstr);
2899 }
2900}
2901
2902void InnerLoopVectorizer::vectorizeMemoryInstruction(
2903 Instruction *Instr, VPTransformState &State, VPValue *Def, VPValue *Addr,
2904 VPValue *StoredValue, VPValue *BlockInMask) {
2905 // Attempt to issue a wide load.
2906 LoadInst *LI = dyn_cast<LoadInst>(Instr);
2907 StoreInst *SI = dyn_cast<StoreInst>(Instr);
2908
2909 assert((LI || SI) && "Invalid Load/Store instruction")(static_cast<void> (0));
2910 assert((!SI || StoredValue) && "No stored value provided for widened store")(static_cast<void> (0));
2911 assert((!LI || !StoredValue) && "Stored value provided for widened load")(static_cast<void> (0));
2912
2913 LoopVectorizationCostModel::InstWidening Decision =
2914 Cost->getWideningDecision(Instr, VF);
2915 assert((Decision == LoopVectorizationCostModel::CM_Widen ||(static_cast<void> (0))
2916 Decision == LoopVectorizationCostModel::CM_Widen_Reverse ||(static_cast<void> (0))
2917 Decision == LoopVectorizationCostModel::CM_GatherScatter) &&(static_cast<void> (0))
2918 "CM decision is not to widen the memory instruction")(static_cast<void> (0));
2919
2920 Type *ScalarDataTy = getLoadStoreType(Instr);
2921
2922 auto *DataTy = VectorType::get(ScalarDataTy, VF);
2923 const Align Alignment = getLoadStoreAlignment(Instr);
2924
2925 // Determine if the pointer operand of the access is either consecutive or
2926 // reverse consecutive.
2927 bool Reverse = (Decision == LoopVectorizationCostModel::CM_Widen_Reverse);
2928 bool ConsecutiveStride =
2929 Reverse || (Decision == LoopVectorizationCostModel::CM_Widen);
2930 bool CreateGatherScatter =
2931 (Decision == LoopVectorizationCostModel::CM_GatherScatter);
2932
2933 // Either Ptr feeds a vector load/store, or a vector GEP should feed a vector
2934 // gather/scatter. Otherwise Decision should have been to Scalarize.
2935 assert((ConsecutiveStride || CreateGatherScatter) &&(static_cast<void> (0))
2936 "The instruction should be scalarized")(static_cast<void> (0));
2937 (void)ConsecutiveStride;
2938
2939 VectorParts BlockInMaskParts(UF);
2940 bool isMaskRequired = BlockInMask;
2941 if (isMaskRequired)
2942 for (unsigned Part = 0; Part < UF; ++Part)
2943 BlockInMaskParts[Part] = State.get(BlockInMask, Part);
2944
2945 const auto CreateVecPtr = [&](unsigned Part, Value *Ptr) -> Value * {
2946 // Calculate the pointer for the specific unroll-part.
2947 GetElementPtrInst *PartPtr = nullptr;
2948
2949 bool InBounds = false;
2950 if (auto *gep = dyn_cast<GetElementPtrInst>(Ptr->stripPointerCasts()))
2951 InBounds = gep->isInBounds();
2952 if (Reverse) {
2953 // If the address is consecutive but reversed, then the
2954 // wide store needs to start at the last vector element.
2955 // RunTimeVF = VScale * VF.getKnownMinValue()
2956 // For fixed-width VScale is 1, then RunTimeVF = VF.getKnownMinValue()
2957 Value *RunTimeVF = getRuntimeVF(Builder, Builder.getInt32Ty(), VF);
2958 // NumElt = -Part * RunTimeVF
2959 Value *NumElt = Builder.CreateMul(Builder.getInt32(-Part), RunTimeVF);
2960 // LastLane = 1 - RunTimeVF
2961 Value *LastLane = Builder.CreateSub(Builder.getInt32(1), RunTimeVF);
2962 PartPtr =
2963 cast<GetElementPtrInst>(Builder.CreateGEP(ScalarDataTy, Ptr, NumElt));
2964 PartPtr->setIsInBounds(InBounds);
2965 PartPtr = cast<GetElementPtrInst>(
2966 Builder.CreateGEP(ScalarDataTy, PartPtr, LastLane));
2967 PartPtr->setIsInBounds(InBounds);
2968 if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
2969 BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
2970 } else {
2971 Value *Increment = createStepForVF(Builder, Builder.getInt32(Part), VF);
2972 PartPtr = cast<GetElementPtrInst>(
2973 Builder.CreateGEP(ScalarDataTy, Ptr, Increment));
2974 PartPtr->setIsInBounds(InBounds);
2975 }
2976
2977 unsigned AddressSpace = Ptr->getType()->getPointerAddressSpace();
2978 return Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace));
2979 };
2980
2981 // Handle Stores:
2982 if (SI) {
2983 setDebugLocFromInst(SI);
2984
2985 for (unsigned Part = 0; Part < UF; ++Part) {
2986 Instruction *NewSI = nullptr;
2987 Value *StoredVal = State.get(StoredValue, Part);
2988 if (CreateGatherScatter) {
2989 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
2990 Value *VectorGep = State.get(Addr, Part);
2991 NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
2992 MaskPart);
2993 } else {
2994 if (Reverse) {
2995 // If we store to reverse consecutive memory locations, then we need
2996 // to reverse the order of elements in the stored value.
2997 StoredVal = reverseVector(StoredVal);
2998 // We don't want to update the value in the map as it might be used in
2999 // another expression. So don't call resetVectorValue(StoredVal).
3000 }
3001 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, VPIteration(0, 0)));
3002 if (isMaskRequired)
3003 NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
3004 BlockInMaskParts[Part]);
3005 else
3006 NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
3007 }
3008 addMetadata(NewSI, SI);
3009 }
3010 return;
3011 }
3012
3013 // Handle loads.
3014 assert(LI && "Must have a load instruction")(static_cast<void> (0));
3015 setDebugLocFromInst(LI);
3016 for (unsigned Part = 0; Part < UF; ++Part) {
3017 Value *NewLI;
3018 if (CreateGatherScatter) {
3019 Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
3020 Value *VectorGep = State.get(Addr, Part);
3021 NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
3022 nullptr, "wide.masked.gather");
3023 addMetadata(NewLI, LI);
3024 } else {
3025 auto *VecPtr = CreateVecPtr(Part, State.get(Addr, VPIteration(0, 0)));
3026 if (isMaskRequired)
3027 NewLI = Builder.CreateMaskedLoad(
3028 DataTy, VecPtr, Alignment, BlockInMaskParts[Part],
3029 PoisonValue::get(DataTy), "wide.masked.load");
3030 else
3031 NewLI =
3032 Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
3033
3034 // Add metadata to the load, but setVectorValue to the reverse shuffle.
3035 addMetadata(NewLI, LI);
3036 if (Reverse)
3037 NewLI = reverseVector(NewLI);
3038 }
3039
3040 State.set(Def, NewLI, Part);
3041 }
3042}
3043
3044void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPValue *Def,
3045 VPUser &User,
3046 const VPIteration &Instance,
3047 bool IfPredicateInstr,
3048 VPTransformState &State) {
3049 assert(!Instr->getType()->isAggregateType() && "Can't handle vectors")(static_cast<void> (0));
3050
3051 // llvm.experimental.noalias.scope.decl intrinsics must only be duplicated for
3052 // the first lane and part.
3053 if (isa<NoAliasScopeDeclInst>(Instr))
3054 if (!Instance.isFirstIteration())
3055 return;
3056
3057 setDebugLocFromInst(Instr);
3058
3059 // Does this instruction return a value ?
3060 bool IsVoidRetTy = Instr->getType()->isVoidTy();
3061
3062 Instruction *Cloned = Instr->clone();
3063 if (!IsVoidRetTy)
3064 Cloned->setName(Instr->getName() + ".cloned");
3065
3066 State.Builder.SetInsertPoint(Builder.GetInsertBlock(),
3067 Builder.GetInsertPoint());
3068 // Replace the operands of the cloned instructions with their scalar
3069 // equivalents in the new loop.
3070 for (unsigned op = 0, e = User.getNumOperands(); op != e; ++op) {
3071 auto *Operand = dyn_cast<Instruction>(Instr->getOperand(op));
3072 auto InputInstance = Instance;
3073 if (!Operand || !OrigLoop->contains(Operand) ||
3074 (Cost->isUniformAfterVectorization(Operand, State.VF)))
3075 InputInstance.Lane = VPLane::getFirstLane();
3076 auto *NewOp = State.get(User.getOperand(op), InputInstance);
3077 Cloned->setOperand(op, NewOp);
3078 }
3079 addNewMetadata(Cloned, Instr);
3080
3081 // Place the cloned scalar in the new loop.
3082 Builder.Insert(Cloned);
3083
3084 State.set(Def, Cloned, Instance);
3085
3086 // If we just cloned a new assumption, add it the assumption cache.
3087 if (auto *II = dyn_cast<AssumeInst>(Cloned))
3088 AC->registerAssumption(II);
3089
3090 // End if-block.
3091 if (IfPredicateInstr)
3092 PredicatedInstructions.push_back(Cloned);
3093}
3094
3095PHINode *InnerLoopVectorizer::createInductionVariable(Loop *L, Value *Start,
3096 Value *End, Value *Step,
3097 Instruction *DL) {
3098 BasicBlock *Header = L->getHeader();
3099 BasicBlock *Latch = L->getLoopLatch();
3100 // As we're just creating this loop, it's possible no latch exists
3101 // yet. If so, use the header as this will be a single block loop.
3102 if (!Latch)
3103 Latch = Header;
3104
3105 IRBuilder<> B(&*Header->getFirstInsertionPt());
3106 Instruction *OldInst = getDebugLocFromInstOrOperands(OldInduction);
3107 setDebugLocFromInst(OldInst, &B);
3108 auto *Induction = B.CreatePHI(Start->getType(), 2, "index");
3109
3110 B.SetInsertPoint(Latch->getTerminator());
3111 setDebugLocFromInst(OldInst, &B);
3112
3113 // Create i+1 and fill the PHINode.
3114 //
3115 // If the tail is not folded, we know that End - Start >= Step (either
3116 // statically or through the minimum iteration checks). We also know that both
3117 // Start % Step == 0 and End % Step == 0. We exit the vector loop if %IV +
3118 // %Step == %End. Hence we must exit the loop before %IV + %Step unsigned
3119 // overflows and we can mark the induction increment as NUW.
3120 Value *Next = B.CreateAdd(Induction, Step, "index.next",
3121 /*NUW=*/!Cost->foldTailByMasking(), /*NSW=*/false);
3122 Induction->addIncoming(Start, L->getLoopPreheader());
3123 Induction->addIncoming(Next, Latch);
3124 // Create the compare.
3125 Value *ICmp = B.CreateICmpEQ(Next, End);
3126 B.CreateCondBr(ICmp, L->getUniqueExitBlock(), Header);
3127
3128 // Now we have two terminators. Remove the old one from the block.
3129 Latch->getTerminator()->eraseFromParent();
3130
3131 return Induction;
3132}
3133
3134Value *InnerLoopVectorizer::getOrCreateTripCount(Loop *L) {
3135 if (TripCount)
3136 return TripCount;
3137
3138 assert(L && "Create Trip Count for null loop.")(static_cast<void> (0));
3139 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
3140 // Find the loop boundaries.
3141 ScalarEvolution *SE = PSE.getSE();
3142 const SCEV *BackedgeTakenCount = PSE.getBackedgeTakenCount();
3143 assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCount) &&(static_cast<void> (0))
3144 "Invalid loop count")(static_cast<void> (0));
3145
3146 Type *IdxTy = Legal->getWidestInductionType();
3147 assert(IdxTy && "No type for induction")(static_cast<void> (0));
3148
3149 // The exit count might have the type of i64 while the phi is i32. This can
3150 // happen if we have an induction variable that is sign extended before the
3151 // compare. The only way that we get a backedge taken count is that the
3152 // induction variable was signed and as such will not overflow. In such a case
3153 // truncation is legal.
3154 if (SE->getTypeSizeInBits(BackedgeTakenCount->getType()) >
3155 IdxTy->getPrimitiveSizeInBits())
3156 BackedgeTakenCount = SE->getTruncateOrNoop(BackedgeTakenCount, IdxTy);
3157 BackedgeTakenCount = SE->getNoopOrZeroExtend(BackedgeTakenCount, IdxTy);
3158
3159 // Get the total trip count from the count by adding 1.
3160 const SCEV *ExitCount = SE->getAddExpr(
3161 BackedgeTakenCount, SE->getOne(BackedgeTakenCount->getType()));
3162
3163 const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
3164
3165 // Expand the trip count and place the new instructions in the preheader.
3166 // Notice that the pre-header does not change, only the loop body.
3167 SCEVExpander Exp(*SE, DL, "induction");
3168
3169 // Count holds the overall loop count (N).
3170 TripCount = Exp.expandCodeFor(ExitCount, ExitCount->getType(),
3171 L->getLoopPreheader()->getTerminator());
3172
3173 if (TripCount->getType()->isPointerTy())
3174 TripCount =
3175 CastInst::CreatePointerCast(TripCount, IdxTy, "exitcount.ptrcnt.to.int",
3176 L->getLoopPreheader()->getTerminator());
3177
3178 return TripCount;
3179}
3180
3181Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
3182 if (VectorTripCount)
3183 return VectorTripCount;
3184
3185 Value *TC = getOrCreateTripCount(L);
3186 IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
3187
3188 Type *Ty = TC->getType();
3189 // This is where we can make the step a runtime constant.
3190 Value *Step = createStepForVF(Builder, ConstantInt::get(Ty, UF), VF);
3191
3192 // If the tail is to be folded by masking, round the number of iterations N
3193 // up to a multiple of Step instead of rounding down. This is done by first
3194 // adding Step-1 and then rounding down. Note that it's ok if this addition
3195 // overflows: the vector induction variable will eventually wrap to zero given
3196 // that it starts at zero and its Step is a power of two; the loop will then
3197 // exit, with the last early-exit vector comparison also producing all-true.
3198 if (Cost->foldTailByMasking()) {
3199 assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&(static_cast<void> (0))
3200 "VF*UF must be a power of 2 when folding tail by masking")(static_cast<void> (0));
3201 assert(!VF.isScalable() &&(static_cast<void> (0))
3202 "Tail folding not yet supported for scalable vectors")(static_cast<void> (0));
3203 TC = Builder.CreateAdd(
3204 TC, ConstantInt::get(Ty, VF.getKnownMinValue() * UF - 1), "n.rnd.up");
3205 }
3206
3207 // Now we need to generate the expression for the part of the loop that the
3208 // vectorized body will execute. This is equal to N - (N % Step) if scalar
3209 // iterations are not required for correctness, or N - Step, otherwise. Step
3210 // is equal to the vectorization factor (number of SIMD elements) times the
3211 // unroll factor (number of SIMD instructions).
3212 Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
3213
3214 // There are cases where we *must* run at least one iteration in the remainder
3215 // loop. See the cost model for when this can happen. If the step evenly
3216 // divides the trip count, we set the remainder to be equal to the step. If
3217 // the step does not evenly divide the trip count, no adjustment is necessary
3218 // since there will already be scalar iterations. Note that the minimum
3219 // iterations check ensures that N >= Step.
3220 if (Cost->requiresScalarEpilogue(VF)) {
3221 auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
3222 R = Builder.CreateSelect(IsZero, Step, R);
3223 }
3224
3225 VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
3226
3227 return VectorTripCount;
3228}
3229
3230Value *InnerLoopVectorizer::createBitOrPointerCast(Value *V, VectorType *DstVTy,
3231 const DataLayout &DL) {
3232 // Verify that V is a vector type with same number of elements as DstVTy.
3233 auto *DstFVTy = cast<FixedVectorType>(DstVTy);
3234 unsigned VF = DstFVTy->getNumElements();
3235 auto *SrcVecTy = cast<FixedVectorType>(V->getType());
3236 assert((VF == SrcVecTy->getNumElements()) && "Vector dimensions do not match")(static_cast<void> (0));
3237 Type *SrcElemTy = SrcVecTy->getElementType();
3238 Type *DstElemTy = DstFVTy->getElementType();
3239 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&(static_cast<void> (0))
3240 "Vector elements must have same size")(static_cast<void> (0));
3241
3242 // Do a direct cast if element types are castable.
3243 if (CastInst::isBitOrNoopPointerCastable(SrcElemTy, DstElemTy, DL)) {
3244 return Builder.CreateBitOrPointerCast(V, DstFVTy);
3245 }
3246 // V cannot be directly casted to desired vector type.
3247 // May happen when V is a floating point vector but DstVTy is a vector of
3248 // pointers or vice-versa. Handle this using a two-step bitcast using an
3249 // intermediate Integer type for the bitcast i.e. Ptr <-> Int <-> Float.
3250 assert((DstElemTy->isPointerTy() != SrcElemTy->isPointerTy()) &&(static_cast<void> (0))
3251 "Only one type should be a pointer type")(static_cast<void> (0));
3252 assert((DstElemTy->isFloatingPointTy() != SrcElemTy->isFloatingPointTy()) &&(static_cast<void> (0))
3253 "Only one type should be a floating point type")(static_cast<void> (0));
3254 Type *IntTy =
3255 IntegerType::getIntNTy(V->getContext(), DL.getTypeSizeInBits(SrcElemTy));
3256 auto *VecIntTy = FixedVectorType::get(IntTy, VF);
3257 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
3258 return Builder.CreateBitOrPointerCast(CastVal, DstFVTy);
3259}
3260
3261void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
3262 BasicBlock *Bypass) {
3263 Value *Count = getOrCreateTripCount(L);
3264 // Reuse existing vector loop preheader for TC checks.
3265 // Note that new preheader block is generated for vector loop.
3266 BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
3267 IRBuilder<> Builder(TCCheckBlock->getTerminator());
3268
3269 // Generate code to check if the loop's trip count is less than VF * UF, or
3270 // equal to it in case a scalar epilogue is required; this implies that the
3271 // vector trip count is zero. This check also covers the case where adding one
3272 // to the backedge-taken count overflowed leading to an incorrect trip count
3273 // of zero. In this case we will also jump to the scalar loop.
3274 auto P = Cost->requiresScalarEpilogue(VF) ? ICmpInst::ICMP_ULE
3275 : ICmpInst::ICMP_ULT;
3276
3277 // If tail is to be folded, vector loop takes care of all iterations.
3278 Value *CheckMinIters = Builder.getFalse();
3279 if (!Cost->foldTailByMasking()) {
3280 Value *Step =
3281 createStepForVF(Builder, ConstantInt::get(Count->getType(), UF), VF);
3282 CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
3283 }
3284 // Create new preheader for vector loop.
3285 LoopVectorPreHeader =
3286 SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(), DT, LI, nullptr,
3287 "vector.ph");
3288
3289 assert(DT->properlyDominates(DT->getNode(TCCheckBlock),(static_cast<void> (0))
3290 DT->getNode(Bypass)->getIDom()) &&(static_cast<void> (0))
3291 "TC check is expected to dominate Bypass")(static_cast<void> (0));
3292
3293 // Update dominator for Bypass & LoopExit (if needed).
3294 DT->changeImmediateDominator(Bypass, TCCheckBlock);
3295 if (!Cost->requiresScalarEpilogue(VF))
3296 // If there is an epilogue which must run, there's no edge from the
3297 // middle block to exit blocks and thus no need to update the immediate
3298 // dominator of the exit blocks.
3299 DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);
3300
3301 ReplaceInstWithInst(
3302 TCCheckBlock->getTerminator(),
3303 BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
3304 LoopBypassBlocks.push_back(TCCheckBlock);
3305}
3306
3307BasicBlock *InnerLoopVectorizer::emitSCEVChecks(Loop *L, BasicBlock *Bypass) {
3308
3309 BasicBlock *const SCEVCheckBlock =
3310 RTChecks.emitSCEVChecks(L, Bypass, LoopVectorPreHeader, LoopExitBlock);
3311 if (!SCEVCheckBlock)
3312 return nullptr;
3313
3314 assert(!(SCEVCheckBlock->getParent()->hasOptSize() ||(static_cast<void> (0))
3315 (OptForSizeBasedOnProfile &&(static_cast<void> (0))
3316 Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) &&(static_cast<void> (0))
3317 "Cannot SCEV check stride or overflow when optimizing for size")(static_cast<void> (0));
3318
3319
3320 // Update dominator only if this is first RT check.
3321 if (LoopBypassBlocks.empty()) {
3322 DT->changeImmediateDominator(Bypass, SCEVCheckBlock);
3323 if (!Cost->requiresScalarEpilogue(VF))
3324 // If there is an epilogue which must run, there's no edge from the
3325 // middle block to exit blocks and thus no need to update the immediate
3326 // dominator of the exit blocks.
3327 DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
3328 }
3329
3330 LoopBypassBlocks.push_back(SCEVCheckBlock);
3331 AddedSafetyChecks = true;
3332 return SCEVCheckBlock;
3333}
3334
3335BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L,
3336 BasicBlock *Bypass) {
3337 // VPlan-native path does not do any analysis for runtime checks currently.
3338 if (EnableVPlanNativePath)
3339 return nullptr;
3340
3341 BasicBlock *const MemCheckBlock =
3342 RTChecks.emitMemRuntimeChecks(L, Bypass, LoopVectorPreHeader);
3343
3344 // Check if we generated code that checks in runtime if arrays overlap. We put
3345 // the checks into a separate block to make the more common case of few
3346 // elements faster.
3347 if (!MemCheckBlock)
3348 return nullptr;
3349
3350 if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) {
3351 assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&(static_cast<void> (0))
3352 "Cannot emit memory checks when optimizing for size, unless forced "(static_cast<void> (0))
3353 "to vectorize.")(static_cast<void> (0));
3354 ORE->emit([&]() {
3355 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationCodeSize",
3356 L->getStartLoc(), L->getHeader())
3357 << "Code-size may be reduced by not forcing "
3358 "vectorization, or by source-code modifications "
3359 "eliminating the need for runtime checks "
3360 "(e.g., adding 'restrict').";
3361 });
3362 }
3363
3364 LoopBypassBlocks.push_back(MemCheckBlock);
3365
3366 AddedSafetyChecks = true;
3367
3368 // We currently don't use LoopVersioning for the actual loop cloning but we
3369 // still use it to add the noalias metadata.
3370 LVer = std::make_unique<LoopVersioning>(
3371 *Legal->getLAI(),
3372 Legal->getLAI()->getRuntimePointerChecking()->getChecks(), OrigLoop, LI,
3373 DT, PSE.getSE());
3374 LVer->prepareNoAliasMetadata();
3375 return MemCheckBlock;
3376}
3377
3378Value *InnerLoopVectorizer::emitTransformedIndex(
3379 IRBuilder<> &B, Value *Index, ScalarEvolution *SE, const DataLayout &DL,
3380 const InductionDescriptor &ID) const {
3381
3382 SCEVExpander Exp(*SE, DL, "induction");
3383 auto Step = ID.getStep();
3384 auto StartValue = ID.getStartValue();
3385 assert(Index->getType()->getScalarType() == Step->getType() &&(static_cast<void> (0))
3386 "Index scalar type does not match StepValue type")(static_cast<void> (0));
3387
3388 // Note: the IR at this point is broken. We cannot use SE to create any new
3389 // SCEV and then expand it, hoping that SCEV's simplification will give us
3390 // a more optimal code. Unfortunately, attempt of doing so on invalid IR may
3391 // lead to various SCEV crashes. So all we can do is to use builder and rely
3392 // on InstCombine for future simplifications. Here we handle some trivial
3393 // cases only.
3394 auto CreateAdd = [&B](Value *X, Value *Y) {
3395 assert(X->getType() == Y->getType() && "Types don't match!")(static_cast<void> (0));
3396 if (auto *CX = dyn_cast<ConstantInt>(X))
3397 if (CX->isZero())
3398 return Y;
3399 if (auto *CY = dyn_cast<ConstantInt>(Y))
3400 if (CY->isZero())
3401 return X;
3402 return B.CreateAdd(X, Y);
3403 };
3404
3405 // We allow X to be a vector type, in which case Y will potentially be
3406 // splatted into a vector with the same element count.
3407 auto CreateMul = [&B](Value *X, Value *Y) {
3408 assert(X->getType()->getScalarType() == Y->getType() &&(static_cast<void> (0))
3409 "Types don't match!")(static_cast<void> (0));
3410 if (auto *CX = dyn_cast<ConstantInt>(X))
3411 if (CX->isOne())
3412 return Y;
3413 if (auto *CY = dyn_cast<ConstantInt>(Y))
3414 if (CY->isOne())
3415 return X;
3416 VectorType *XVTy = dyn_cast<VectorType>(X->getType());
3417 if (XVTy && !isa<VectorType>(Y->getType()))
3418 Y = B.CreateVectorSplat(XVTy->getElementCount(), Y);
3419 return B.CreateMul(X, Y);
3420 };
3421
3422 // Get a suitable insert point for SCEV expansion. For blocks in the vector
3423 // loop, choose the end of the vector loop header (=LoopVectorBody), because
3424 // the DomTree is not kept up-to-date for additional blocks generated in the
3425 // vector loop. By using the header as insertion point, we guarantee that the
3426 // expanded instructions dominate all their uses.
3427 auto GetInsertPoint = [this, &B]() {
3428 BasicBlock *InsertBB = B.GetInsertPoint()->getParent();
3429 if (InsertBB != LoopVectorBody &&
3430 LI->getLoopFor(LoopVectorBody) == LI->getLoopFor(InsertBB))
3431 return LoopVectorBody->getTerminator();
3432 return &*B.GetInsertPoint();
3433 };
3434
3435 switch (ID.getKind()) {
3436 case InductionDescriptor::IK_IntInduction: {
3437 assert(!isa<VectorType>(Index->getType()) &&(static_cast<void> (0))
3438 "Vector indices not supported for integer inductions yet")(static_cast<void> (0));
3439 assert(Index->getType() == StartValue->getType() &&(static_cast<void> (0))
3440 "Index type does not match StartValue type")(static_cast<void> (0));
3441 if (ID.getConstIntStepValue() && ID.getConstIntStepValue()->isMinusOne())
3442 return B.CreateSub(StartValue, Index);
3443 auto *Offset = CreateMul(
3444 Index, Exp.expandCodeFor(Step, Index->getType(), GetInsertPoint()));
3445 return CreateAdd(StartValue, Offset);
3446 }
3447 case InductionDescriptor::IK_PtrInduction: {
3448 assert(isa<SCEVConstant>(Step) &&(static_cast<void> (0))
3449 "Expected constant step for pointer induction")(static_cast<void> (0));
3450 return B.CreateGEP(
3451 ID.getElementType(), StartValue,
3452 CreateMul(Index,
3453 Exp.expandCodeFor(Step, Index->getType()->getScalarType(),
3454 GetInsertPoint())));
3455 }
3456 case InductionDescriptor::IK_FpInduction: {
3457 assert(!isa<VectorType>(Index->getType()) &&(static_cast<void> (0))
3458 "Vector indices not supported for FP inductions yet")(static_cast<void> (0));
3459 assert(Step->getType()->isFloatingPointTy() && "Expected FP Step value")(static_cast<void> (0));
3460 auto InductionBinOp = ID.getInductionBinOp();
3461 assert(InductionBinOp &&(static_cast<void> (0))
3462 (InductionBinOp->getOpcode() == Instruction::FAdd ||(static_cast<void> (0))
3463 InductionBinOp->getOpcode() == Instruction::FSub) &&(static_cast<void> (0))
3464 "Original bin op should be defined for FP induction")(static_cast<void> (0));
3465
3466 Value *StepValue = cast<SCEVUnknown>(Step)->getValue();
3467 Value *MulExp = B.CreateFMul(StepValue, Index);
3468 return B.CreateBinOp(InductionBinOp->getOpcode(), StartValue, MulExp,
3469 "induction");
3470 }
3471 case InductionDescriptor::IK_NoInduction:
3472 return nullptr;
3473 }
3474 llvm_unreachable("invalid enum")__builtin_unreachable();
3475}
3476
3477Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
3478 LoopScalarBody = OrigLoop->getHeader();
3479 LoopVectorPreHeader = OrigLoop->getLoopPreheader();
3480 assert(LoopVectorPreHeader && "Invalid loop structure")(static_cast<void> (0));
3481 LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr
3482 assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF)) &&(static_cast<void> (0))
3483 "multiple exit loop without required epilogue?")(static_cast<void> (0));
3484
3485 LoopMiddleBlock =
3486 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3487 LI, nullptr, Twine(Prefix) + "middle.block");
3488 LoopScalarPreHeader =
3489 SplitBlock(LoopMiddleBlock, LoopMiddleBlock->getTerminator(), DT, LI,
3490 nullptr, Twine(Prefix) + "scalar.ph");
3491
3492 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
3493
3494 // Set up the middle block terminator. Two cases:
3495 // 1) If we know that we must execute the scalar epilogue, emit an
3496 // unconditional branch.
3497 // 2) Otherwise, we must have a single unique exit block (due to how we
3498 // implement the multiple exit case). In this case, set up a conditonal
3499 // branch from the middle block to the loop scalar preheader, and the
3500 // exit block. completeLoopSkeleton will update the condition to use an
3501 // iteration check, if required to decide whether to execute the remainder.
3502 BranchInst *BrInst = Cost->requiresScalarEpilogue(VF) ?
3503 BranchInst::Create(LoopScalarPreHeader) :
3504 BranchInst::Create(LoopExitBlock, LoopScalarPreHeader,
3505 Builder.getTrue());
3506 BrInst->setDebugLoc(ScalarLatchTerm->getDebugLoc());
3507 ReplaceInstWithInst(LoopMiddleBlock->getTerminator(), BrInst);
3508
3509 // We intentionally don't let SplitBlock to update LoopInfo since
3510 // LoopVectorBody should belong to another loop than LoopVectorPreHeader.
3511 // LoopVectorBody is explicitly added to the correct place few lines later.
3512 LoopVectorBody =
3513 SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
3514 nullptr, nullptr, Twine(Prefix) + "vector.body");
3515
3516 // Update dominator for loop exit.
3517 if (!Cost->requiresScalarEpilogue(VF))
3518 // If there is an epilogue which must run, there's no edge from the
3519 // middle block to exit blocks and thus no need to update the immediate
3520 // dominator of the exit blocks.
3521 DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);
3522
3523 // Create and register the new vector loop.
3524 Loop *Lp = LI->AllocateLoop();
3525 Loop *ParentLoop = OrigLoop->getParentLoop();
3526
3527 // Insert the new loop into the loop nest and register the new basic blocks
3528 // before calling any utilities such as SCEV that require valid LoopInfo.
3529 if (ParentLoop) {
3530 ParentLoop->addChildLoop(Lp);
3531 } else {
3532 LI->addTopLevelLoop(Lp);
3533 }
3534 Lp->addBasicBlockToLoop(LoopVectorBody, *LI);
3535 return Lp;
3536}
3537
3538void InnerLoopVectorizer::createInductionResumeValues(
3539 Loop *L, Value *VectorTripCount,
3540 std::pair<BasicBlock *, Value *> AdditionalBypass) {
3541 assert(VectorTripCount && L && "Expected valid arguments")(static_cast<void> (0));
3542 assert(((AdditionalBypass.first && AdditionalBypass.second) ||(static_cast<void> (0))
3543 (!AdditionalBypass.first && !AdditionalBypass.second)) &&(static_cast<void> (0))
3544 "Inconsistent information about additional bypass.")(static_cast<void> (0));
3545 // We are going to resume the execution of the scalar loop.
3546 // Go over all of the induction variables that we found and fix the
3547 // PHIs that are left in the scalar version of the loop.
3548 // The starting values of PHI nodes depend on the counter of the last
3549 // iteration in the vectorized loop.
3550 // If we come from a bypass edge then we need to start from the original
3551 // start value.
3552 for (auto &InductionEntry : Legal->getInductionVars()) {
3553 PHINode *OrigPhi = InductionEntry.first;
3554 InductionDescriptor II = InductionEntry.second;
3555
3556 // Create phi nodes to merge from the backedge-taken check block.
3557 PHINode *BCResumeVal =
3558 PHINode::Create(OrigPhi->getType(), 3, "bc.resume.val",
3559 LoopScalarPreHeader->getTerminator());
3560 // Copy original phi DL over to the new one.
3561 BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
3562 Value *&EndValue = IVEndValues[OrigPhi];
3563 Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
3564 if (OrigPhi == OldInduction) {
3565 // We know what the end value is.
3566 EndValue = VectorTripCount;
3567 } else {
3568 IRBuilder<> B(L->getLoopPreheader()->getTerminator());
3569
3570 // Fast-math-flags propagate from the original induction instruction.
3571 if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
3572 B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());
3573
3574 Type *StepType = II.getStep()->getType();
3575 Instruction::CastOps CastOp =
3576 CastInst::getCastOpcode(VectorTripCount, true, StepType, true);
3577 Value *CRD = B.CreateCast(CastOp, VectorTripCount, StepType, "cast.crd");
3578 const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout();
3579 EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
3580 EndValue->setName("ind.end");
3581
3582 // Compute the end value for the additional bypass (if applicable).
3583 if (AdditionalBypass.first) {
3584 B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt()));
3585 CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true,
3586 StepType, true);
3587 CRD =
3588 B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd");
3589 EndValueFromAdditionalBypass =
3590 emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
3591 EndValueFromAdditionalBypass->setName("ind.end");
3592 }
3593 }
3594 // The new PHI merges the original incoming value, in case of a bypass,
3595 // or the value at the end of the vectorized loop.
3596 BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
3597
3598 // Fix the scalar body counter (PHI node).
3599 // The old induction's phi node in the scalar body needs the truncated
3600 // value.
3601 for (BasicBlock *BB : LoopBypassBlocks)
3602 BCResumeVal->addIncoming(II.getStartValue(), BB);
3603
3604 if (AdditionalBypass.first)
3605 BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
3606 EndValueFromAdditionalBypass);
3607
3608 OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
3609 }
3610}
3611
3612BasicBlock *InnerLoopVectorizer::completeLoopSkeleton(Loop *L,
3613 MDNode *OrigLoopID) {
3614 assert(L && "Expected valid loop.")(static_cast<void> (0));
3615
3616 // The trip counts should be cached by now.
3617 Value *Count = getOrCreateTripCount(L);
3618 Value *VectorTripCount = getOrCreateVectorTripCount(L);
3619
3620 auto *ScalarLatchTerm = OrigLoop->getLoopLatch()->getTerminator();
3621
3622 // Add a check in the middle block to see if we have completed
3623 // all of the iterations in the first vector loop. Three cases:
3624 // 1) If we require a scalar epilogue, there is no conditional branch as
3625 // we unconditionally branch to the scalar preheader. Do nothing.
3626 // 2) If (N - N%VF) == N, then we *don't* need to run the remainder.
3627 // Thus if tail is to be folded, we know we don't need to run the
3628 // remainder and we can use the previous value for the condition (true).
3629 // 3) Otherwise, construct a runtime check.
3630 if (!Cost->requiresScalarEpilogue(VF) && !Cost->foldTailByMasking()) {
3631 Instruction *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ,
3632 Count, VectorTripCount, "cmp.n",
3633 LoopMiddleBlock->getTerminator());
3634
3635 // Here we use the same DebugLoc as the scalar loop latch terminator instead
3636 // of the corresponding compare because they may have ended up with
3637 // different line numbers and we want to avoid awkward line stepping while
3638 // debugging. Eg. if the compare has got a line number inside the loop.
3639 CmpN->setDebugLoc(ScalarLatchTerm->getDebugLoc());
3640 cast<BranchInst>(LoopMiddleBlock->getTerminator())->setCondition(CmpN);
3641 }
3642
3643 // Get ready to start creating new instructions into the vectorized body.
3644 assert(LoopVectorPreHeader == L->getLoopPreheader() &&(static_cast<void> (0))
3645 "Inconsistent vector loop preheader")(static_cast<void> (0));
3646 Builder.SetInsertPoint(&*LoopVectorBody->getFirstInsertionPt());
3647
3648 Optional<MDNode *> VectorizedLoopID =
3649 makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
3650 LLVMLoopVectorizeFollowupVectorized});
3651 if (VectorizedLoopID.hasValue()) {
3652 L->setLoopID(VectorizedLoopID.getValue());
3653
3654 // Do not setAlreadyVectorized if loop attributes have been defined
3655 // explicitly.
3656 return LoopVectorPreHeader;
3657 }
3658
3659 // Keep all loop hints from the original loop on the vector loop (we'll
3660 // replace the vectorizer-specific hints below).
3661 if (MDNode *LID = OrigLoop->getLoopID())
3662 L->setLoopID(LID);
3663
3664 LoopVectorizeHints Hints(L, true, *ORE);
3665 Hints.setAlreadyVectorized();
3666
3667#ifdef EXPENSIVE_CHECKS
3668 assert(DT->verify(DominatorTree::VerificationLevel::Fast))(static_cast<void> (0));
3669 LI->verify(*DT);
3670#endif
3671
3672 return LoopVectorPreHeader;
3673}
3674
3675BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
3676 /*
3677 In this function we generate a new loop. The new loop will contain
3678 the vectorized instructions while the old loop will continue to run the
3679 scalar remainder.
3680
3681 [ ] <-- loop iteration number check.
3682 / |
3683 / v
3684 | [ ] <-- vector loop bypass (may consist of multiple blocks).
3685 | / |
3686 | / v
3687 || [ ] <-- vector pre header.
3688 |/ |
3689 | v
3690 | [ ] \
3691 | [ ]_| <-- vector loop.
3692 | |
3693 | v
3694 \ -[ ] <--- middle-block.
3695 \/ |
3696 /\ v
3697 | ->[ ] <--- new preheader.
3698 | |
3699 (opt) v <-- edge from middle to exit iff epilogue is not required.
3700 | [ ] \
3701 | [ ]_| <-- old scalar loop to handle remainder (scalar epilogue).
3702 \ |
3703 \ v
3704 >[ ] <-- exit block(s).
3705 ...
3706 */
3707
3708 // Get the metadata of the original loop before it gets modified.
3709 MDNode *OrigLoopID = OrigLoop->getLoopID();
3710
3711 // Workaround! Compute the trip count of the original loop and cache it
3712 // before we start modifying the CFG. This code has a systemic problem
3713 // wherein it tries to run analysis over partially constructed IR; this is
3714 // wrong, and not simply for SCEV. The trip count of the original loop
3715 // simply happens to be prone to hitting this in practice. In theory, we
3716 // can hit the same issue for any SCEV, or ValueTracking query done during
3717 // mutation. See PR49900.
3718 getOrCreateTripCount(OrigLoop);
3719
3720 // Create an empty vector loop, and prepare basic blocks for the runtime
3721 // checks.
3722 Loop *Lp = createVectorLoopSkeleton("");
3723
3724 // Now, compare the new count to zero. If it is zero skip the vector loop and
3725 // jump to the scalar loop. This check also covers the case where the
3726 // backedge-taken count is uint##_max: adding one to it will overflow leading
3727 // to an incorrect trip count of zero. In this (rare) case we will also jump
3728 // to the scalar loop.
3729 emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader);
3730
3731 // Generate the code to check any assumptions that we've made for SCEV
3732 // expressions.
3733 emitSCEVChecks(Lp, LoopScalarPreHeader);
3734
3735 // Generate the code that checks in runtime if arrays overlap. We put the
3736 // checks into a separate block to make the more common case of few elements
3737 // faster.
3738 emitMemRuntimeChecks(Lp, LoopScalarPreHeader);
3739
3740 // Some loops have a single integer induction variable, while other loops
3741 // don't. One example is c++ iterators that often have multiple pointer
3742 // induction variables. In the code below we also support a case where we
3743 // don't have a single induction variable.
3744 //
3745 // We try to obtain an induction variable from the original loop as hard
3746 // as possible. However if we don't find one that:
3747 // - is an integer
3748 // - counts from zero, stepping by one
3749 // - is the size of the widest induction variable type
3750 // then we create a new one.
3751 OldInduction = Legal->getPrimaryInduction();
3752 Type *IdxTy = Legal->getWidestInductionType();
3753 Value *StartIdx = ConstantInt::get(IdxTy, 0);
3754 // The loop step is equal to the vectorization factor (num of SIMD elements)
3755 // times the unroll factor (num of SIMD instructions).
3756 Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt());
3757 Value *Step = createStepForVF(Builder, ConstantInt::get(IdxTy, UF), VF);
3758 Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
3759 Induction =
3760 createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
3761 getDebugLocFromInstOrOperands(OldInduction));
3762
3763 // Emit phis for the new starting index of the scalar loop.
3764 createInductionResumeValues(Lp, CountRoundDown);
3765
3766 return completeLoopSkeleton(Lp, OrigLoopID);
3767}
3768
3769// Fix up external users of the induction variable. At this point, we are
3770// in LCSSA form, with all external PHIs that use the IV having one input value,
3771// coming from the remainder loop. We need those PHIs to also have a correct
3772// value for the IV when arriving directly from the middle block.
3773void InnerLoopVectorizer::fixupIVUsers(PHINode *OrigPhi,
3774 const InductionDescriptor &II,
3775 Value *CountRoundDown, Value *EndValue,
3776 BasicBlock *MiddleBlock) {
3777 // There are two kinds of external IV usages - those that use the value
3778 // computed in the last iteration (the PHI) and those that use the penultimate
3779 // value (the value that feeds into the phi from the loop latch).
3780 // We allow both, but they, obviously, have different values.
3781
3782 assert(OrigLoop->getUniqueExitBlock() && "Expected a single exit block")(static_cast<void> (0));
3783
3784 DenseMap<Value *, Value *> MissingVals;
3785
3786 // An external user of the last iteration's value should see the value that
3787 // the remainder loop uses to initialize its own IV.
3788 Value *PostInc = OrigPhi->getIncomingValueForBlock(OrigLoop->getLoopLatch());
3789 for (User *U : PostInc->users()) {
3790 Instruction *UI = cast<Instruction>(U);
3791 if (!OrigLoop->contains(UI)) {
3792 assert(isa<PHINode>(UI) && "Expected LCSSA form")(static_cast<void> (0));
3793 MissingVals[UI] = EndValue;
3794 }
3795 }
3796
3797 // An external user of the penultimate value need to see EndValue - Step.
3798 // The simplest way to get this is to recompute it from the constituent SCEVs,
3799 // that is Start + (Step * (CRD - 1)).
3800 for (User *U : OrigPhi->users()) {
3801 auto *UI = cast<Instruction>(U);
3802 if (!OrigLoop->contains(UI)) {
3803 const DataLayout &DL =
3804 OrigLoop->getHeader()->getModule()->getDataLayout();
3805 assert(isa<PHINode>(UI) && "Expected LCSSA form")(static_cast<void> (0));
3806
3807 IRBuilder<> B(MiddleBlock->getTerminator());
3808
3809 // Fast-math-flags propagate from the original induction instruction.
3810 if (II.getInductionBinOp() && isa<FPMathOperator>(II.getInductionBinOp()))
3811 B.setFastMathFlags(II.getInductionBinOp()->getFastMathFlags());
3812
3813 Value *CountMinusOne = B.CreateSub(
3814 CountRoundDown, ConstantInt::get(CountRoundDown->getType(), 1));
3815 Value *CMO =
3816 !II.getStep()->getType()->isIntegerTy()
3817 ? B.CreateCast(Instruction::SIToFP, CountMinusOne,
3818 II.getStep()->getType())
3819 : B.CreateSExtOrTrunc(CountMinusOne, II.getStep()->getType());
3820 CMO->setName("cast.cmo");
3821 Value *Escape = emitTransformedIndex(B, CMO, PSE.getSE(), DL, II);
3822 Escape->setName("ind.escape");
3823 MissingVals[UI] = Escape;
3824 }
3825 }
3826
3827 for (auto &I : MissingVals) {
3828 PHINode *PHI = cast<PHINode>(I.first);
3829 // One corner case we have to handle is two IVs "chasing" each-other,
3830 // that is %IV2 = phi [...], [ %IV1, %latch ]
3831 // In this case, if IV1 has an external use, we need to avoid adding both
3832 // "last value of IV1" and "penultimate value of IV2". So, verify that we
3833 // don't already have an incoming value for the middle block.
3834 if (PHI->getBasicBlockIndex(MiddleBlock) == -1)
3835 PHI->addIncoming(I.second, MiddleBlock);
3836 }
3837}
3838
3839namespace {
3840
3841struct CSEDenseMapInfo {
3842 static bool canHandle(const Instruction *I) {
3843 return isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
3844 isa<ShuffleVectorInst>(I) || isa<GetElementPtrInst>(I);
3845 }
3846
3847 static inline Instruction *getEmptyKey() {
3848 return DenseMapInfo<Instruction *>::getEmptyKey();
3849 }
3850
3851 static inline Instruction *getTombstoneKey() {
3852 return DenseMapInfo<Instruction *>::getTombstoneKey();
3853 }
3854
3855 static unsigned getHashValue(const Instruction *I) {
3856 assert(canHandle(I) && "Unknown instruction!")(static_cast<void> (0));
3857 return hash_combine(I->getOpcode(), hash_combine_range(I->value_op_begin(),
3858 I->value_op_end()));
3859 }
3860
3861 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
3862 if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
3863 LHS == getTombstoneKey() || RHS == getTombstoneKey())
3864 return LHS == RHS;
3865 return LHS->isIdenticalTo(RHS);
3866 }
3867};
3868
3869} // end anonymous namespace
3870
3871///Perform cse of induction variable instructions.
3872static void cse(BasicBlock *BB) {
3873 // Perform simple cse.
3874 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
3875 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
3876 Instruction *In = &*I++;
3877
3878 if (!CSEDenseMapInfo::canHandle(In))
3879 continue;
3880
3881 // Check if we can replace this instruction with any of the
3882 // visited instructions.
3883 if (Instruction *V = CSEMap.lookup(In)) {
3884 In->replaceAllUsesWith(V);
3885 In->eraseFromParent();
3886 continue;
3887 }
3888
3889 CSEMap[In] = In;
3890 }
3891}
3892
3893InstructionCost
3894LoopVectorizationCostModel::getVectorCallCost(CallInst *CI, ElementCount VF,
3895 bool &NeedToScalarize) const {
3896 Function *F = CI->getCalledFunction();
3897 Type *ScalarRetTy = CI->getType();
3898 SmallVector<Type *, 4> Tys, ScalarTys;
3899 for (auto &ArgOp : CI->arg_operands())
3900 ScalarTys.push_back(ArgOp->getType());
3901
3902 // Estimate cost of scalarized vector call. The source operands are assumed
3903 // to be vectors, so we need to extract individual elements from there,
3904 // execute VF scalar calls, and then gather the result into the vector return
3905 // value.
3906 InstructionCost ScalarCallCost =
3907 TTI.getCallInstrCost(F, ScalarRetTy, ScalarTys, TTI::TCK_RecipThroughput);
3908 if (VF.isScalar())
3909 return ScalarCallCost;
3910
3911 // Compute corresponding vector type for return value and arguments.
3912 Type *RetTy = ToVectorTy(ScalarRetTy, VF);
3913 for (Type *ScalarTy : ScalarTys)
3914 Tys.push_back(ToVectorTy(ScalarTy, VF));
3915
3916 // Compute costs of unpacking argument values for the scalar calls and
3917 // packing the return values to a vector.
3918 InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
3919
3920 InstructionCost Cost =
3921 ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
3922
3923 // If we can't emit a vector call for this function, then the currently found
3924 // cost is the cost we need to return.
3925 NeedToScalarize = true;
3926 VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
3927 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3928
3929 if (!TLI || CI->isNoBuiltin() || !VecFunc)
3930 return Cost;
3931
3932 // If the corresponding vector cost is cheaper, return its cost.
3933 InstructionCost VectorCallCost =
3934 TTI.getCallInstrCost(nullptr, RetTy, Tys, TTI::TCK_RecipThroughput);
3935 if (VectorCallCost < Cost) {
3936 NeedToScalarize = false;
3937 Cost = VectorCallCost;
3938 }
3939 return Cost;
3940}
3941
3942static Type *MaybeVectorizeType(Type *Elt, ElementCount VF) {
3943 if (VF.isScalar() || (!Elt->isIntOrPtrTy() && !Elt->isFloatingPointTy()))
3944 return Elt;
3945 return VectorType::get(Elt, VF);
3946}
3947
3948InstructionCost
3949LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
3950 ElementCount VF) const {
3951 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3952 assert(ID && "Expected intrinsic call!")(static_cast<void> (0));
3953 Type *RetTy = MaybeVectorizeType(CI->getType(), VF);
3954 FastMathFlags FMF;
3955 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
3956 FMF = FPMO->getFastMathFlags();
3957
3958 SmallVector<const Value *> Arguments(CI->arg_begin(), CI->arg_end());
3959 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
3960 SmallVector<Type *> ParamTys;
3961 std::transform(FTy->param_begin(), FTy->param_end(),
3962 std::back_inserter(ParamTys),
3963 [&](Type *Ty) { return MaybeVectorizeType(Ty, VF); });
3964
3965 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
3966 dyn_cast<IntrinsicInst>(CI));
3967 return TTI.getIntrinsicInstrCost(CostAttrs,
3968 TargetTransformInfo::TCK_RecipThroughput);
3969}
3970
3971static Type *smallestIntegerVectorType(Type *T1, Type *T2) {
3972 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3973 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3974 return I1->getBitWidth() < I2->getBitWidth() ? T1 : T2;
3975}
3976
3977static Type *largestIntegerVectorType(Type *T1, Type *T2) {
3978 auto *I1 = cast<IntegerType>(cast<VectorType>(T1)->getElementType());
3979 auto *I2 = cast<IntegerType>(cast<VectorType>(T2)->getElementType());
3980 return I1->getBitWidth() > I2->getBitWidth() ? T1 : T2;
3981}
3982
3983void InnerLoopVectorizer::truncateToMinimalBitwidths(VPTransformState &State) {
3984 // For every instruction `I` in MinBWs, truncate the operands, create a
3985 // truncated version of `I` and reextend its result. InstCombine runs
3986 // later and will remove any ext/trunc pairs.
3987 SmallPtrSet<Value *, 4> Erased;
3988 for (const auto &KV : Cost->getMinimalBitwidths()) {
3989 // If the value wasn't vectorized, we must maintain the original scalar
3990 // type. The absence of the value from State indicates that it
3991 // wasn't vectorized.
3992 // FIXME: Should not rely on getVPValue at this point.
3993 VPValue *Def = State.Plan->getVPValue(KV.first, true);
3994 if (!State.hasAnyVectorValue(Def))
3995 continue;
3996 for (unsigned Part = 0; Part < UF; ++Part) {
3997 Value *I = State.get(Def, Part);
3998 if (Erased.count(I) || I->use_empty() || !isa<Instruction>(I))
3999 continue;
4000 Type *OriginalTy = I->getType();
4001 Type *ScalarTruncatedTy =
4002 IntegerType::get(OriginalTy->getContext(), KV.second);
4003 auto *TruncatedTy = VectorType::get(
4004 ScalarTruncatedTy, cast<VectorType>(OriginalTy)->getElementCount());
4005 if (TruncatedTy == OriginalTy)
4006 continue;
4007
4008 IRBuilder<> B(cast<Instruction>(I));
4009 auto ShrinkOperand = [&](Value *V) -> Value * {
4010 if (auto *ZI = dyn_cast<ZExtInst>(V))
4011 if (ZI->getSrcTy() == TruncatedTy)
4012 return ZI->getOperand(0);
4013 return B.CreateZExtOrTrunc(V, TruncatedTy);
4014 };
4015
4016 // The actual instruction modification depends on the instruction type,
4017 // unfortunately.
4018 Value *NewI = nullptr;
4019 if (auto *BO = dyn_cast<BinaryOperator>(I)) {
4020 NewI = B.CreateBinOp(BO->getOpcode(), ShrinkOperand(BO->getOperand(0)),
4021 ShrinkOperand(BO->getOperand(1)));
4022
4023 // Any wrapping introduced by shrinking this operation shouldn't be
4024 // considered undefined behavior. So, we can't unconditionally copy
4025 // arithmetic wrapping flags to NewI.
4026 cast<BinaryOperator>(NewI)->copyIRFlags(I, /*IncludeWrapFlags=*/false);
4027 } else if (auto *CI = dyn_cast<ICmpInst>(I)) {
4028 NewI =
4029 B.CreateICmp(CI->getPredicate(), ShrinkOperand(CI->getOperand(0)),
4030 ShrinkOperand(CI->getOperand(1)));
4031 } else if (auto *SI = dyn_cast<SelectInst>(I)) {
4032 NewI = B.CreateSelect(SI->getCondition(),
4033 ShrinkOperand(SI->getTrueValue()),
4034 ShrinkOperand(SI->getFalseValue()));
4035 } else if (auto *CI = dyn_cast<CastInst>(I)) {
4036 switch (CI->getOpcode()) {
4037 default:
4038 llvm_unreachable("Unhandled cast!")__builtin_unreachable();
4039 case Instruction::Trunc:
4040 NewI = ShrinkOperand(CI->getOperand(0));
4041 break;
4042 case Instruction::SExt:
4043 NewI = B.CreateSExtOrTrunc(
4044 CI->getOperand(0),
4045 smallestIntegerVectorType(OriginalTy, TruncatedTy));
4046 break;
4047 case Instruction::ZExt:
4048 NewI = B.CreateZExtOrTrunc(
4049 CI->getOperand(0),
4050 smallestIntegerVectorType(OriginalTy, TruncatedTy));
4051 break;
4052 }
4053 } else if (auto *SI = dyn_cast<ShuffleVectorInst>(I)) {
4054 auto Elements0 =
4055 cast<VectorType>(SI->getOperand(0)->getType())->getElementCount();
4056 auto *O0 = B.CreateZExtOrTrunc(
4057 SI->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements0));
4058 auto Elements1 =
4059 cast<VectorType>(SI->getOperand(1)->getType())->getElementCount();
4060 auto *O1 = B.CreateZExtOrTrunc(
4061 SI->getOperand(1), VectorType::get(ScalarTruncatedTy, Elements1));
4062
4063 NewI = B.CreateShuffleVector(O0, O1, SI->getShuffleMask());
4064 } else if (isa<LoadInst>(I) || isa<PHINode>(I)) {
4065 // Don't do anything with the operands, just extend the result.
4066 continue;
4067 } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
4068 auto Elements =
4069 cast<VectorType>(IE->getOperand(0)->getType())->getElementCount();
4070 auto *O0 = B.CreateZExtOrTrunc(
4071 IE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));
4072 auto *O1 = B.CreateZExtOrTrunc(IE->getOperand(1), ScalarTruncatedTy);
4073 NewI = B.CreateInsertElement(O0, O1, IE->getOperand(2));
4074 } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
4075 auto Elements =
4076 cast<VectorType>(EE->getOperand(0)->getType())->getElementCount();
4077 auto *O0 = B.CreateZExtOrTrunc(
4078 EE->getOperand(0), VectorType::get(ScalarTruncatedTy, Elements));
4079 NewI = B.CreateExtractElement(O0, EE->getOperand(2));
4080 } else {
4081 // If we don't know what to do, be conservative and don't do anything.
4082 continue;
4083 }
4084
4085 // Lastly, extend the result.
4086 NewI->takeName(cast<Instruction>(I));
4087 Value *Res = B.CreateZExtOrTrunc(NewI, OriginalTy);
4088 I->replaceAllUsesWith(Res);
4089 cast<Instruction>(I)->eraseFromParent();
4090 Erased.insert(I);
4091 State.reset(Def, Res, Part);
4092 }
4093 }
4094
4095 // We'll have created a bunch of ZExts that are now parentless. Clean up.
4096 for (const auto &KV : Cost->getMinimalBitwidths()) {
4097 // If the value wasn't vectorized, we must maintain the original scalar
4098 // type. The absence of the value from State indicates that it
4099 // wasn't vectorized.
4100 // FIXME: Should not rely on getVPValue at this point.
4101 VPValue *Def = State.Plan->getVPValue(KV.first, true);
4102 if (!State.hasAnyVectorValue(Def))
4103 continue;
4104 for (unsigned Part = 0; Part < UF; ++Part) {
4105 Value *I = State.get(Def, Part);
4106 ZExtInst *Inst = dyn_cast<ZExtInst>(I);
4107 if (Inst && Inst->use_empty()) {
4108 Value *NewI = Inst->getOperand(0);
4109 Inst->eraseFromParent();
4110 State.reset(Def, NewI, Part);
4111 }
4112 }
4113 }
4114}
4115
4116void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
4117 // Insert truncates and extends for any truncated instructions as hints to
4118 // InstCombine.
4119 if (VF.isVector())
4120 truncateToMinimalBitwidths(State);
4121
4122 // Fix widened non-induction PHIs by setting up the PHI operands.
4123 if (OrigPHIsToFix.size()) {
4124 assert(EnableVPlanNativePath &&(static_cast<void> (0))
4125 "Unexpected non-induction PHIs for fixup in non VPlan-native path")(static_cast<void> (0));
4126 fixNonInductionPHIs(State);
4127 }
4128
4129 // At this point every instruction in the original loop is widened to a
4130 // vector form. Now we need to fix the recurrences in the loop. These PHI
4131 // nodes are currently empty because we did not want to introduce cycles.
4132 // This is the second stage of vectorizing recurrences.
4133 fixCrossIterationPHIs(State);
4134
4135 // Forget the original basic block.
4136 PSE.getSE()->forgetLoop(OrigLoop);
4137
4138 // If we inserted an edge from the middle block to the unique exit block,
4139 // update uses outside the loop (phis) to account for the newly inserted
4140 // edge.
4141 if (!Cost->requiresScalarEpilogue(VF)) {
4142 // Fix-up external users of the induction variables.
4143 for (auto &Entry : Legal->getInductionVars())
4144 fixupIVUsers(Entry.first, Entry.second,
4145 getOrCreateVectorTripCount(LI->getLoopFor(LoopVectorBody)),
4146 IVEndValues[Entry.first], LoopMiddleBlock);
4147
4148 fixLCSSAPHIs(State);
4149 }
4150
4151 for (Instruction *PI : PredicatedInstructions)
4152 sinkScalarOperands(&*PI);
4153
4154 // Remove redundant induction instructions.
4155 cse(LoopVectorBody);
4156
4157 // Set/update profile weights for the vector and remainder loops as original
4158 // loop iterations are now distributed among them. Note that original loop
4159 // represented by LoopScalarBody becomes remainder loop after vectorization.
4160 //
4161 // For cases like foldTailByMasking() and requiresScalarEpiloque() we may
4162 // end up getting slightly roughened result but that should be OK since
4163 // profile is not inherently precise anyway. Note also possible bypass of
4164 // vector code caused by legality checks is ignored, assigning all the weight
4165 // to the vector loop, optimistically.
4166 //
4167 // For scalable vectorization we can't know at compile time how many iterations
4168 // of the loop are handled in one vector iteration, so instead assume a pessimistic
4169 // vscale of '1'.
4170 setProfileInfoAfterUnrolling(
4171 LI->getLoopFor(LoopScalarBody), LI->getLoopFor(LoopVectorBody),
4172 LI->getLoopFor(LoopScalarBody), VF.getKnownMinValue() * UF);
4173}
4174
4175void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
4176 // In order to support recurrences we need to be able to vectorize Phi nodes.
4177 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
4178 // stage #2: We now need to fix the recurrences by adding incoming edges to
4179 // the currently empty PHI nodes. At this point every instruction in the
4180 // original loop is widened to a vector form so we can use them to construct
4181 // the incoming edges.
4182 VPBasicBlock *Header = State.Plan->getEntry()->getEntryBasicBlock();
4183 for (VPRecipeBase &R : Header->phis()) {
4184 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R))
4185 fixReduction(ReductionPhi, State);
4186 else if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
4187 fixFirstOrderRecurrence(FOR, State);
4188 }
4189}
4190
4191void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR,
4192 VPTransformState &State) {
4193 // This is the second phase of vectorizing first-order recurrences. An
4194 // overview of the transformation is described below. Suppose we have the
4195 // following loop.
4196 //
4197 // for (int i = 0; i < n; ++i)
4198 // b[i] = a[i] - a[i - 1];
4199 //
4200 // There is a first-order recurrence on "a". For this loop, the shorthand
4201 // scalar IR looks like:
4202 //
4203 // scalar.ph:
4204 // s_init = a[-1]
4205 // br scalar.body
4206 //
4207 // scalar.body:
4208 // i = phi [0, scalar.ph], [i+1, scalar.body]
4209 // s1 = phi [s_init, scalar.ph], [s2, scalar.body]
4210 // s2 = a[i]
4211 // b[i] = s2 - s1
4212 // br cond, scalar.body, ...
4213 //
4214 // In this example, s1 is a recurrence because it's value depends on the
4215 // previous iteration. In the first phase of vectorization, we created a
4216 // vector phi v1 for s1. We now complete the vectorization and produce the
4217 // shorthand vector IR shown below (for VF = 4, UF = 1).
4218 //
4219 // vector.ph:
4220 // v_init = vector(..., ..., ..., a[-1])
4221 // br vector.body
4222 //
4223 // vector.body
4224 // i = phi [0, vector.ph], [i+4, vector.body]
4225 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4226 // v2 = a[i, i+1, i+2, i+3];
4227 // v3 = vector(v1(3), v2(0, 1, 2))
4228 // b[i, i+1, i+2, i+3] = v2 - v3
4229 // br cond, vector.body, middle.block
4230 //
4231 // middle.block:
4232 // x = v2(3)
4233 // br scalar.ph
4234 //
4235 // scalar.ph:
4236 // s_init = phi [x, middle.block], [a[-1], otherwise]
4237 // br scalar.body
4238 //
4239 // After execution completes the vector loop, we extract the next value of
4240 // the recurrence (x) to use as the initial value in the scalar loop.
4241
4242 // Extract the last vector element in the middle block. This will be the
4243 // initial value for the recurrence when jumping to the scalar loop.
4244 VPValue *PreviousDef = PhiR->getBackedgeValue();
4245 Value *Incoming = State.get(PreviousDef, UF - 1);
4246 auto *ExtractForScalar = Incoming;
4247 auto *IdxTy = Builder.getInt32Ty();
4248 if (VF.isVector()) {
4249 auto *One = ConstantInt::get(IdxTy, 1);
4250 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
4251 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF);
4252 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
4253 ExtractForScalar = Builder.CreateExtractElement(ExtractForScalar, LastIdx,
4254 "vector.recur.extract");
4255 }
4256 // Extract the second last element in the middle block if the
4257 // Phi is used outside the loop. We need to extract the phi itself
4258 // and not the last element (the phi update in the current iteration). This
4259 // will be the value when jumping to the exit block from the LoopMiddleBlock,
4260 // when the scalar loop is not run at all.
4261 Value *ExtractForPhiUsedOutsideLoop = nullptr;
4262 if (VF.isVector()) {
4263 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, VF);
4264 auto *Idx = Builder.CreateSub(RuntimeVF, ConstantInt::get(IdxTy, 2));
4265 ExtractForPhiUsedOutsideLoop = Builder.CreateExtractElement(
4266 Incoming, Idx, "vector.recur.extract.for.phi");
4267 } else if (UF > 1)
4268 // When loop is unrolled without vectorizing, initialize
4269 // ExtractForPhiUsedOutsideLoop with the value just prior to unrolled value
4270 // of `Incoming`. This is analogous to the vectorized case above: extracting
4271 // the second last element when VF > 1.
4272 ExtractForPhiUsedOutsideLoop = State.get(PreviousDef, UF - 2);
4273
4274 // Fix the initial value of the original recurrence in the scalar loop.
4275 Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
4276 PHINode *Phi = cast<PHINode>(PhiR->getUnderlyingValue());
4277 auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
4278 auto *ScalarInit = PhiR->getStartValue()->getLiveInIRValue();
4279 for (auto *BB : predecessors(LoopScalarPreHeader)) {
4280 auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
4281 Start->addIncoming(Incoming, BB);
4282 }
4283
4284 Phi->setIncomingValueForBlock(LoopScalarPreHeader, Start);
4285 Phi->setName("scalar.recur");
4286
4287 // Finally, fix users of the recurrence outside the loop. The users will need
4288 // either the last value of the scalar recurrence or the last value of the
4289 // vector recurrence we extracted in the middle block. Since the loop is in
4290 // LCSSA form, we just need to find all the phi nodes for the original scalar
4291 // recurrence in the exit block, and then add an edge for the middle block.
4292 // Note that LCSSA does not imply single entry when the original scalar loop
4293 // had multiple exiting edges (as we always run the last iteration in the
4294 // scalar epilogue); in that case, there is no edge from middle to exit and
4295 // and thus no phis which needed updated.
4296 if (!Cost->requiresScalarEpilogue(VF))
4297 for (PHINode &LCSSAPhi : LoopExitBlock->phis())
4298 if (any_of(LCSSAPhi.incoming_values(),
4299 [Phi](Value *V) { return V == Phi; }))
4300 LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
4301}
4302
4303void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
4304 VPTransformState &State) {
4305 PHINode *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
4306 // Get it's reduction variable descriptor.
4307 assert(Legal->isReductionVariable(OrigPhi) &&(static_cast<void> (0))
4308 "Unable to find the reduction variable")(static_cast<void> (0));
4309 const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
4310
4311 RecurKind RK = RdxDesc.getRecurrenceKind();
4312 TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
4313 Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
4314 setDebugLocFromInst(ReductionStartValue);
4315
4316 VPValue *LoopExitInstDef = PhiR->getBackedgeValue();
4317 // This is the vector-clone of the value that leaves the loop.
4318 Type *VecTy = State.get(LoopExitInstDef, 0)->getType();
4319
4320 // Wrap flags are in general invalid after vectorization, clear them.
4321 clearReductionWrapFlags(RdxDesc, State);
4322
4323 // Before each round, move the insertion point right between
4324 // the PHIs and the values we are going to write.
4325 // This allows us to write both PHINodes and the extractelement
4326 // instructions.
4327 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
4328
4329 setDebugLocFromInst(LoopExitInst);
4330
4331 Type *PhiTy = OrigPhi->getType();
4332 // If tail is folded by masking, the vector value to leave the loop should be
4333 // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
4334 // instead of the former. For an inloop reduction the reduction will already
4335 // be predicated, and does not need to be handled here.
4336 if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
4337 for (unsigned Part = 0; Part < UF; ++Part) {
4338 Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
4339 Value *Sel = nullptr;
4340 for (User *U : VecLoopExitInst->users()) {
4341 if (isa<SelectInst>(U)) {
4342 assert(!Sel && "Reduction exit feeding two selects")(static_cast<void> (0));
4343 Sel = U;
4344 } else
4345 assert(isa<PHINode>(U) && "Reduction exit must feed Phi's or select")(static_cast<void> (0));
4346 }
4347 assert(Sel && "Reduction exit feeds no select")(static_cast<void> (0));
4348 State.reset(LoopExitInstDef, Sel, Part);
4349
4350 // If the target can create a predicated operator for the reduction at no
4351 // extra cost in the loop (for example a predicated vadd), it can be
4352 // cheaper for the select to remain in the loop than be sunk out of it,
4353 // and so use the select value for the phi instead of the old
4354 // LoopExitValue.
4355 if (PreferPredicatedReductionSelect ||
4356 TTI->preferPredicatedReductionSelect(
4357 RdxDesc.getOpcode(), PhiTy,
4358 TargetTransformInfo::ReductionFlags())) {
4359 auto *VecRdxPhi =
4360 cast<PHINode>(State.get(PhiR->getVPSingleValue(), Part));
4361 VecRdxPhi->setIncomingValueForBlock(
4362 LI->getLoopFor(LoopVectorBody)->getLoopLatch(), Sel);
4363 }
4364 }
4365 }
4366
4367 // If the vector reduction can be performed in a smaller type, we truncate
4368 // then extend the loop exit value to enable InstCombine to evaluate the
4369 // entire expression in the smaller type.
4370 if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
4371 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!")(static_cast<void> (0));
4372 Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
4373 Builder.SetInsertPoint(
4374 LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
4375 VectorParts RdxParts(UF);
4376 for (unsigned Part = 0; Part < UF; ++Part) {
4377 RdxParts[Part] = State.get(LoopExitInstDef, Part);
4378 Value *Trunc = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
4379 Value *Extnd = RdxDesc.isSigned() ? Builder.CreateSExt(Trunc, VecTy)
4380 : Builder.CreateZExt(Trunc, VecTy);
4381 for (Value::user_iterator UI = RdxParts[Part]->user_begin();
4382 UI != RdxParts[Part]->user_end();)
4383 if (*UI != Trunc) {
4384 (*UI++)->replaceUsesOfWith(RdxParts[Part], Extnd);
4385 RdxParts[Part] = Extnd;
4386 } else {
4387 ++UI;
4388 }
4389 }
4390 Builder.SetInsertPoint(&*LoopMiddleBlock->getFirstInsertionPt());
4391 for (unsigned Part = 0; Part < UF; ++Part) {
4392 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
4393 State.reset(LoopExitInstDef, RdxParts[Part], Part);
4394 }
4395 }
4396
4397 // Reduce all of the unrolled parts into a single vector.
4398 Value *ReducedPartRdx = State.get(LoopExitInstDef, 0);
4399 unsigned Op = RecurrenceDescriptor::getOpcode(RK);
4400
4401 // The middle block terminator has already been assigned a DebugLoc here (the
4402 // OrigLoop's single latch terminator). We want the whole middle block to
4403 // appear to execute on this line because: (a) it is all compiler generated,
4404 // (b) these instructions are always executed after evaluating the latch
4405 // conditional branch, and (c) other passes may add new predecessors which
4406 // terminate on this line. This is the easiest way to ensure we don't
4407 // accidentally cause an extra step back into the loop while debugging.
4408 setDebugLocFromInst(LoopMiddleBlock->getTerminator());
4409 if (PhiR->isOrdered())
4410 ReducedPartRdx = State.get(LoopExitInstDef, UF - 1);
4411 else {
4412 // Floating-point operations should have some FMF to enable the reduction.
4413 IRBuilderBase::FastMathFlagGuard FMFG(Builder);
4414 Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
4415 for (unsigned Part = 1; Part < UF; ++Part) {
4416 Value *RdxPart = State.get(LoopExitInstDef, Part);
4417 if (Op != Instruction::ICmp && Op != Instruction::FCmp) {
4418 ReducedPartRdx = Builder.CreateBinOp(
4419 (Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, "bin.rdx");
4420 } else {
4421 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
4422 }
4423 }
4424 }
4425
4426 // Create the reduction after the loop. Note that inloop reductions create the
4427 // target reduction in the loop using a Reduction recipe.
4428 if (VF.isVector() && !PhiR->isInLoop()) {
4429 ReducedPartRdx =
4430 createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
4431 // If the reduction can be performed in a smaller type, we need to extend
4432 // the reduction to the wider type before we branch to the original loop.
4433 if (PhiTy != RdxDesc.getRecurrenceType())
4434 ReducedPartRdx = RdxDesc.isSigned()
4435 ? Builder.CreateSExt(ReducedPartRdx, PhiTy)
4436 : Builder.CreateZExt(ReducedPartRdx, PhiTy);
4437 }
4438
4439 // Create a phi node that merges control-flow from the backedge-taken check
4440 // block and the middle block.
4441 PHINode *BCBlockPhi = PHINode::Create(PhiTy, 2, "bc.merge.rdx",
4442 LoopScalarPreHeader->getTerminator());
4443 for (unsigned I = 0, E = LoopBypassBlocks.size(); I != E; ++I)
4444 BCBlockPhi->addIncoming(ReductionStartValue, LoopBypassBlocks[I]);
4445 BCBlockPhi->addIncoming(ReducedPartRdx, LoopMiddleBlock);
4446
4447 // Now, we need to fix the users of the reduction variable
4448 // inside and outside of the scalar remainder loop.
4449
4450 // We know that the loop is in LCSSA form. We need to update the PHI nodes
4451 // in the exit blocks. See comment on analogous loop in
4452 // fixFirstOrderRecurrence for a more complete explaination of the logic.
4453 if (!Cost->requiresScalarEpilogue(VF))
4454 for (PHINode &LCSSAPhi : LoopExitBlock->phis())
4455 if (any_of(LCSSAPhi.incoming_values(),
4456 [LoopExitInst](Value *V) { return V == LoopExitInst; }))
4457 LCSSAPhi.addIncoming(ReducedPartRdx, LoopMiddleBlock);
4458
4459 // Fix the scalar loop reduction variable with the incoming reduction sum
4460 // from the vector body and from the backedge value.
4461 int IncomingEdgeBlockIdx =
4462 OrigPhi->getBasicBlockIndex(OrigLoop->getLoopLatch());
4463 assert(IncomingEdgeBlockIdx >= 0 && "Invalid block index")(static_cast<void> (0));
4464 // Pick the other block.
4465 int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1);
4466 OrigPhi->setIncomingValue(SelfEdgeBlockIdx, BCBlockPhi);
4467 OrigPhi->setIncomingValue(IncomingEdgeBlockIdx, LoopExitInst);
4468}
4469
4470void InnerLoopVectorizer::clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
4471 VPTransformState &State) {
4472 RecurKind RK = RdxDesc.getRecurrenceKind();
4473 if (RK != RecurKind::Add && RK != RecurKind::Mul)
4474 return;
4475
4476 Instruction *LoopExitInstr = RdxDesc.getLoopExitInstr();
4477 assert(LoopExitInstr && "null loop exit instruction")(static_cast<void> (0));
4478 SmallVector<Instruction *, 8> Worklist;
4479 SmallPtrSet<Instruction *, 8> Visited;
4480 Worklist.push_back(LoopExitInstr);
4481 Visited.insert(LoopExitInstr);
4482
4483 while (!Worklist.empty()) {
4484 Instruction *Cur = Worklist.pop_back_val();
4485 if (isa<OverflowingBinaryOperator>(Cur))
4486 for (unsigned Part = 0; Part < UF; ++Part) {
4487 // FIXME: Should not rely on getVPValue at this point.
4488 Value *V = State.get(State.Plan->getVPValue(Cur, true), Part);
4489 cast<Instruction>(V)->dropPoisonGeneratingFlags();
4490 }
4491
4492 for (User *U : Cur->users()) {
4493 Instruction *UI = cast<Instruction>(U);
4494 if ((Cur != LoopExitInstr || OrigLoop->contains(UI->getParent())) &&
4495 Visited.insert(UI).second)
4496 Worklist.push_back(UI);
4497 }
4498 }
4499}
4500
4501void InnerLoopVectorizer::fixLCSSAPHIs(VPTransformState &State) {
4502 for (PHINode &LCSSAPhi : LoopExitBlock->phis()) {
4503 if (LCSSAPhi.getBasicBlockIndex(LoopMiddleBlock) != -1)
4504 // Some phis were already hand updated by the reduction and recurrence
4505 // code above, leave them alone.
4506 continue;
4507
4508 auto *IncomingValue = LCSSAPhi.getIncomingValue(0);
4509 // Non-instruction incoming values will have only one value.
4510
4511 VPLane Lane = VPLane::getFirstLane();
4512 if (isa<Instruction>(IncomingValue) &&
4513 !Cost->isUniformAfterVectorization(cast<Instruction>(IncomingValue),
4514 VF))
4515 Lane = VPLane::getLastLaneForVF(VF);
4516
4517 // Can be a loop invariant incoming value or the last scalar value to be
4518 // extracted from the vectorized loop.
4519 // FIXME: Should not rely on getVPValue at this point.
4520 Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
4521 Value *lastIncomingValue =
4522 OrigLoop->isLoopInvariant(IncomingValue)
4523 ? IncomingValue
4524 : State.get(State.Plan->getVPValue(IncomingValue, true),
4525 VPIteration(UF - 1, Lane));
4526 LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock);
4527 }
4528}
4529
4530void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
4531 // The basic block and loop containing the predicated instruction.
4532 auto *PredBB = PredInst->getParent();
4533 auto *VectorLoop = LI->getLoopFor(PredBB);
4534
4535 // Initialize a worklist with the operands of the predicated instruction.
4536 SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());
4537
4538 // Holds instructions that we need to analyze again. An instruction may be
4539 // reanalyzed if we don't yet know if we can sink it or not.
4540 SmallVector<Instruction *, 8> InstsToReanalyze;
4541
4542 // Returns true if a given use occurs in the predicated block. Phi nodes use
4543 // their operands in their corresponding predecessor blocks.
4544 auto isBlockOfUsePredicated = [&](Use &U) -> bool {
4545 auto *I = cast<Instruction>(U.getUser());
4546 BasicBlock *BB = I->getParent();
4547 if (auto *Phi = dyn_cast<PHINode>(I))
4548 BB = Phi->getIncomingBlock(
4549 PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
4550 return BB == PredBB;
4551 };
4552
4553 // Iteratively sink the scalarized operands of the predicated instruction
4554 // into the block we created for it. When an instruction is sunk, it's
4555 // operands are then added to the worklist. The algorithm ends after one pass
4556 // through the worklist doesn't sink a single instruction.
4557 bool Changed;
4558 do {
4559 // Add the instructions that need to be reanalyzed to the worklist, and
4560 // reset the changed indicator.
4561 Worklist.insert(InstsToReanalyze.begin(), InstsToReanalyze.end());
4562 InstsToReanalyze.clear();
4563 Changed = false;
4564
4565 while (!Worklist.empty()) {
4566 auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());
4567
4568 // We can't sink an instruction if it is a phi node, is not in the loop,
4569 // or may have side effects.
4570 if (!I || isa<PHINode>(I) || !VectorLoop->contains(I) ||
4571 I->mayHaveSideEffects())
4572 continue;
4573
4574 // If the instruction is already in PredBB, check if we can sink its
4575 // operands. In that case, VPlan's sinkScalarOperands() succeeded in
4576 // sinking the scalar instruction I, hence it appears in PredBB; but it
4577 // may have failed to sink I's operands (recursively), which we try
4578 // (again) here.
4579 if (I->getParent() == PredBB) {
4580 Worklist.insert(I->op_begin(), I->op_end());
4581 continue;
4582 }
4583
4584 // It's legal to sink the instruction if all its uses occur in the
4585 // predicated block. Otherwise, there's nothing to do yet, and we may
4586 // need to reanalyze the instruction.
4587 if (!llvm::all_of(I->uses(), isBlockOfUsePredicated)) {
4588 InstsToReanalyze.push_back(I);
4589 continue;
4590 }
4591
4592 // Move the instruction to the beginning of the predicated block, and add
4593 // it's operands to the worklist.
4594 I->moveBefore(&*PredBB->getFirstInsertionPt());
4595 Worklist.insert(I->op_begin(), I->op_end());
4596
4597 // The sinking may have enabled other instructions to be sunk, so we will
4598 // need to iterate.
4599 Changed = true;
4600 }
4601 } while (Changed);
4602}
4603
4604void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
4605 for (PHINode *OrigPhi : OrigPHIsToFix) {
4606 VPWidenPHIRecipe *VPPhi =
4607 cast<VPWidenPHIRecipe>(State.Plan->getVPValue(OrigPhi));
4608 PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, 0));
4609 // Make sure the builder has a valid insert point.
4610 Builder.SetInsertPoint(NewPhi);
4611 for (unsigned i = 0; i < VPPhi->getNumOperands(); ++i) {
4612 VPValue *Inc = VPPhi->getIncomingValue(i);
4613 VPBasicBlock *VPBB = VPPhi->getIncomingBlock(i);
4614 NewPhi->addIncoming(State.get(Inc, 0), State.CFG.VPBB2IRBB[VPBB]);
4615 }
4616 }
4617}
4618
4619bool InnerLoopVectorizer::useOrderedReductions(RecurrenceDescriptor &RdxDesc) {
4620 return Cost->useOrderedReductions(RdxDesc);
4621}
4622
4623void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
4624 VPUser &Operands, unsigned UF,
4625 ElementCount VF, bool IsPtrLoopInvariant,
4626 SmallBitVector &IsIndexLoopInvariant,
4627 VPTransformState &State) {
4628 // Construct a vector GEP by widening the operands of the scalar GEP as
4629 // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
4630 // results in a vector of pointers when at least one operand of the GEP
4631 // is vector-typed. Thus, to keep the representation compact, we only use
4632 // vector-typed operands for loop-varying values.
4633
4634 if (VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
4635 // If we are vectorizing, but the GEP has only loop-invariant operands,
4636 // the GEP we build (by only using vector-typed operands for
4637 // loop-varying values) would be a scalar pointer. Thus, to ensure we
4638 // produce a vector of pointers, we need to either arbitrarily pick an
4639 // operand to broadcast, or broadcast a clone of the original GEP.
4640 // Here, we broadcast a clone of the original.
4641 //
4642 // TODO: If at some point we decide to scalarize instructions having
4643 // loop-invariant operands, this special case will no longer be
4644 // required. We would add the scalarization decision to
4645 // collectLoopScalars() and teach getVectorValue() to broadcast
4646 // the lane-zero scalar value.
4647 auto *Clone = Builder.Insert(GEP->clone());
4648 for (unsigned Part = 0; Part < UF; ++Part) {
4649 Value *EntryPart = Builder.CreateVectorSplat(VF, Clone);
4650 State.set(VPDef, EntryPart, Part);
4651 addMetadata(EntryPart, GEP);
4652 }
4653 } else {
4654 // If the GEP has at least one loop-varying operand, we are sure to
4655 // produce a vector of pointers. But if we are only unrolling, we want
4656 // to produce a scalar GEP for each unroll part. Thus, the GEP we
4657 // produce with the code below will be scalar (if VF == 1) or vector
4658 // (otherwise). Note that for the unroll-only case, we still maintain
4659 // values in the vector mapping with initVector, as we do for other
4660 // instructions.
4661 for (unsigned Part = 0; Part < UF; ++Part) {
4662 // The pointer operand of the new GEP. If it's loop-invariant, we
4663 // won't broadcast it.
4664 auto *Ptr = IsPtrLoopInvariant
4665 ? State.get(Operands.getOperand(0), VPIteration(0, 0))
4666 : State.get(Operands.getOperand(0), Part);
4667
4668 // Collect all the indices for the new GEP. If any index is
4669 // loop-invariant, we won't broadcast it.
4670 SmallVector<Value *, 4> Indices;
4671 for (unsigned I = 1, E = Operands.getNumOperands(); I < E; I++) {
4672 VPValue *Operand = Operands.getOperand(I);
4673 if (IsIndexLoopInvariant[I - 1])
4674 Indices.push_back(State.get(Operand, VPIteration(0, 0)));
4675 else
4676 Indices.push_back(State.get(Operand, Part));
4677 }
4678
4679 // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
4680 // but it should be a vector, otherwise.
4681 auto *NewGEP =
4682 GEP->isInBounds()
4683 ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
4684 Indices)
4685 : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
4686 assert((VF.isScalar() || NewGEP->getType()->isVectorTy()) &&(static_cast<void> (0))
4687 "NewGEP is not a pointer vector")(static_cast<void> (0));
4688 State.set(VPDef, NewGEP, Part);
4689 addMetadata(NewGEP, GEP);
4690 }
4691 }
4692}
4693
4694void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
4695 VPWidenPHIRecipe *PhiR,
4696 VPTransformState &State) {
4697 PHINode *P = cast<PHINode>(PN);
4698 if (EnableVPlanNativePath) {
4699 // Currently we enter here in the VPlan-native path for non-induction
4700 // PHIs where all control flow is uniform. We simply widen these PHIs.
4701 // Create a vector phi with no operands - the vector phi operands will be
4702 // set at the end of vector code generation.
4703 Type *VecTy = (State.VF.isScalar())
4704 ? PN->getType()
4705 : VectorType::get(PN->getType(), State.VF);
4706 Value *VecPhi = Builder.CreatePHI(VecTy, PN->getNumOperands(), "vec.phi");
4707 State.set(PhiR, VecPhi, 0);
4708 OrigPHIsToFix.push_back(P);
4709
4710 return;
4711 }
4712
4713 assert(PN->getParent() == OrigLoop->getHeader() &&(static_cast<void> (0))
4714 "Non-header phis should have been handled elsewhere")(static_cast<void> (0));
4715
4716 // In order to support recurrences we need to be able to vectorize Phi nodes.
4717 // Phi nodes have cycles, so we need to vectorize them in two stages. This is
4718 // stage #1: We create a new vector PHI node with no incoming edges. We'll use
4719 // this value when we vectorize all of the instructions that use the PHI.
4720
4721 assert(!Legal->isReductionVariable(P) &&(static_cast<void> (0))
4722 "reductions should be handled elsewhere")(static_cast<void> (0));
4723
4724 setDebugLocFromInst(P);
4725
4726 // This PHINode must be an induction variable.
4727 // Make sure that we know about it.
4728 assert(Legal->getInductionVars().count(P) && "Not an induction variable")(static_cast<void> (0));
4729
4730 InductionDescriptor II = Legal->getInductionVars().lookup(P);
4731 const DataLayout &DL = OrigLoop->getHeader()->getModule()->getDataLayout();
4732
4733 // FIXME: The newly created binary instructions should contain nsw/nuw flags,
4734 // which can be found from the original scalar operations.
4735 switch (II.getKind()) {
4736 case InductionDescriptor::IK_NoInduction:
4737 llvm_unreachable("Unknown induction")__builtin_unreachable();
4738 case InductionDescriptor::IK_IntInduction:
4739 case InductionDescriptor::IK_FpInduction:
4740 llvm_unreachable("Integer/fp induction is handled elsewhere.")__builtin_unreachable();
4741 case InductionDescriptor::IK_PtrInduction: {
4742 // Handle the pointer induction variable case.
4743 assert(P->getType()->isPointerTy() && "Unexpected type.")(static_cast<void> (0));
4744
4745 if (Cost->isScalarAfterVectorization(P, State.VF)) {
4746 // This is the normalized GEP that starts counting at zero.
4747 Value *PtrInd =
4748 Builder.CreateSExtOrTrunc(Induction, II.getStep()->getType());
4749 // Determine the number of scalars we need to generate for each unroll
4750 // iteration. If the instruction is uniform, we only need to generate the
4751 // first lane. Otherwise, we generate all VF values.
4752 bool IsUniform = Cost->isUniformAfterVectorization(P, State.VF);
4753 unsigned Lanes = IsUniform ? 1 : State.VF.getKnownMinValue();
4754
4755 bool NeedsVectorIndex = !IsUniform && VF.isScalable();
4756 Value *UnitStepVec = nullptr, *PtrIndSplat = nullptr;
4757 if (NeedsVectorIndex) {
4758 Type *VecIVTy = VectorType::get(PtrInd->getType(), VF);
4759 UnitStepVec = Builder.CreateStepVector(VecIVTy);
4760 PtrIndSplat = Builder.CreateVectorSplat(VF, PtrInd);
4761 }
4762
4763 for (unsigned Part = 0; Part < UF; ++Part) {
4764 Value *PartStart = createStepForVF(
4765 Builder, ConstantInt::get(PtrInd->getType(), Part), VF);
4766
4767 if (NeedsVectorIndex) {
4768 Value *PartStartSplat = Builder.CreateVectorSplat(VF, PartStart);
4769 Value *Indices = Builder.CreateAdd(PartStartSplat, UnitStepVec);
4770 Value *GlobalIndices = Builder.CreateAdd(PtrIndSplat, Indices);
4771 Value *SclrGep =
4772 emitTransformedIndex(Builder, GlobalIndices, PSE.getSE(), DL, II);
4773 SclrGep->setName("next.gep");
4774 State.set(PhiR, SclrGep, Part);
4775 // We've cached the whole vector, which means we can support the
4776 // extraction of any lane.
4777 continue;
4778 }
4779
4780 for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
4781 Value *Idx = Builder.CreateAdd(
4782 PartStart, ConstantInt::get(PtrInd->getType(), Lane));
4783 Value *GlobalIdx = Builder.CreateAdd(PtrInd, Idx);
4784 Value *SclrGep =
4785 emitTransformedIndex(Builder, GlobalIdx, PSE.getSE(), DL, II);
4786 SclrGep->setName("next.gep");
4787 State.set(PhiR, SclrGep, VPIteration(Part, Lane));
4788 }
4789 }
4790 return;
4791 }
4792 assert(isa<SCEVConstant>(II.getStep()) &&(static_cast<void> (0))
4793 "Induction step not a SCEV constant!")(static_cast<void> (0));
4794 Type *PhiType = II.getStep()->getType();
4795
4796 // Build a pointer phi
4797 Value *ScalarStartValue = II.getStartValue();
4798 Type *ScStValueType = ScalarStartValue->getType();
4799 PHINode *NewPointerPhi =
4800 PHINode::Create(ScStValueType, 2, "pointer.phi", Induction);
4801 NewPointerPhi->addIncoming(ScalarStartValue, LoopVectorPreHeader);
4802
4803 // A pointer induction, performed by using a gep
4804 BasicBlock *LoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
4805 Instruction *InductionLoc = LoopLatch->getTerminator();
4806 const SCEV *ScalarStep = II.getStep();
4807 SCEVExpander Exp(*PSE.getSE(), DL, "induction");
4808 Value *ScalarStepValue =
4809 Exp.expandCodeFor(ScalarStep, PhiType, InductionLoc);
4810 Value *RuntimeVF = getRuntimeVF(Builder, PhiType, VF);
4811 Value *NumUnrolledElems =
4812 Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF));
4813 Value *InductionGEP = GetElementPtrInst::Create(
4814 II.getElementType(), NewPointerPhi,
4815 Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
4816 InductionLoc);
4817 NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
4818
4819 // Create UF many actual address geps that use the pointer
4820 // phi as base and a vectorized version of the step value
4821 // (<step*0, ..., step*N>) as offset.
4822 for (unsigned Part = 0; Part < State.UF; ++Part) {
4823 Type *VecPhiType = VectorType::get(PhiType, State.VF);
4824 Value *StartOffsetScalar =
4825 Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, Part));
4826 Value *StartOffset =
4827 Builder.CreateVectorSplat(State.VF, StartOffsetScalar);
4828 // Create a vector of consecutive numbers from zero to VF.
4829 StartOffset =
4830 Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType));
4831
4832 Value *GEP = Builder.CreateGEP(
4833 II.getElementType(), NewPointerPhi,
4834 Builder.CreateMul(
4835 StartOffset, Builder.CreateVectorSplat(State.VF, ScalarStepValue),
4836 "vector.gep"));
4837 State.set(PhiR, GEP, Part);
4838 }
4839 }
4840 }
4841}
4842
4843/// A helper function for checking whether an integer division-related
4844/// instruction may divide by zero (in which case it must be predicated if
4845/// executed conditionally in the scalar code).
4846/// TODO: It may be worthwhile to generalize and check isKnownNonZero().
4847/// Non-zero divisors that are non compile-time constants will not be
4848/// converted into multiplication, so we will still end up scalarizing
4849/// the division, but can do so w/o predication.
4850static bool mayDivideByZero(Instruction &I) {
4851 assert((I.getOpcode() == Instruction::UDiv ||(static_cast<void> (0))
4852 I.getOpcode() == Instruction::SDiv ||(static_cast<void> (0))
4853 I.getOpcode() == Instruction::URem ||(static_cast<void> (0))
4854 I.getOpcode() == Instruction::SRem) &&(static_cast<void> (0))
4855 "Unexpected instruction")(static_cast<void> (0));
4856 Value *Divisor = I.getOperand(1);
4857 auto *CInt = dyn_cast<ConstantInt>(Divisor);
4858 return !CInt || CInt->isZero();
4859}
4860
4861void InnerLoopVectorizer::widenInstruction(Instruction &I, VPValue *Def,
4862 VPUser &User,
4863 VPTransformState &State) {
4864 switch (I.getOpcode()) {
4865 case Instruction::Call:
4866 case Instruction::Br:
4867 case Instruction::PHI:
4868 case Instruction::GetElementPtr:
4869 case Instruction::Select:
4870 llvm_unreachable("This instruction is handled by a different recipe.")__builtin_unreachable();
4871 case Instruction::UDiv:
4872 case Instruction::SDiv:
4873 case Instruction::SRem:
4874 case Instruction::URem:
4875 case Instruction::Add:
4876 case Instruction::FAdd:
4877 case Instruction::Sub:
4878 case Instruction::FSub:
4879 case Instruction::FNeg:
4880 case Instruction::Mul:
4881 case Instruction::FMul:
4882 case Instruction::FDiv:
4883 case Instruction::FRem:
4884 case Instruction::Shl:
4885 case Instruction::LShr:
4886 case Instruction::AShr:
4887 case Instruction::And:
4888 case Instruction::Or:
4889 case Instruction::Xor: {
4890 // Just widen unops and binops.
4891 setDebugLocFromInst(&I);
4892
4893 for (unsigned Part = 0; Part < UF; ++Part) {
4894 SmallVector<Value *, 2> Ops;
4895 for (VPValue *VPOp : User.operands())
4896 Ops.push_back(State.get(VPOp, Part));
4897
4898 Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
4899
4900 if (auto *VecOp = dyn_cast<Instruction>(V))
4901 VecOp->copyIRFlags(&I);
4902
4903 // Use this vector value for all users of the original instruction.
4904 State.set(Def, V, Part);
4905 addMetadata(V, &I);
4906 }
4907
4908 break;
4909 }
4910 case Instruction::ICmp:
4911 case Instruction::FCmp: {
4912 // Widen compares. Generate vector compares.
4913 bool FCmp = (I.getOpcode() == Instruction::FCmp);
4914 auto *Cmp = cast<CmpInst>(&I);
4915 setDebugLocFromInst(Cmp);
4916 for (unsigned Part = 0; Part < UF; ++Part) {
4917 Value *A = State.get(User.getOperand(0), Part);
4918 Value *B = State.get(User.getOperand(1), Part);
4919 Value *C = nullptr;
4920 if (FCmp) {
4921 // Propagate fast math flags.
4922 IRBuilder<>::FastMathFlagGuard FMFG(Builder);
4923 Builder.setFastMathFlags(Cmp->getFastMathFlags());
4924 C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
4925 } else {
4926 C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
4927 }
4928 State.set(Def, C, Part);
4929 addMetadata(C, &I);
4930 }
4931
4932 break;
4933 }
4934
4935 case Instruction::ZExt:
4936 case Instruction::SExt:
4937 case Instruction::FPToUI:
4938 case Instruction::FPToSI:
4939 case Instruction::FPExt:
4940 case Instruction::PtrToInt:
4941 case Instruction::IntToPtr:
4942 case Instruction::SIToFP:
4943 case Instruction::UIToFP:
4944 case Instruction::Trunc:
4945 case Instruction::FPTrunc:
4946 case Instruction::BitCast: {
4947 auto *CI = cast<CastInst>(&I);
4948 setDebugLocFromInst(CI);
4949
4950 /// Vectorize casts.
4951 Type *DestTy =
4952 (VF.isScalar()) ? CI->getType() : VectorType::get(CI->getType(), VF);
4953
4954 for (unsigned Part = 0; Part < UF; ++Part) {
4955 Value *A = State.get(User.getOperand(0), Part);
4956 Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
4957 State.set(Def, Cast, Part);
4958 addMetadata(Cast, &I);
4959 }
4960 break;
4961 }
4962 default:
4963 // This instruction is not vectorized by simple widening.
4964 LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I)do { } while (false);
4965 llvm_unreachable("Unhandled instruction!")__builtin_unreachable();
4966 } // end of switch.
4967}
4968
4969void InnerLoopVectorizer::widenCallInstruction(CallInst &I, VPValue *Def,
4970 VPUser &ArgOperands,
4971 VPTransformState &State) {
4972 assert(!isa<DbgInfoIntrinsic>(I) &&(static_cast<void> (0))
4973 "DbgInfoIntrinsic should have been dropped during VPlan construction")(static_cast<void> (0));
4974 setDebugLocFromInst(&I);
4975
4976 Module *M = I.getParent()->getParent()->getParent();
4977 auto *CI = cast<CallInst>(&I);
4978
4979 SmallVector<Type *, 4> Tys;
4980 for (Value *ArgOperand : CI->arg_operands())
4981 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF.getKnownMinValue()));
4982
4983 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4984
4985 // The flag shows whether we use Intrinsic or a usual Call for vectorized
4986 // version of the instruction.
4987 // Is it beneficial to perform intrinsic call compared to lib call?
4988 bool NeedToScalarize = false;
4989 InstructionCost CallCost = Cost->getVectorCallCost(CI, VF, NeedToScalarize);
4990 InstructionCost IntrinsicCost = ID ? Cost->getVectorIntrinsicCost(CI, VF) : 0;
4991 bool UseVectorIntrinsic = ID && IntrinsicCost <= CallCost;
4992 assert((UseVectorIntrinsic || !NeedToScalarize) &&(static_cast<void> (0))
4993 "Instruction should be scalarized elsewhere.")(static_cast<void> (0));
4994 assert((IntrinsicCost.isValid() || CallCost.isValid()) &&(static_cast<void> (0))
4995 "Either the intrinsic cost or vector call cost must be valid")(static_cast<void> (0));
4996
4997 for (unsigned Part = 0; Part < UF; ++Part) {
4998 SmallVector<Type *, 2> TysForDecl = {CI->getType()};
4999 SmallVector<Value *, 4> Args;
5000 for (auto &I : enumerate(ArgOperands.operands())) {
5001 // Some intrinsics have a scalar argument - don't replace it with a
5002 // vector.
5003 Value *Arg;
5004 if (!UseVectorIntrinsic || !hasVectorInstrinsicScalarOpd(ID, I.index()))
5005 Arg = State.get(I.value(), Part);
5006 else {
5007 Arg = State.get(I.value(), VPIteration(0, 0));
5008 if (hasVectorInstrinsicOverloadedScalarOpd(ID, I.index()))
5009 TysForDecl.push_back(Arg->getType());
5010 }
5011 Args.push_back(Arg);
5012 }
5013
5014 Function *VectorF;
5015 if (UseVectorIntrinsic) {
5016 // Use vector version of the intrinsic.
5017 if (VF.isVector())
5018 TysForDecl[0] = VectorType::get(CI->getType()->getScalarType(), VF);
5019 VectorF = Intrinsic::getDeclaration(M, ID, TysForDecl);
5020 assert(VectorF && "Can't retrieve vector intrinsic.")(static_cast<void> (0));
5021 } else {
5022 // Use vector version of the function call.
5023 const VFShape Shape = VFShape::get(*CI, VF, false /*HasGlobalPred*/);
5024#ifndef NDEBUG1
5025 assert(VFDatabase(*CI).getVectorizedFunction(Shape) != nullptr &&(static_cast<void> (0))
5026 "Can't create vector function.")(static_cast<void> (0));
5027#endif
5028 VectorF = VFDatabase(*CI).getVectorizedFunction(Shape);
5029 }
5030 SmallVector<OperandBundleDef, 1> OpBundles;
5031 CI->getOperandBundlesAsDefs(OpBundles);
5032 CallInst *V = Builder.CreateCall(VectorF, Args, OpBundles);
5033
5034 if (isa<FPMathOperator>(V))
5035 V->copyFastMathFlags(CI);
5036
5037 State.set(Def, V, Part);
5038 addMetadata(V, &I);
5039 }
5040}
5041
5042void InnerLoopVectorizer::widenSelectInstruction(SelectInst &I, VPValue *VPDef,
5043 VPUser &Operands,
5044 bool InvariantCond,
5045 VPTransformState &State) {
5046 setDebugLocFromInst(&I);
5047
5048 // The condition can be loop invariant but still defined inside the
5049 // loop. This means that we can't just use the original 'cond' value.
5050 // We have to take the 'vectorized' value and pick the first lane.
5051 // Instcombine will make this a no-op.
5052 auto *InvarCond = InvariantCond
5053 ? State.get(Operands.getOperand(0), VPIteration(0, 0))
5054 : nullptr;
5055
5056 for (unsigned Part = 0; Part < UF; ++Part) {
5057 Value *Cond =
5058 InvarCond ? InvarCond : State.get(Operands.getOperand(0), Part);
5059 Value *Op0 = State.get(Operands.getOperand(1), Part);
5060 Value *Op1 = State.get(Operands.getOperand(2), Part);
5061 Value *Sel = Builder.CreateSelect(Cond, Op0, Op1);
5062 State.set(VPDef, Sel, Part);
5063 addMetadata(Sel, &I);
5064 }
5065}
5066
5067void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
5068 // We should not collect Scalars more than once per VF. Right now, this
5069 // function is called from collectUniformsAndScalars(), which already does
5070 // this check. Collecting Scalars for VF=1 does not make any sense.
5071 assert(VF.isVector() && Scalars.find(VF) == Scalars.end() &&(static_cast<void> (0))
5072 "This function should not be visited twice for the same VF")(static_cast<void> (0));
5073
5074 SmallSetVector<Instruction *, 8> Worklist;
5075
5076 // These sets are used to seed the analysis with pointers used by memory
5077 // accesses that will remain scalar.
5078 SmallSetVector<Instruction *, 8> ScalarPtrs;
5079 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
5080 auto *Latch = TheLoop->getLoopLatch();
5081
5082 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
5083 // The pointer operands of loads and stores will be scalar as long as the
5084 // memory access is not a gather or scatter operation. The value operand of a
5085 // store will remain scalar if the store is scalarized.
5086 auto isScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
5087 InstWidening WideningDecision = getWideningDecision(MemAccess, VF);
5088 assert(WideningDecision != CM_Unknown &&(static_cast<void> (0))
5089 "Widening decision should be ready at this moment")(static_cast<void> (0));
5090 if (auto *Store = dyn_cast<StoreInst>(MemAccess))
5091 if (Ptr == Store->getValueOperand())
5092 return WideningDecision == CM_Scalarize;
5093 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&(static_cast<void> (0))
5094 "Ptr is neither a value or pointer operand")(static_cast<void> (0));
5095 return WideningDecision != CM_GatherScatter;
5096 };
5097
5098 // A helper that returns true if the given value is a bitcast or
5099 // getelementptr instruction contained in the loop.
5100 auto isLoopVaryingBitCastOrGEP = [&](Value *V) {
5101 return ((isa<BitCastInst>(V) && V->getType()->isPointerTy()) ||
5102 isa<GetElementPtrInst>(V)) &&
5103 !TheLoop->isLoopInvariant(V);
5104 };
5105
5106 auto isScalarPtrInduction = [&](Instruction *MemAccess, Value *Ptr) {
5107 if (!isa<PHINode>(Ptr) ||
5108 !Legal->getInductionVars().count(cast<PHINode>(Ptr)))
5109 return false;
5110 auto &Induction = Legal->getInductionVars()[cast<PHINode>(Ptr)];
5111 if (Induction.getKind() != InductionDescriptor::IK_PtrInduction)
5112 return false;
5113 return isScalarUse(MemAccess, Ptr);
5114 };
5115
5116 // A helper that evaluates a memory access's use of a pointer. If the
5117 // pointer is actually the pointer induction of a loop, it is being
5118 // inserted into Worklist. If the use will be a scalar use, and the
5119 // pointer is only used by memory accesses, we place the pointer in
5120 // ScalarPtrs. Otherwise, the pointer is placed in PossibleNonScalarPtrs.
5121 auto evaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
5122 if (isScalarPtrInduction(MemAccess, Ptr)) {
5123 Worklist.insert(cast<Instruction>(Ptr));
5124 LLVM_DEBUG(dbgs() << "LV: Found new scalar instruction: " << *Ptrdo { } while (false)
5125 << "\n")do { } while (false);
5126
5127 Instruction *Update = cast<Instruction>(
5128 cast<PHINode>(Ptr)->getIncomingValueForBlock(Latch));
5129 ScalarPtrs.insert(Update);
5130 return;
5131 }
5132 // We only care about bitcast and getelementptr instructions contained in
5133 // the loop.
5134 if (!isLoopVaryingBitCastOrGEP(Ptr))
5135 return;
5136
5137 // If the pointer has already been identified as scalar (e.g., if it was
5138 // also identified as uniform), there's nothing to do.
5139 auto *I = cast<Instruction>(Ptr);
5140 if (Worklist.count(I))
5141 return;
5142
5143 // If the use of the pointer will be a scalar use, and all users of the
5144 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
5145 // place the pointer in PossibleNonScalarPtrs.
5146 if (isScalarUse(MemAccess, Ptr) && llvm::all_of(I->users(), [&](User *U) {
5147 return isa<LoadInst>(U) || isa<StoreInst>(U);
5148 }))
5149 ScalarPtrs.insert(I);
5150 else
5151 PossibleNonScalarPtrs.insert(I);
5152 };
5153
5154 // We seed the scalars analysis with three classes of instructions: (1)
5155 // instructions marked uniform-after-vectorization and (2) bitcast,
5156 // getelementptr and (pointer) phi instructions used by memory accesses
5157 // requiring a scalar use.
5158 //
5159 // (1) Add to the worklist all instructions that have been identified as
5160 // uniform-after-vectorization.
5161 Worklist.insert(Uniforms[VF].begin(), Uniforms[VF].end());
5162
5163 // (2) Add to the worklist all bitcast and getelementptr instructions used by
5164 // memory accesses requiring a scalar use. The pointer operands of loads and
5165 // stores will be scalar as long as the memory accesses is not a gather or
5166 // scatter operation. The value operand of a store will remain scalar if the
5167 // store is scalarized.
5168 for (auto *BB : TheLoop->blocks())
5169 for (auto &I : *BB) {
5170 if (auto *Load = dyn_cast<LoadInst>(&I)) {
5171 evaluatePtrUse(Load, Load->getPointerOperand());
5172 } else if (auto *Store = dyn_cast<StoreInst>(&I)) {
5173 evaluatePtrUse(Store, Store->getPointerOperand());
5174 evaluatePtrUse(Store, Store->getValueOperand());
5175 }
5176 }
5177 for (auto *I : ScalarPtrs)
5178 if (!PossibleNonScalarPtrs.count(I)) {
5179 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n")do { } while (false);
5180 Worklist.insert(I);
5181 }
5182
5183 // Insert the forced scalars.
5184 // FIXME: Currently widenPHIInstruction() often creates a dead vector
5185 // induction variable when the PHI user is scalarized.
5186 auto ForcedScalar = ForcedScalars.find(VF);
5187 if (ForcedScalar != ForcedScalars.end())
5188 for (auto *I : ForcedScalar->second)
5189 Worklist.insert(I);
5190
5191 // Expand the worklist by looking through any bitcasts and getelementptr
5192 // instructions we've already identified as scalar. This is similar to the
5193 // expansion step in collectLoopUniforms(); however, here we're only
5194 // expanding to include additional bitcasts and getelementptr instructions.
5195 unsigned Idx = 0;
5196 while (Idx != Worklist.size()) {
5197 Instruction *Dst = Worklist[Idx++];
5198 if (!isLoopVaryingBitCastOrGEP(Dst->getOperand(0)))
5199 continue;
5200 auto *Src = cast<Instruction>(Dst->getOperand(0));
5201 if (llvm::all_of(Src->users(), [&](User *U) -> bool {
5202 auto *J = cast<Instruction>(U);
5203 return !TheLoop->contains(J) || Worklist.count(J) ||
5204 ((isa<LoadInst>(J) || isa<StoreInst>(J)) &&
5205 isScalarUse(J, Src));
5206 })) {
5207 Worklist.insert(Src);
5208 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n")do { } while (false);
5209 }
5210 }
5211
5212 // An induction variable will remain scalar if all users of the induction
5213 // variable and induction variable update remain scalar.
5214 for (auto &Induction : Legal->getInductionVars()) {
5215 auto *Ind = Induction.first;
5216 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
5217
5218 // If tail-folding is applied, the primary induction variable will be used
5219 // to feed a vector compare.
5220 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
5221 continue;
5222
5223 // Determine if all users of the induction variable are scalar after
5224 // vectorization.
5225 auto ScalarInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
5226 auto *I = cast<Instruction>(U);
5227 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I);
5228 });
5229 if (!ScalarInd)
5230 continue;
5231
5232 // Determine if all users of the induction variable update instruction are
5233 // scalar after vectorization.
5234 auto ScalarIndUpdate =
5235 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
5236 auto *I = cast<Instruction>(U);
5237 return I == Ind || !TheLoop->contains(I) || Worklist.count(I);
5238 });
5239 if (!ScalarIndUpdate)
5240 continue;
5241
5242 // The induction variable and its update instruction will remain scalar.
5243 Worklist.insert(Ind);
5244 Worklist.insert(IndUpdate);
5245 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n")do { } while (false);
5246 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdatedo { } while (false)
5247 << "\n")do { } while (false);
5248 }
5249
5250 Scalars[VF].insert(Worklist.begin(), Worklist.end());
5251}
5252
5253bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I) const {
5254 if (!blockNeedsPredication(I->getParent()))
16
Calling 'LoopVectorizationCostModel::blockNeedsPredication'
19
Returning from 'LoopVectorizationCostModel::blockNeedsPredication'
20
Assuming the condition is true
21
Taking true branch
5255 return false;
22
Returning zero, which participates in a condition later
5256 switch(I->getOpcode()) {
5257 default:
5258 break;
5259 case Instruction::Load:
5260 case Instruction::Store: {
5261 if (!Legal->isMaskRequired(I))
5262 return false;
5263 auto *Ptr = getLoadStorePointerOperand(I);
5264 auto *Ty = getLoadStoreType(I);
5265 const Align Alignment = getLoadStoreAlignment(I);
5266 return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
5267 TTI.isLegalMaskedGather(Ty, Alignment))
5268 : !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
5269 TTI.isLegalMaskedScatter(Ty, Alignment));
5270 }
5271 case Instruction::UDiv:
5272 case Instruction::SDiv:
5273 case Instruction::SRem:
5274 case Instruction::URem:
5275 return mayDivideByZero(*I);
5276 }
5277 return false;
5278}
5279
5280bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
5281 Instruction *I, ElementCount VF) {
5282 assert(isAccessInterleaved(I) && "Expecting interleaved access.")(static_cast<void> (0));
5283 assert(getWideningDecision(I, VF) == CM_Unknown &&(static_cast<void> (0))
5284 "Decision should not be set yet.")(static_cast<void> (0));
5285 auto *Group = getInterleavedAccessGroup(I);
5286 assert(Group && "Must have a group.")(static_cast<void> (0));
5287
5288 // If the instruction's allocated size doesn't equal it's type size, it
5289 // requires padding and will be scalarized.
5290 auto &DL = I->getModule()->getDataLayout();
5291 auto *ScalarTy = getLoadStoreType(I);
5292 if (hasIrregularType(ScalarTy, DL))
5293 return false;
5294
5295 // Check if masking is required.
5296 // A Group may need masking for one of two reasons: it resides in a block that
5297 // needs predication, or it was decided to use masking to deal with gaps
5298 // (either a gap at the end of a load-access that may result in a speculative
5299 // load, or any gaps in a store-access).
5300 bool PredicatedAccessRequiresMasking =
5301 Legal->blockNeedsPredication(I->getParent()) && Legal->isMaskRequired(I);
5302 bool LoadAccessWithGapsRequiresEpilogMasking =
5303 isa<LoadInst>(I) && Group->requiresScalarEpilogue() &&
5304 !isScalarEpilogueAllowed();
5305 bool StoreAccessWithGapsRequiresMasking =
5306 isa<StoreInst>(I) && (Group->getNumMembers() < Group->getFactor());
5307 if (!PredicatedAccessRequiresMasking &&
5308 !LoadAccessWithGapsRequiresEpilogMasking &&
5309 !StoreAccessWithGapsRequiresMasking)
5310 return true;
5311
5312 // If masked interleaving is required, we expect that the user/target had
5313 // enabled it, because otherwise it either wouldn't have been created or
5314 // it should have been invalidated by the CostModel.
5315 assert(useMaskedInterleavedAccesses(TTI) &&(static_cast<void> (0))
5316 "Masked interleave-groups for predicated accesses are not enabled.")(static_cast<void> (0));
5317
5318 auto *Ty = getLoadStoreType(I);
5319 const Align Alignment = getLoadStoreAlignment(I);
5320 return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
5321 : TTI.isLegalMaskedStore(Ty, Alignment);
5322}
5323
5324bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
5325 Instruction *I, ElementCount VF) {
5326 // Get and ensure we have a valid memory instruction.
5327 LoadInst *LI = dyn_cast<LoadInst>(I);
10
Assuming 'I' is not a 'LoadInst'
5328 StoreInst *SI = dyn_cast<StoreInst>(I);
11
Assuming 'I' is not a 'StoreInst'
12
'SI' initialized to a null pointer value
5329 assert((LI || SI) && "Invalid memory instruction")(static_cast<void> (0));
5330
5331 auto *Ptr = getLoadStorePointerOperand(I);
5332
5333 // In order to be widened, the pointer should be consecutive, first of all.
5334 if (!Legal->isConsecutivePtr(Ptr))
13
Assuming the condition is false
14
Taking false branch
5335 return false;
5336
5337 // If the instruction is a store located in a predicated block, it will be
5338 // scalarized.
5339 if (isScalarWithPredication(I))
15
Calling 'LoopVectorizationCostModel::isScalarWithPredication'
23
Returning from 'LoopVectorizationCostModel::isScalarWithPredication'
24
Taking false branch
5340 return false;
5341
5342 // If the instruction's allocated size doesn't equal it's type size, it
5343 // requires padding and will be scalarized.
5344 auto &DL = I->getModule()->getDataLayout();
5345 auto *ScalarTy = LI
24.1
'LI' is null
? LI->getType() : SI->getValueOperand()->getType();
25
'?' condition is false
26
Called C++ object pointer is null
5346 if (hasIrregularType(ScalarTy, DL))
5347 return false;
5348
5349 return true;
5350}
5351
5352void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
5353 // We should not collect Uniforms more than once per VF. Right now,
5354 // this function is called from collectUniformsAndScalars(), which
5355 // already does this check. Collecting Uniforms for VF=1 does not make any
5356 // sense.
5357
5358 assert(VF.isVector() && Uniforms.find(VF) == Uniforms.end() &&(static_cast<void> (0))
5359 "This function should not be visited twice for the same VF")(static_cast<void> (0));
5360
5361 // Visit the list of Uniforms. If we'll not find any uniform value, we'll
5362 // not analyze again. Uniforms.count(VF) will return 1.
5363 Uniforms[VF].clear();
5364
5365 // We now know that the loop is vectorizable!
5366 // Collect instructions inside the loop that will remain uniform after
5367 // vectorization.
5368
5369 // Global values, params and instructions outside of current loop are out of
5370 // scope.
5371 auto isOutOfScope = [&](Value *V) -> bool {
5372 Instruction *I = dyn_cast<Instruction>(V);
5373 return (!I || !TheLoop->contains(I));
5374 };
5375
5376 SetVector<Instruction *> Worklist;
5377 BasicBlock *Latch = TheLoop->getLoopLatch();
5378
5379 // Instructions that are scalar with predication must not be considered
5380 // uniform after vectorization, because that would create an erroneous
5381 // replicating region where only a single instance out of VF should be formed.
5382 // TODO: optimize such seldom cases if found important, see PR40816.
5383 auto addToWorklistIfAllowed = [&](Instruction *I) -> void {
5384 if (isOutOfScope(I)) {
5385 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "do { } while (false)
5386 << *I << "\n")do { } while (false);
5387 return;
5388 }
5389 if (isScalarWithPredication(I)) {
5390 LLVM_DEBUG(dbgs() << "LV: Found not uniform being ScalarWithPredication: "do { } while (false)
5391 << *I << "\n")do { } while (false);
5392 return;
5393 }
5394 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n")do { } while (false);
5395 Worklist.insert(I);
5396 };
5397
5398 // Start with the conditional branch. If the branch condition is an
5399 // instruction contained in the loop that is only used by the branch, it is
5400 // uniform.
5401 auto *Cmp = dyn_cast<Instruction>(Latch->getTerminator()->getOperand(0));
5402 if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
5403 addToWorklistIfAllowed(Cmp);
5404
5405 auto isUniformDecision = [&](Instruction *I, ElementCount VF) {
5406 InstWidening WideningDecision = getWideningDecision(I, VF);
5407 assert(WideningDecision != CM_Unknown &&(static_cast<void> (0))
5408 "Widening decision should be ready at this moment")(static_cast<void> (0));
5409
5410 // A uniform memory op is itself uniform. We exclude uniform stores
5411 // here as they demand the last lane, not the first one.
5412 if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) {
5413 assert(WideningDecision == CM_Scalarize)(static_cast<void> (0));
5414 return true;
5415 }
5416
5417 return (WideningDecision == CM_Widen ||
5418 WideningDecision == CM_Widen_Reverse ||
5419 WideningDecision == CM_Interleave);
5420 };
5421
5422
5423 // Returns true if Ptr is the pointer operand of a memory access instruction
5424 // I, and I is known to not require scalarization.
5425 auto isVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
5426 return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
5427 };
5428
5429 // Holds a list of values which are known to have at least one uniform use.
5430 // Note that there may be other uses which aren't uniform. A "uniform use"
5431 // here is something which only demands lane 0 of the unrolled iterations;
5432 // it does not imply that all lanes produce the same value (e.g. this is not
5433 // the usual meaning of uniform)
5434 SetVector<Value *> HasUniformUse;
5435
5436 // Scan the loop for instructions which are either a) known to have only
5437 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
5438 for (auto *BB : TheLoop->blocks())
5439 for (auto &I : *BB) {
5440 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I)) {
5441 switch (II->getIntrinsicID()) {
5442 case Intrinsic::sideeffect:
5443 case Intrinsic::experimental_noalias_scope_decl:
5444 case Intrinsic::assume:
5445 case Intrinsic::lifetime_start:
5446 case Intrinsic::lifetime_end:
5447 if (TheLoop->hasLoopInvariantOperands(&I))
5448 addToWorklistIfAllowed(&I);
5449 break;
5450 default:
5451 break;
5452 }
5453 }
5454
5455 // ExtractValue instructions must be uniform, because the operands are
5456 // known to be loop-invariant.
5457 if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
5458 assert(isOutOfScope(EVI->getAggregateOperand()) &&(static_cast<void> (0))
5459 "Expected aggregate value to be loop invariant")(static_cast<void> (0));
5460 addToWorklistIfAllowed(EVI);
5461 continue;
5462 }
5463
5464 // If there's no pointer operand, there's nothing to do.
5465 auto *Ptr = getLoadStorePointerOperand(&I);
5466 if (!Ptr)
5467 continue;
5468
5469 // A uniform memory op is itself uniform. We exclude uniform stores
5470 // here as they demand the last lane, not the first one.
5471 if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))
5472 addToWorklistIfAllowed(&I);
5473
5474 if (isUniformDecision(&I, VF)) {
5475 assert(isVectorizedMemAccessUse(&I, Ptr) && "consistency check")(static_cast<void> (0));
5476 HasUniformUse.insert(Ptr);
5477 }
5478 }
5479
5480 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
5481 // demanding) users. Since loops are assumed to be in LCSSA form, this
5482 // disallows uses outside the loop as well.
5483 for (auto *V : HasUniformUse) {
5484 if (isOutOfScope(V))
5485 continue;
5486 auto *I = cast<Instruction>(V);
5487 auto UsersAreMemAccesses =
5488 llvm::all_of(I->users(), [&](User *U) -> bool {
5489 return isVectorizedMemAccessUse(cast<Instruction>(U), V);
5490 });
5491 if (UsersAreMemAccesses)
5492 addToWorklistIfAllowed(I);
5493 }
5494
5495 // Expand Worklist in topological order: whenever a new instruction
5496 // is added , its users should be already inside Worklist. It ensures
5497 // a uniform instruction will only be used by uniform instructions.
5498 unsigned idx = 0;
5499 while (idx != Worklist.size()) {
5500 Instruction *I = Worklist[idx++];
5501
5502 for (auto OV : I->operand_values()) {
5503 // isOutOfScope operands cannot be uniform instructions.
5504 if (isOutOfScope(OV))
5505 continue;
5506 // First order recurrence Phi's should typically be considered
5507 // non-uniform.
5508 auto *OP = dyn_cast<PHINode>(OV);
5509 if (OP && Legal->isFirstOrderRecurrence(OP))
5510 continue;
5511 // If all the users of the operand are uniform, then add the
5512 // operand into the uniform worklist.
5513 auto *OI = cast<Instruction>(OV);
5514 if (llvm::all_of(OI->users(), [&](User *U) -> bool {
5515 auto *J = cast<Instruction>(U);
5516 return Worklist.count(J) || isVectorizedMemAccessUse(J, OI);
5517 }))
5518 addToWorklistIfAllowed(OI);
5519 }
5520 }
5521
5522 // For an instruction to be added into Worklist above, all its users inside
5523 // the loop should also be in Worklist. However, this condition cannot be
5524 // true for phi nodes that form a cyclic dependence. We must process phi
5525 // nodes separately. An induction variable will remain uniform if all users
5526 // of the induction variable and induction variable update remain uniform.
5527 // The code below handles both pointer and non-pointer induction variables.
5528 for (auto &Induction : Legal->getInductionVars()) {
5529 auto *Ind = Induction.first;
5530 auto *IndUpdate = cast<Instruction>(Ind->getIncomingValueForBlock(Latch));
5531
5532 // Determine if all users of the induction variable are uniform after
5533 // vectorization.
5534 auto UniformInd = llvm::all_of(Ind->users(), [&](User *U) -> bool {
5535 auto *I = cast<Instruction>(U);
5536 return I == IndUpdate || !TheLoop->contains(I) || Worklist.count(I) ||
5537 isVectorizedMemAccessUse(I, Ind);
5538 });
5539 if (!UniformInd)
5540 continue;
5541
5542 // Determine if all users of the induction variable update instruction are
5543 // uniform after vectorization.
5544 auto UniformIndUpdate =
5545 llvm::all_of(IndUpdate->users(), [&](User *U) -> bool {
5546 auto *I = cast<Instruction>(U);
5547 return I == Ind || !TheLoop->contains(I) || Worklist.count(I) ||
5548 isVectorizedMemAccessUse(I, IndUpdate);
5549 });
5550 if (!UniformIndUpdate)
5551 continue;
5552
5553 // The induction variable and its update instruction will remain uniform.
5554 addToWorklistIfAllowed(Ind);
5555 addToWorklistIfAllowed(IndUpdate);
5556 }
5557
5558 Uniforms[VF].insert(Worklist.begin(), Worklist.end());
5559}
5560
5561bool LoopVectorizationCostModel::runtimeChecksRequired() {
5562 LLVM_DEBUG(dbgs() << "LV: Performing code size checks.\n")do { } while (false);
5563
5564 if (Legal->getRuntimePointerChecking()->Need) {
5565 reportVectorizationFailure("Runtime ptr check is required with -Os/-Oz",
5566 "runtime pointer checks needed. Enable vectorization of this "
5567 "loop with '#pragma clang loop vectorize(enable)' when "
5568 "compiling with -Os/-Oz",
5569 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5570 return true;
5571 }
5572
5573 if (!PSE.getUnionPredicate().getPredicates().empty()) {
5574 reportVectorizationFailure("Runtime SCEV check is required with -Os/-Oz",
5575 "runtime SCEV checks needed. Enable vectorization of this "
5576 "loop with '#pragma clang loop vectorize(enable)' when "
5577 "compiling with -Os/-Oz",
5578 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5579 return true;
5580 }
5581
5582 // FIXME: Avoid specializing for stride==1 instead of bailing out.
5583 if (!Legal->getLAI()->getSymbolicStrides().empty()) {
5584 reportVectorizationFailure("Runtime stride check for small trip count",
5585 "runtime stride == 1 checks needed. Enable vectorization of "
5586 "this loop without such check by compiling with -Os/-Oz",
5587 "CantVersionLoopWithOptForSize", ORE, TheLoop);
5588 return true;
5589 }
5590
5591 return false;
5592}
5593
5594ElementCount
5595LoopVectorizationCostModel::getMaxLegalScalableVF(unsigned MaxSafeElements) {
5596 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors)
5597 return ElementCount::getScalable(0);
5598
5599 if (Hints->isScalableVectorizationDisabled()) {
5600 reportVectorizationInfo("Scalable vectorization is explicitly disabled",
5601 "ScalableVectorizationDisabled", ORE, TheLoop);
5602 return ElementCount::getScalable(0);
5603 }
5604
5605 LLVM_DEBUG(dbgs() << "LV: Scalable vectorization is available\n")do { } while (false);
5606
5607 auto MaxScalableVF = ElementCount::getScalable(
5608 std::numeric_limits<ElementCount::ScalarTy>::max());
5609
5610 // Test that the loop-vectorizer can legalize all operations for this MaxVF.
5611 // FIXME: While for scalable vectors this is currently sufficient, this should
5612 // be replaced by a more detailed mechanism that filters out specific VFs,
5613 // instead of invalidating vectorization for a whole set of VFs based on the
5614 // MaxVF.
5615
5616 // Disable scalable vectorization if the loop contains unsupported reductions.
5617 if (!canVectorizeReductions(MaxScalableVF)) {
5618 reportVectorizationInfo(
5619 "Scalable vectorization not supported for the reduction "
5620 "operations found in this loop.",
5621 "ScalableVFUnfeasible", ORE, TheLoop);
5622 return ElementCount::getScalable(0);
5623 }
5624
5625 // Disable scalable vectorization if the loop contains any instructions
5626 // with element types not supported for scalable vectors.
5627 if (any_of(ElementTypesInLoop, [&](Type *Ty) {
5628 return !Ty->isVoidTy() &&
5629 !this->TTI.isElementTypeLegalForScalableVector(Ty);
5630 })) {
5631 reportVectorizationInfo("Scalable vectorization is not supported "
5632 "for all element types found in this loop.",
5633 "ScalableVFUnfeasible", ORE, TheLoop);
5634 return ElementCount::getScalable(0);
5635 }
5636
5637 if (Legal->isSafeForAnyVectorWidth())
5638 return MaxScalableVF;
5639
5640 // Limit MaxScalableVF by the maximum safe dependence distance.
5641 Optional<unsigned> MaxVScale = TTI.getMaxVScale();
5642 if (!MaxVScale && TheFunction->hasFnAttribute(Attribute::VScaleRange)) {
5643 unsigned VScaleMax = TheFunction->getFnAttribute(Attribute::VScaleRange)
5644 .getVScaleRangeArgs()
5645 .second;
5646 if (VScaleMax > 0)
5647 MaxVScale = VScaleMax;
5648 }
5649 MaxScalableVF = ElementCount::getScalable(
5650 MaxVScale ? (MaxSafeElements / MaxVScale.getValue()) : 0);
5651 if (!MaxScalableVF)
5652 reportVectorizationInfo(
5653 "Max legal vector width too small, scalable vectorization "
5654 "unfeasible.",
5655 "ScalableVFUnfeasible", ORE, TheLoop);
5656
5657 return MaxScalableVF;
5658}
5659
5660FixedScalableVFPair
5661LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
5662 ElementCount UserVF) {
5663 MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
5664 unsigned SmallestType, WidestType;
5665 std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
5666
5667 // Get the maximum safe dependence distance in bits computed by LAA.
5668 // It is computed by MaxVF * sizeOf(type) * 8, where type is taken from
5669 // the memory accesses that is most restrictive (involved in the smallest
5670 // dependence distance).
5671 unsigned MaxSafeElements =
5672 PowerOf2Floor(Legal->getMaxSafeVectorWidthInBits() / WidestType);
5673
5674 auto MaxSafeFixedVF = ElementCount::getFixed(MaxSafeElements);
5675 auto MaxSafeScalableVF = getMaxLegalScalableVF(MaxSafeElements);
5676
5677 LLVM_DEBUG(dbgs() << "LV: The max safe fixed VF is: " << MaxSafeFixedVFdo { } while (false)
5678 << ".\n")do { } while (false);
5679 LLVM_DEBUG(dbgs() << "LV: The max safe scalable VF is: " << MaxSafeScalableVFdo { } while (false)
5680 << ".\n")do { } while (false);
5681
5682 // First analyze the UserVF, fall back if the UserVF should be ignored.
5683 if (UserVF) {
5684 auto MaxSafeUserVF =
5685 UserVF.isScalable() ? MaxSafeScalableVF : MaxSafeFixedVF;
5686
5687 if (ElementCount::isKnownLE(UserVF, MaxSafeUserVF)) {
5688 // If `VF=vscale x N` is safe, then so is `VF=N`
5689 if (UserVF.isScalable())
5690 return FixedScalableVFPair(
5691 ElementCount::getFixed(UserVF.getKnownMinValue()), UserVF);
5692 else
5693 return UserVF;
5694 }
5695
5696 assert(ElementCount::isKnownGT(UserVF, MaxSafeUserVF))(static_cast<void> (0));
5697
5698 // Only clamp if the UserVF is not scalable. If the UserVF is scalable, it
5699 // is better to ignore the hint and let the compiler choose a suitable VF.
5700 if (!UserVF.isScalable()) {
5701 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { } while (false)
5702 << " is unsafe, clamping to max safe VF="do { } while (false)
5703 << MaxSafeFixedVF << ".\n")do { } while (false);
5704 ORE->emit([&]() {
5705 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
5706 TheLoop->getStartLoc(),
5707 TheLoop->getHeader())
5708 << "User-specified vectorization factor "
5709 << ore::NV("UserVectorizationFactor", UserVF)
5710 << " is unsafe, clamping to maximum safe vectorization factor "
5711 << ore::NV("VectorizationFactor", MaxSafeFixedVF);
5712 });
5713 return MaxSafeFixedVF;
5714 }
5715
5716 if (!TTI.supportsScalableVectors() && !ForceTargetSupportsScalableVectors) {
5717 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { } while (false)
5718 << " is ignored because scalable vectors are not "do { } while (false)
5719 "available.\n")do { } while (false);
5720 ORE->emit([&]() {
5721 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
5722 TheLoop->getStartLoc(),
5723 TheLoop->getHeader())
5724 << "User-specified vectorization factor "
5725 << ore::NV("UserVectorizationFactor", UserVF)
5726 << " is ignored because the target does not support scalable "
5727 "vectors. The compiler will pick a more suitable value.";
5728 });
5729 } else {
5730 LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVFdo { } while (false)
5731 << " is unsafe. Ignoring scalable UserVF.\n")do { } while (false);
5732 ORE->emit([&]() {
5733 return OptimizationRemarkAnalysis(DEBUG_TYPE"loop-vectorize", "VectorizationFactor",
5734 TheLoop->getStartLoc(),
5735 TheLoop->getHeader())
5736 << "User-specified vectorization factor "
5737 << ore::NV("UserVectorizationFactor", UserVF)
5738 << " is unsafe. Ignoring the hint to let the compiler pick a "
5739 "more suitable value.";
5740 });
5741 }
5742 }
5743
5744 LLVM_DEBUG(dbgs() << "LV: The Smallest and Widest types: " << SmallestTypedo { } while (false)
5745 << " / " << WidestType << " bits.\n")do { } while (false);
5746
5747 FixedScalableVFPair Result(ElementCount::getFixed(1),
5748 ElementCount::getScalable(0));
5749 if (auto MaxVF = getMaximizedVFForTarget(ConstTripCount, SmallestType,
5750 WidestType, MaxSafeFixedVF))
5751 Result.FixedVF = MaxVF;
5752
5753 if (auto MaxVF = getMaximizedVFForTarget(ConstTripCount, SmallestType,
5754 WidestType, MaxSafeScalableVF))
5755 if (MaxVF.isScalable()) {
5756 Result.ScalableVF = MaxVF;
5757 LLVM_DEBUG(dbgs() << "LV: Found feasible scalable VF = " << MaxVFdo { } while (false)
5758 << "\n")do { } while (false);
5759 }
5760
5761 return Result;
5762}
5763
5764FixedScalableVFPair
5765LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
5766 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
5767 // TODO: It may by useful to do since it's still likely to be dynamically
5768 // uniform if the target can skip.
5769 reportVectorizationFailure(
5770 "Not inserting runtime ptr check for divergent target",
5771 "runtime pointer checks needed. Not enabled for divergent target",
5772 "CantVersionLoopWithDivergentTarget", ORE, TheLoop);