Bug Summary

File:llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 1984, column 23
Access to field 'IsScheduled' results in a dereference of a null pointer (loaded from variable 'SD')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SLPVectorizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-11/lib/clang/11.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/include -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-11/lib/clang/11.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-03-09-184146-41876-1 -x c++ /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10// stores that can be put together into vector-stores. Next, it attempts to
11// construct vectorizable tree using the use-def chains. If a profitable tree
12// was found, the SLP vectorizer performs vectorization on the tree.
13//
14// The pass is inspired by the work described in the paper:
15// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
20#include "llvm/ADT/ArrayRef.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/ADT/DenseSet.h"
23#include "llvm/ADT/MapVector.h"
24#include "llvm/ADT/None.h"
25#include "llvm/ADT/Optional.h"
26#include "llvm/ADT/PostOrderIterator.h"
27#include "llvm/ADT/STLExtras.h"
28#include "llvm/ADT/SetVector.h"
29#include "llvm/ADT/SmallBitVector.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/iterator.h"
35#include "llvm/ADT/iterator_range.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/Analysis/CodeMetrics.h"
38#include "llvm/Analysis/DemandedBits.h"
39#include "llvm/Analysis/GlobalsModRef.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PassManager.h"
69#include "llvm/IR/PatternMatch.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/IR/Verifier.h"
76#include "llvm/InitializePasses.h"
77#include "llvm/Pass.h"
78#include "llvm/Support/Casting.h"
79#include "llvm/Support/CommandLine.h"
80#include "llvm/Support/Compiler.h"
81#include "llvm/Support/DOTGraphTraits.h"
82#include "llvm/Support/Debug.h"
83#include "llvm/Support/ErrorHandling.h"
84#include "llvm/Support/GraphWriter.h"
85#include "llvm/Support/KnownBits.h"
86#include "llvm/Support/MathExtras.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Transforms/Utils/LoopUtils.h"
89#include "llvm/Transforms/Vectorize.h"
90#include <algorithm>
91#include <cassert>
92#include <cstdint>
93#include <iterator>
94#include <memory>
95#include <set>
96#include <string>
97#include <tuple>
98#include <utility>
99#include <vector>
100
101using namespace llvm;
102using namespace llvm::PatternMatch;
103using namespace slpvectorizer;
104
105#define SV_NAME"slp-vectorizer" "slp-vectorizer"
106#define DEBUG_TYPE"SLP" "SLP"
107
108STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated"}
;
109
110cl::opt<bool>
111 llvm::RunSLPVectorization("vectorize-slp", cl::init(false), cl::Hidden,
112 cl::desc("Run the SLP vectorization passes"));
113
114static cl::opt<int>
115 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
116 cl::desc("Only vectorize if you gain more than this "
117 "number "));
118
119static cl::opt<bool>
120ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
121 cl::desc("Attempt to vectorize horizontal reductions"));
122
123static cl::opt<bool> ShouldStartVectorizeHorAtStore(
124 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
125 cl::desc(
126 "Attempt to vectorize horizontal reductions feeding into a store"));
127
128static cl::opt<int>
129MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
130 cl::desc("Attempt to vectorize for this register size in bits"));
131
132static cl::opt<int>
133MaxStoreLookup("slp-max-store-lookup", cl::init(32), cl::Hidden,
134 cl::desc("Maximum depth of the lookup for consecutive stores."));
135
136/// Limits the size of scheduling regions in a block.
137/// It avoid long compile times for _very_ large blocks where vector
138/// instructions are spread over a wide range.
139/// This limit is way higher than needed by real-world functions.
140static cl::opt<int>
141ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
142 cl::desc("Limit the size of the SLP scheduling region per block"));
143
144static cl::opt<int> MinVectorRegSizeOption(
145 "slp-min-reg-size", cl::init(128), cl::Hidden,
146 cl::desc("Attempt to vectorize for this register size in bits"));
147
148static cl::opt<unsigned> RecursionMaxDepth(
149 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
150 cl::desc("Limit the recursion depth when building a vectorizable tree"));
151
152static cl::opt<unsigned> MinTreeSize(
153 "slp-min-tree-size", cl::init(3), cl::Hidden,
154 cl::desc("Only vectorize small trees if they are fully vectorizable"));
155
156// The maximum depth that the look-ahead score heuristic will explore.
157// The higher this value, the higher the compilation time overhead.
158static cl::opt<int> LookAheadMaxDepth(
159 "slp-max-look-ahead-depth", cl::init(2), cl::Hidden,
160 cl::desc("The maximum look-ahead depth for operand reordering scores"));
161
162// The Look-ahead heuristic goes through the users of the bundle to calculate
163// the users cost in getExternalUsesCost(). To avoid compilation time increase
164// we limit the number of users visited to this value.
165static cl::opt<unsigned> LookAheadUsersBudget(
166 "slp-look-ahead-users-budget", cl::init(2), cl::Hidden,
167 cl::desc("The maximum number of users to visit while visiting the "
168 "predecessors. This prevents compilation time increase."));
169
170static cl::opt<bool>
171 ViewSLPTree("view-slp-tree", cl::Hidden,
172 cl::desc("Display the SLP trees with Graphviz"));
173
174// Limit the number of alias checks. The limit is chosen so that
175// it has no negative effect on the llvm benchmarks.
176static const unsigned AliasedCheckLimit = 10;
177
178// Another limit for the alias checks: The maximum distance between load/store
179// instructions where alias checks are done.
180// This limit is useful for very large basic blocks.
181static const unsigned MaxMemDepDistance = 160;
182
183/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
184/// regions to be handled.
185static const int MinScheduleRegionSize = 16;
186
187/// Predicate for the element types that the SLP vectorizer supports.
188///
189/// The most important thing to filter here are types which are invalid in LLVM
190/// vectors. We also filter target specific types which have absolutely no
191/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
192/// avoids spending time checking the cost model and realizing that they will
193/// be inevitably scalarized.
194static bool isValidElementType(Type *Ty) {
195 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
196 !Ty->isPPC_FP128Ty();
197}
198
199/// \returns true if all of the instructions in \p VL are in the same block or
200/// false otherwise.
201static bool allSameBlock(ArrayRef<Value *> VL) {
202 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
203 if (!I0)
204 return false;
205 BasicBlock *BB = I0->getParent();
206 for (int i = 1, e = VL.size(); i < e; i++) {
207 Instruction *I = dyn_cast<Instruction>(VL[i]);
208 if (!I)
209 return false;
210
211 if (BB != I->getParent())
212 return false;
213 }
214 return true;
215}
216
217/// \returns True if all of the values in \p VL are constants (but not
218/// globals/constant expressions).
219static bool allConstant(ArrayRef<Value *> VL) {
220 // Constant expressions and globals can't be vectorized like normal integer/FP
221 // constants.
222 for (Value *i : VL)
223 if (!isa<Constant>(i) || isa<ConstantExpr>(i) || isa<GlobalValue>(i))
224 return false;
225 return true;
226}
227
228/// \returns True if all of the values in \p VL are identical.
229static bool isSplat(ArrayRef<Value *> VL) {
230 for (unsigned i = 1, e = VL.size(); i < e; ++i)
231 if (VL[i] != VL[0])
232 return false;
233 return true;
234}
235
236/// \returns True if \p I is commutative, handles CmpInst as well as Instruction.
237static bool isCommutative(Instruction *I) {
238 if (auto *IC = dyn_cast<CmpInst>(I))
239 return IC->isCommutative();
240 return I->isCommutative();
241}
242
243/// Checks if the vector of instructions can be represented as a shuffle, like:
244/// %x0 = extractelement <4 x i8> %x, i32 0
245/// %x3 = extractelement <4 x i8> %x, i32 3
246/// %y1 = extractelement <4 x i8> %y, i32 1
247/// %y2 = extractelement <4 x i8> %y, i32 2
248/// %x0x0 = mul i8 %x0, %x0
249/// %x3x3 = mul i8 %x3, %x3
250/// %y1y1 = mul i8 %y1, %y1
251/// %y2y2 = mul i8 %y2, %y2
252/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
253/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
254/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
255/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
256/// ret <4 x i8> %ins4
257/// can be transformed into:
258/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
259/// i32 6>
260/// %2 = mul <4 x i8> %1, %1
261/// ret <4 x i8> %2
262/// We convert this initially to something like:
263/// %x0 = extractelement <4 x i8> %x, i32 0
264/// %x3 = extractelement <4 x i8> %x, i32 3
265/// %y1 = extractelement <4 x i8> %y, i32 1
266/// %y2 = extractelement <4 x i8> %y, i32 2
267/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
268/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
269/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
270/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
271/// %5 = mul <4 x i8> %4, %4
272/// %6 = extractelement <4 x i8> %5, i32 0
273/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
274/// %7 = extractelement <4 x i8> %5, i32 1
275/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
276/// %8 = extractelement <4 x i8> %5, i32 2
277/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
278/// %9 = extractelement <4 x i8> %5, i32 3
279/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
280/// ret <4 x i8> %ins4
281/// InstCombiner transforms this into a shuffle and vector mul
282/// TODO: Can we split off and reuse the shuffle mask detection from
283/// TargetTransformInfo::getInstructionThroughput?
284static Optional<TargetTransformInfo::ShuffleKind>
285isShuffle(ArrayRef<Value *> VL) {
286 auto *EI0 = cast<ExtractElementInst>(VL[0]);
287 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
288 Value *Vec1 = nullptr;
289 Value *Vec2 = nullptr;
290 enum ShuffleMode { Unknown, Select, Permute };
291 ShuffleMode CommonShuffleMode = Unknown;
292 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
293 auto *EI = cast<ExtractElementInst>(VL[I]);
294 auto *Vec = EI->getVectorOperand();
295 // All vector operands must have the same number of vector elements.
296 if (Vec->getType()->getVectorNumElements() != Size)
297 return None;
298 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
299 if (!Idx)
300 return None;
301 // Undefined behavior if Idx is negative or >= Size.
302 if (Idx->getValue().uge(Size))
303 continue;
304 unsigned IntIdx = Idx->getValue().getZExtValue();
305 // We can extractelement from undef vector.
306 if (isa<UndefValue>(Vec))
307 continue;
308 // For correct shuffling we have to have at most 2 different vector operands
309 // in all extractelement instructions.
310 if (!Vec1 || Vec1 == Vec)
311 Vec1 = Vec;
312 else if (!Vec2 || Vec2 == Vec)
313 Vec2 = Vec;
314 else
315 return None;
316 if (CommonShuffleMode == Permute)
317 continue;
318 // If the extract index is not the same as the operation number, it is a
319 // permutation.
320 if (IntIdx != I) {
321 CommonShuffleMode = Permute;
322 continue;
323 }
324 CommonShuffleMode = Select;
325 }
326 // If we're not crossing lanes in different vectors, consider it as blending.
327 if (CommonShuffleMode == Select && Vec2)
328 return TargetTransformInfo::SK_Select;
329 // If Vec2 was never used, we have a permutation of a single vector, otherwise
330 // we have permutation of 2 vectors.
331 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
332 : TargetTransformInfo::SK_PermuteSingleSrc;
333}
334
335namespace {
336
337/// Main data required for vectorization of instructions.
338struct InstructionsState {
339 /// The very first instruction in the list with the main opcode.
340 Value *OpValue = nullptr;
341
342 /// The main/alternate instruction.
343 Instruction *MainOp = nullptr;
344 Instruction *AltOp = nullptr;
345
346 /// The main/alternate opcodes for the list of instructions.
347 unsigned getOpcode() const {
348 return MainOp ? MainOp->getOpcode() : 0;
349 }
350
351 unsigned getAltOpcode() const {
352 return AltOp ? AltOp->getOpcode() : 0;
353 }
354
355 /// Some of the instructions in the list have alternate opcodes.
356 bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
357
358 bool isOpcodeOrAlt(Instruction *I) const {
359 unsigned CheckedOpcode = I->getOpcode();
360 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
361 }
362
363 InstructionsState() = delete;
364 InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
365 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
366};
367
368} // end anonymous namespace
369
370/// Chooses the correct key for scheduling data. If \p Op has the same (or
371/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
372/// OpValue.
373static Value *isOneOf(const InstructionsState &S, Value *Op) {
374 auto *I = dyn_cast<Instruction>(Op);
375 if (I && S.isOpcodeOrAlt(I))
376 return Op;
377 return S.OpValue;
378}
379
380/// \returns true if \p Opcode is allowed as part of of the main/alternate
381/// instruction for SLP vectorization.
382///
383/// Example of unsupported opcode is SDIV that can potentially cause UB if the
384/// "shuffled out" lane would result in division by zero.
385static bool isValidForAlternation(unsigned Opcode) {
386 if (Instruction::isIntDivRem(Opcode))
387 return false;
388
389 return true;
390}
391
392/// \returns analysis of the Instructions in \p VL described in
393/// InstructionsState, the Opcode that we suppose the whole list
394/// could be vectorized even if its structure is diverse.
395static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
396 unsigned BaseIndex = 0) {
397 // Make sure these are all Instructions.
398 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
399 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
400
401 bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
402 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
403 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
404 unsigned AltOpcode = Opcode;
405 unsigned AltIndex = BaseIndex;
406
407 // Check for one alternate opcode from another BinaryOperator.
408 // TODO - generalize to support all operators (types, calls etc.).
409 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
410 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
411 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
412 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
413 continue;
414 if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) &&
415 isValidForAlternation(Opcode)) {
416 AltOpcode = InstOpcode;
417 AltIndex = Cnt;
418 continue;
419 }
420 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
421 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
422 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
423 if (Ty0 == Ty1) {
424 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
425 continue;
426 if (Opcode == AltOpcode) {
427 assert(isValidForAlternation(Opcode) &&((isValidForAlternation(Opcode) && isValidForAlternation
(InstOpcode) && "Cast isn't safe for alternation, logic needs to be updated!"
) ? static_cast<void> (0) : __assert_fail ("isValidForAlternation(Opcode) && isValidForAlternation(InstOpcode) && \"Cast isn't safe for alternation, logic needs to be updated!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 429, __PRETTY_FUNCTION__))
428 isValidForAlternation(InstOpcode) &&((isValidForAlternation(Opcode) && isValidForAlternation
(InstOpcode) && "Cast isn't safe for alternation, logic needs to be updated!"
) ? static_cast<void> (0) : __assert_fail ("isValidForAlternation(Opcode) && isValidForAlternation(InstOpcode) && \"Cast isn't safe for alternation, logic needs to be updated!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 429, __PRETTY_FUNCTION__))
429 "Cast isn't safe for alternation, logic needs to be updated!")((isValidForAlternation(Opcode) && isValidForAlternation
(InstOpcode) && "Cast isn't safe for alternation, logic needs to be updated!"
) ? static_cast<void> (0) : __assert_fail ("isValidForAlternation(Opcode) && isValidForAlternation(InstOpcode) && \"Cast isn't safe for alternation, logic needs to be updated!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 429, __PRETTY_FUNCTION__))
;
430 AltOpcode = InstOpcode;
431 AltIndex = Cnt;
432 continue;
433 }
434 }
435 } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
436 continue;
437 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
438 }
439
440 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
441 cast<Instruction>(VL[AltIndex]));
442}
443
444/// \returns true if all of the values in \p VL have the same type or false
445/// otherwise.
446static bool allSameType(ArrayRef<Value *> VL) {
447 Type *Ty = VL[0]->getType();
448 for (int i = 1, e = VL.size(); i < e; i++)
449 if (VL[i]->getType() != Ty)
450 return false;
451
452 return true;
453}
454
455/// \returns True if Extract{Value,Element} instruction extracts element Idx.
456static Optional<unsigned> getExtractIndex(Instruction *E) {
457 unsigned Opcode = E->getOpcode();
458 assert((Opcode == Instruction::ExtractElement ||(((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? static_cast<void> (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 460, __PRETTY_FUNCTION__))
459 Opcode == Instruction::ExtractValue) &&(((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? static_cast<void> (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 460, __PRETTY_FUNCTION__))
460 "Expected extractelement or extractvalue instruction.")(((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? static_cast<void> (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 460, __PRETTY_FUNCTION__))
;
461 if (Opcode == Instruction::ExtractElement) {
462 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
463 if (!CI)
464 return None;
465 return CI->getZExtValue();
466 }
467 ExtractValueInst *EI = cast<ExtractValueInst>(E);
468 if (EI->getNumIndices() != 1)
469 return None;
470 return *EI->idx_begin();
471}
472
473/// \returns True if in-tree use also needs extract. This refers to
474/// possible scalar operand in vectorized instruction.
475static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
476 TargetLibraryInfo *TLI) {
477 unsigned Opcode = UserInst->getOpcode();
478 switch (Opcode) {
479 case Instruction::Load: {
480 LoadInst *LI = cast<LoadInst>(UserInst);
481 return (LI->getPointerOperand() == Scalar);
482 }
483 case Instruction::Store: {
484 StoreInst *SI = cast<StoreInst>(UserInst);
485 return (SI->getPointerOperand() == Scalar);
486 }
487 case Instruction::Call: {
488 CallInst *CI = cast<CallInst>(UserInst);
489 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
490 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
491 if (hasVectorInstrinsicScalarOpd(ID, i))
492 return (CI->getArgOperand(i) == Scalar);
493 }
494 LLVM_FALLTHROUGH[[gnu::fallthrough]];
495 }
496 default:
497 return false;
498 }
499}
500
501/// \returns the AA location that is being access by the instruction.
502static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
503 if (StoreInst *SI = dyn_cast<StoreInst>(I))
504 return MemoryLocation::get(SI);
505 if (LoadInst *LI = dyn_cast<LoadInst>(I))
506 return MemoryLocation::get(LI);
507 return MemoryLocation();
508}
509
510/// \returns True if the instruction is not a volatile or atomic load/store.
511static bool isSimple(Instruction *I) {
512 if (LoadInst *LI = dyn_cast<LoadInst>(I))
513 return LI->isSimple();
514 if (StoreInst *SI = dyn_cast<StoreInst>(I))
515 return SI->isSimple();
516 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
517 return !MI->isVolatile();
518 return true;
519}
520
521namespace llvm {
522
523namespace slpvectorizer {
524
525/// Bottom Up SLP Vectorizer.
526class BoUpSLP {
527 struct TreeEntry;
528 struct ScheduleData;
529
530public:
531 using ValueList = SmallVector<Value *, 8>;
532 using InstrList = SmallVector<Instruction *, 16>;
533 using ValueSet = SmallPtrSet<Value *, 16>;
534 using StoreList = SmallVector<StoreInst *, 8>;
535 using ExtraValueToDebugLocsMap =
536 MapVector<Value *, SmallVector<Instruction *, 2>>;
537
538 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
539 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
540 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
541 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
542 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
543 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
544 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
545 // Use the vector register size specified by the target unless overridden
546 // by a command-line option.
547 // TODO: It would be better to limit the vectorization factor based on
548 // data type rather than just register size. For example, x86 AVX has
549 // 256-bit registers, but it does not support integer operations
550 // at that width (that requires AVX2).
551 if (MaxVectorRegSizeOption.getNumOccurrences())
552 MaxVecRegSize = MaxVectorRegSizeOption;
553 else
554 MaxVecRegSize = TTI->getRegisterBitWidth(true);
555
556 if (MinVectorRegSizeOption.getNumOccurrences())
557 MinVecRegSize = MinVectorRegSizeOption;
558 else
559 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
560 }
561
562 /// Vectorize the tree that starts with the elements in \p VL.
563 /// Returns the vectorized root.
564 Value *vectorizeTree();
565
566 /// Vectorize the tree but with the list of externally used values \p
567 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
568 /// generated extractvalue instructions.
569 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
570
571 /// \returns the cost incurred by unwanted spills and fills, caused by
572 /// holding live values over call sites.
573 int getSpillCost() const;
574
575 /// \returns the vectorization cost of the subtree that starts at \p VL.
576 /// A negative number means that this is profitable.
577 int getTreeCost();
578
579 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
580 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
581 void buildTree(ArrayRef<Value *> Roots,
582 ArrayRef<Value *> UserIgnoreLst = None);
583
584 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
585 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
586 /// into account (and updating it, if required) list of externally used
587 /// values stored in \p ExternallyUsedValues.
588 void buildTree(ArrayRef<Value *> Roots,
589 ExtraValueToDebugLocsMap &ExternallyUsedValues,
590 ArrayRef<Value *> UserIgnoreLst = None);
591
592 /// Clear the internal data structures that are created by 'buildTree'.
593 void deleteTree() {
594 VectorizableTree.clear();
595 ScalarToTreeEntry.clear();
596 MustGather.clear();
597 ExternalUses.clear();
598 NumOpsWantToKeepOrder.clear();
599 NumOpsWantToKeepOriginalOrder = 0;
600 for (auto &Iter : BlocksSchedules) {
601 BlockScheduling *BS = Iter.second.get();
602 BS->clear();
603 }
604 MinBWs.clear();
605 }
606
607 unsigned getTreeSize() const { return VectorizableTree.size(); }
608
609 /// Perform LICM and CSE on the newly generated gather sequences.
610 void optimizeGatherSequence();
611
612 /// \returns The best order of instructions for vectorization.
613 Optional<ArrayRef<unsigned>> bestOrder() const {
614 auto I = std::max_element(
615 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
616 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
617 const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
618 return D1.second < D2.second;
619 });
620 if (I == NumOpsWantToKeepOrder.end() ||
621 I->getSecond() <= NumOpsWantToKeepOriginalOrder)
622 return None;
623
624 return makeArrayRef(I->getFirst());
625 }
626
627 /// \return The vector element size in bits to use when vectorizing the
628 /// expression tree ending at \p V. If V is a store, the size is the width of
629 /// the stored value. Otherwise, the size is the width of the largest loaded
630 /// value reaching V. This method is used by the vectorizer to calculate
631 /// vectorization factors.
632 unsigned getVectorElementSize(Value *V) const;
633
634 /// Compute the minimum type sizes required to represent the entries in a
635 /// vectorizable tree.
636 void computeMinimumValueSizes();
637
638 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
639 unsigned getMaxVecRegSize() const {
640 return MaxVecRegSize;
641 }
642
643 // \returns minimum vector register size as set by cl::opt.
644 unsigned getMinVecRegSize() const {
645 return MinVecRegSize;
646 }
647
648 /// Check if homogeneous aggregate is isomorphic to some VectorType.
649 /// Accepts homogeneous multidimensional aggregate of scalars/vectors like
650 /// {[4 x i16], [4 x i16]}, { <2 x float>, <2 x float> },
651 /// {{{i16, i16}, {i16, i16}}, {{i16, i16}, {i16, i16}}} and so on.
652 ///
653 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
654 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
655
656 /// \returns True if the VectorizableTree is both tiny and not fully
657 /// vectorizable. We do not vectorize such trees.
658 bool isTreeTinyAndNotFullyVectorizable() const;
659
660 /// Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values
661 /// can be load combined in the backend. Load combining may not be allowed in
662 /// the IR optimizer, so we do not want to alter the pattern. For example,
663 /// partially transforming a scalar bswap() pattern into vector code is
664 /// effectively impossible for the backend to undo.
665 /// TODO: If load combining is allowed in the IR optimizer, this analysis
666 /// may not be necessary.
667 bool isLoadCombineReductionCandidate(unsigned ReductionOpcode) const;
668
669 OptimizationRemarkEmitter *getORE() { return ORE; }
670
671 /// This structure holds any data we need about the edges being traversed
672 /// during buildTree_rec(). We keep track of:
673 /// (i) the user TreeEntry index, and
674 /// (ii) the index of the edge.
675 struct EdgeInfo {
676 EdgeInfo() = default;
677 EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx)
678 : UserTE(UserTE), EdgeIdx(EdgeIdx) {}
679 /// The user TreeEntry.
680 TreeEntry *UserTE = nullptr;
681 /// The operand index of the use.
682 unsigned EdgeIdx = UINT_MAX(2147483647 *2U +1U);
683#ifndef NDEBUG
684 friend inline raw_ostream &operator<<(raw_ostream &OS,
685 const BoUpSLP::EdgeInfo &EI) {
686 EI.dump(OS);
687 return OS;
688 }
689 /// Debug print.
690 void dump(raw_ostream &OS) const {
691 OS << "{User:" << (UserTE ? std::to_string(UserTE->Idx) : "null")
692 << " EdgeIdx:" << EdgeIdx << "}";
693 }
694 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dump() const { dump(dbgs()); }
695#endif
696 };
697
698 /// A helper data structure to hold the operands of a vector of instructions.
699 /// This supports a fixed vector length for all operand vectors.
700 class VLOperands {
701 /// For each operand we need (i) the value, and (ii) the opcode that it
702 /// would be attached to if the expression was in a left-linearized form.
703 /// This is required to avoid illegal operand reordering.
704 /// For example:
705 /// \verbatim
706 /// 0 Op1
707 /// |/
708 /// Op1 Op2 Linearized + Op2
709 /// \ / ----------> |/
710 /// - -
711 ///
712 /// Op1 - Op2 (0 + Op1) - Op2
713 /// \endverbatim
714 ///
715 /// Value Op1 is attached to a '+' operation, and Op2 to a '-'.
716 ///
717 /// Another way to think of this is to track all the operations across the
718 /// path from the operand all the way to the root of the tree and to
719 /// calculate the operation that corresponds to this path. For example, the
720 /// path from Op2 to the root crosses the RHS of the '-', therefore the
721 /// corresponding operation is a '-' (which matches the one in the
722 /// linearized tree, as shown above).
723 ///
724 /// For lack of a better term, we refer to this operation as Accumulated
725 /// Path Operation (APO).
726 struct OperandData {
727 OperandData() = default;
728 OperandData(Value *V, bool APO, bool IsUsed)
729 : V(V), APO(APO), IsUsed(IsUsed) {}
730 /// The operand value.
731 Value *V = nullptr;
732 /// TreeEntries only allow a single opcode, or an alternate sequence of
733 /// them (e.g, +, -). Therefore, we can safely use a boolean value for the
734 /// APO. It is set to 'true' if 'V' is attached to an inverse operation
735 /// in the left-linearized form (e.g., Sub/Div), and 'false' otherwise
736 /// (e.g., Add/Mul)
737 bool APO = false;
738 /// Helper data for the reordering function.
739 bool IsUsed = false;
740 };
741
742 /// During operand reordering, we are trying to select the operand at lane
743 /// that matches best with the operand at the neighboring lane. Our
744 /// selection is based on the type of value we are looking for. For example,
745 /// if the neighboring lane has a load, we need to look for a load that is
746 /// accessing a consecutive address. These strategies are summarized in the
747 /// 'ReorderingMode' enumerator.
748 enum class ReorderingMode {
749 Load, ///< Matching loads to consecutive memory addresses
750 Opcode, ///< Matching instructions based on opcode (same or alternate)
751 Constant, ///< Matching constants
752 Splat, ///< Matching the same instruction multiple times (broadcast)
753 Failed, ///< We failed to create a vectorizable group
754 };
755
756 using OperandDataVec = SmallVector<OperandData, 2>;
757
758 /// A vector of operand vectors.
759 SmallVector<OperandDataVec, 4> OpsVec;
760
761 const DataLayout &DL;
762 ScalarEvolution &SE;
763 const BoUpSLP &R;
764
765 /// \returns the operand data at \p OpIdx and \p Lane.
766 OperandData &getData(unsigned OpIdx, unsigned Lane) {
767 return OpsVec[OpIdx][Lane];
768 }
769
770 /// \returns the operand data at \p OpIdx and \p Lane. Const version.
771 const OperandData &getData(unsigned OpIdx, unsigned Lane) const {
772 return OpsVec[OpIdx][Lane];
773 }
774
775 /// Clears the used flag for all entries.
776 void clearUsed() {
777 for (unsigned OpIdx = 0, NumOperands = getNumOperands();
778 OpIdx != NumOperands; ++OpIdx)
779 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
780 ++Lane)
781 OpsVec[OpIdx][Lane].IsUsed = false;
782 }
783
784 /// Swap the operand at \p OpIdx1 with that one at \p OpIdx2.
785 void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) {
786 std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
787 }
788
789 // The hard-coded scores listed here are not very important. When computing
790 // the scores of matching one sub-tree with another, we are basically
791 // counting the number of values that are matching. So even if all scores
792 // are set to 1, we would still get a decent matching result.
793 // However, sometimes we have to break ties. For example we may have to
794 // choose between matching loads vs matching opcodes. This is what these
795 // scores are helping us with: they provide the order of preference.
796
797 /// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
798 static const int ScoreConsecutiveLoads = 3;
799 /// ExtractElementInst from same vector and consecutive indexes.
800 static const int ScoreConsecutiveExtracts = 3;
801 /// Constants.
802 static const int ScoreConstants = 2;
803 /// Instructions with the same opcode.
804 static const int ScoreSameOpcode = 2;
805 /// Instructions with alt opcodes (e.g, add + sub).
806 static const int ScoreAltOpcodes = 1;
807 /// Identical instructions (a.k.a. splat or broadcast).
808 static const int ScoreSplat = 1;
809 /// Matching with an undef is preferable to failing.
810 static const int ScoreUndef = 1;
811 /// Score for failing to find a decent match.
812 static const int ScoreFail = 0;
813 /// User exteranl to the vectorized code.
814 static const int ExternalUseCost = 1;
815 /// The user is internal but in a different lane.
816 static const int UserInDiffLaneCost = ExternalUseCost;
817
818 /// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
819 static int getShallowScore(Value *V1, Value *V2, const DataLayout &DL,
820 ScalarEvolution &SE) {
821 auto *LI1 = dyn_cast<LoadInst>(V1);
822 auto *LI2 = dyn_cast<LoadInst>(V2);
823 if (LI1 && LI2)
824 return isConsecutiveAccess(LI1, LI2, DL, SE)
825 ? VLOperands::ScoreConsecutiveLoads
826 : VLOperands::ScoreFail;
827
828 auto *C1 = dyn_cast<Constant>(V1);
829 auto *C2 = dyn_cast<Constant>(V2);
830 if (C1 && C2)
831 return VLOperands::ScoreConstants;
832
833 // Extracts from consecutive indexes of the same vector better score as
834 // the extracts could be optimized away.
835 Value *EV;
836 ConstantInt *Ex1Idx, *Ex2Idx;
837 if (match(V1, m_ExtractElement(m_Value(EV), m_ConstantInt(Ex1Idx))) &&
838 match(V2, m_ExtractElement(m_Deferred(EV), m_ConstantInt(Ex2Idx))) &&
839 Ex1Idx->getZExtValue() + 1 == Ex2Idx->getZExtValue())
840 return VLOperands::ScoreConsecutiveExtracts;
841
842 auto *I1 = dyn_cast<Instruction>(V1);
843 auto *I2 = dyn_cast<Instruction>(V2);
844 if (I1 && I2) {
845 if (I1 == I2)
846 return VLOperands::ScoreSplat;
847 InstructionsState S = getSameOpcode({I1, I2});
848 // Note: Only consider instructions with <= 2 operands to avoid
849 // complexity explosion.
850 if (S.getOpcode() && S.MainOp->getNumOperands() <= 2)
851 return S.isAltShuffle() ? VLOperands::ScoreAltOpcodes
852 : VLOperands::ScoreSameOpcode;
853 }
854
855 if (isa<UndefValue>(V2))
856 return VLOperands::ScoreUndef;
857
858 return VLOperands::ScoreFail;
859 }
860
861 /// Holds the values and their lane that are taking part in the look-ahead
862 /// score calculation. This is used in the external uses cost calculation.
863 SmallDenseMap<Value *, int> InLookAheadValues;
864
865 /// \Returns the additinal cost due to uses of \p LHS and \p RHS that are
866 /// either external to the vectorized code, or require shuffling.
867 int getExternalUsesCost(const std::pair<Value *, int> &LHS,
868 const std::pair<Value *, int> &RHS) {
869 int Cost = 0;
870 std::array<std::pair<Value *, int>, 2> Values = {{LHS, RHS}};
871 for (int Idx = 0, IdxE = Values.size(); Idx != IdxE; ++Idx) {
872 Value *V = Values[Idx].first;
873 // Calculate the absolute lane, using the minimum relative lane of LHS
874 // and RHS as base and Idx as the offset.
875 int Ln = std::min(LHS.second, RHS.second) + Idx;
876 assert(Ln >= 0 && "Bad lane calculation")((Ln >= 0 && "Bad lane calculation") ? static_cast
<void> (0) : __assert_fail ("Ln >= 0 && \"Bad lane calculation\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 876, __PRETTY_FUNCTION__))
;
877 unsigned UsersBudget = LookAheadUsersBudget;
878 for (User *U : V->users()) {
879 if (const TreeEntry *UserTE = R.getTreeEntry(U)) {
880 // The user is in the VectorizableTree. Check if we need to insert.
881 auto It = llvm::find(UserTE->Scalars, U);
882 assert(It != UserTE->Scalars.end() && "U is in UserTE")((It != UserTE->Scalars.end() && "U is in UserTE")
? static_cast<void> (0) : __assert_fail ("It != UserTE->Scalars.end() && \"U is in UserTE\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 882, __PRETTY_FUNCTION__))
;
883 int UserLn = std::distance(UserTE->Scalars.begin(), It);
884 assert(UserLn >= 0 && "Bad lane")((UserLn >= 0 && "Bad lane") ? static_cast<void
> (0) : __assert_fail ("UserLn >= 0 && \"Bad lane\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 884, __PRETTY_FUNCTION__))
;
885 if (UserLn != Ln)
886 Cost += UserInDiffLaneCost;
887 } else {
888 // Check if the user is in the look-ahead code.
889 auto It2 = InLookAheadValues.find(U);
890 if (It2 != InLookAheadValues.end()) {
891 // The user is in the look-ahead code. Check the lane.
892 if (It2->second != Ln)
893 Cost += UserInDiffLaneCost;
894 } else {
895 // The user is neither in SLP tree nor in the look-ahead code.
896 Cost += ExternalUseCost;
897 }
898 }
899 // Limit the number of visited uses to cap compilation time.
900 if (--UsersBudget == 0)
901 break;
902 }
903 }
904 return Cost;
905 }
906
907 /// Go through the operands of \p LHS and \p RHS recursively until \p
908 /// MaxLevel, and return the cummulative score. For example:
909 /// \verbatim
910 /// A[0] B[0] A[1] B[1] C[0] D[0] B[1] A[1]
911 /// \ / \ / \ / \ /
912 /// + + + +
913 /// G1 G2 G3 G4
914 /// \endverbatim
915 /// The getScoreAtLevelRec(G1, G2) function will try to match the nodes at
916 /// each level recursively, accumulating the score. It starts from matching
917 /// the additions at level 0, then moves on to the loads (level 1). The
918 /// score of G1 and G2 is higher than G1 and G3, because {A[0],A[1]} and
919 /// {B[0],B[1]} match with VLOperands::ScoreConsecutiveLoads, while
920 /// {A[0],C[0]} has a score of VLOperands::ScoreFail.
921 /// Please note that the order of the operands does not matter, as we
922 /// evaluate the score of all profitable combinations of operands. In
923 /// other words the score of G1 and G4 is the same as G1 and G2. This
924 /// heuristic is based on ideas described in:
925 /// Look-ahead SLP: Auto-vectorization in the presence of commutative
926 /// operations, CGO 2018 by Vasileios Porpodas, Rodrigo C. O. Rocha,
927 /// Luís F. W. Góes
928 int getScoreAtLevelRec(const std::pair<Value *, int> &LHS,
929 const std::pair<Value *, int> &RHS, int CurrLevel,
930 int MaxLevel) {
931
932 Value *V1 = LHS.first;
933 Value *V2 = RHS.first;
934 // Get the shallow score of V1 and V2.
935 int ShallowScoreAtThisLevel =
936 std::max((int)ScoreFail, getShallowScore(V1, V2, DL, SE) -
937 getExternalUsesCost(LHS, RHS));
938 int Lane1 = LHS.second;
939 int Lane2 = RHS.second;
940
941 // If reached MaxLevel,
942 // or if V1 and V2 are not instructions,
943 // or if they are SPLAT,
944 // or if they are not consecutive, early return the current cost.
945 auto *I1 = dyn_cast<Instruction>(V1);
946 auto *I2 = dyn_cast<Instruction>(V2);
947 if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 ||
948 ShallowScoreAtThisLevel == VLOperands::ScoreFail ||
949 (isa<LoadInst>(I1) && isa<LoadInst>(I2) && ShallowScoreAtThisLevel))
950 return ShallowScoreAtThisLevel;
951 assert(I1 && I2 && "Should have early exited.")((I1 && I2 && "Should have early exited.") ? static_cast
<void> (0) : __assert_fail ("I1 && I2 && \"Should have early exited.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 951, __PRETTY_FUNCTION__))
;
952
953 // Keep track of in-tree values for determining the external-use cost.
954 InLookAheadValues[V1] = Lane1;
955 InLookAheadValues[V2] = Lane2;
956
957 // Contains the I2 operand indexes that got matched with I1 operands.
958 SmallSet<unsigned, 4> Op2Used;
959
960 // Recursion towards the operands of I1 and I2. We are trying all possbile
961 // operand pairs, and keeping track of the best score.
962 for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands();
963 OpIdx1 != NumOperands1; ++OpIdx1) {
964 // Try to pair op1I with the best operand of I2.
965 int MaxTmpScore = 0;
966 unsigned MaxOpIdx2 = 0;
967 bool FoundBest = false;
968 // If I2 is commutative try all combinations.
969 unsigned FromIdx = isCommutative(I2) ? 0 : OpIdx1;
970 unsigned ToIdx = isCommutative(I2)
971 ? I2->getNumOperands()
972 : std::min(I2->getNumOperands(), OpIdx1 + 1);
973 assert(FromIdx <= ToIdx && "Bad index")((FromIdx <= ToIdx && "Bad index") ? static_cast<
void> (0) : __assert_fail ("FromIdx <= ToIdx && \"Bad index\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 973, __PRETTY_FUNCTION__))
;
974 for (unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) {
975 // Skip operands already paired with OpIdx1.
976 if (Op2Used.count(OpIdx2))
977 continue;
978 // Recursively calculate the cost at each level
979 int TmpScore = getScoreAtLevelRec({I1->getOperand(OpIdx1), Lane1},
980 {I2->getOperand(OpIdx2), Lane2},
981 CurrLevel + 1, MaxLevel);
982 // Look for the best score.
983 if (TmpScore > VLOperands::ScoreFail && TmpScore > MaxTmpScore) {
984 MaxTmpScore = TmpScore;
985 MaxOpIdx2 = OpIdx2;
986 FoundBest = true;
987 }
988 }
989 if (FoundBest) {
990 // Pair {OpIdx1, MaxOpIdx2} was found to be best. Never revisit it.
991 Op2Used.insert(MaxOpIdx2);
992 ShallowScoreAtThisLevel += MaxTmpScore;
993 }
994 }
995 return ShallowScoreAtThisLevel;
996 }
997
998 /// \Returns the look-ahead score, which tells us how much the sub-trees
999 /// rooted at \p LHS and \p RHS match, the more they match the higher the
1000 /// score. This helps break ties in an informed way when we cannot decide on
1001 /// the order of the operands by just considering the immediate
1002 /// predecessors.
1003 int getLookAheadScore(const std::pair<Value *, int> &LHS,
1004 const std::pair<Value *, int> &RHS) {
1005 InLookAheadValues.clear();
1006 return getScoreAtLevelRec(LHS, RHS, 1, LookAheadMaxDepth);
1007 }
1008
1009 // Search all operands in Ops[*][Lane] for the one that matches best
1010 // Ops[OpIdx][LastLane] and return its opreand index.
1011 // If no good match can be found, return None.
1012 Optional<unsigned>
1013 getBestOperand(unsigned OpIdx, int Lane, int LastLane,
1014 ArrayRef<ReorderingMode> ReorderingModes) {
1015 unsigned NumOperands = getNumOperands();
1016
1017 // The operand of the previous lane at OpIdx.
1018 Value *OpLastLane = getData(OpIdx, LastLane).V;
1019
1020 // Our strategy mode for OpIdx.
1021 ReorderingMode RMode = ReorderingModes[OpIdx];
1022
1023 // The linearized opcode of the operand at OpIdx, Lane.
1024 bool OpIdxAPO = getData(OpIdx, Lane).APO;
1025
1026 // The best operand index and its score.
1027 // Sometimes we have more than one option (e.g., Opcode and Undefs), so we
1028 // are using the score to differentiate between the two.
1029 struct BestOpData {
1030 Optional<unsigned> Idx = None;
1031 unsigned Score = 0;
1032 } BestOp;
1033
1034 // Iterate through all unused operands and look for the best.
1035 for (unsigned Idx = 0; Idx != NumOperands; ++Idx) {
1036 // Get the operand at Idx and Lane.
1037 OperandData &OpData = getData(Idx, Lane);
1038 Value *Op = OpData.V;
1039 bool OpAPO = OpData.APO;
1040
1041 // Skip already selected operands.
1042 if (OpData.IsUsed)
1043 continue;
1044
1045 // Skip if we are trying to move the operand to a position with a
1046 // different opcode in the linearized tree form. This would break the
1047 // semantics.
1048 if (OpAPO != OpIdxAPO)
1049 continue;
1050
1051 // Look for an operand that matches the current mode.
1052 switch (RMode) {
1053 case ReorderingMode::Load:
1054 case ReorderingMode::Constant:
1055 case ReorderingMode::Opcode: {
1056 bool LeftToRight = Lane > LastLane;
1057 Value *OpLeft = (LeftToRight) ? OpLastLane : Op;
1058 Value *OpRight = (LeftToRight) ? Op : OpLastLane;
1059 unsigned Score =
1060 getLookAheadScore({OpLeft, LastLane}, {OpRight, Lane});
1061 if (Score > BestOp.Score) {
1062 BestOp.Idx = Idx;
1063 BestOp.Score = Score;
1064 }
1065 break;
1066 }
1067 case ReorderingMode::Splat:
1068 if (Op == OpLastLane)
1069 BestOp.Idx = Idx;
1070 break;
1071 case ReorderingMode::Failed:
1072 return None;
1073 }
1074 }
1075
1076 if (BestOp.Idx) {
1077 getData(BestOp.Idx.getValue(), Lane).IsUsed = true;
1078 return BestOp.Idx;
1079 }
1080 // If we could not find a good match return None.
1081 return None;
1082 }
1083
1084 /// Helper for reorderOperandVecs. \Returns the lane that we should start
1085 /// reordering from. This is the one which has the least number of operands
1086 /// that can freely move about.
1087 unsigned getBestLaneToStartReordering() const {
1088 unsigned BestLane = 0;
1089 unsigned Min = UINT_MAX(2147483647 *2U +1U);
1090 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
1091 ++Lane) {
1092 unsigned NumFreeOps = getMaxNumOperandsThatCanBeReordered(Lane);
1093 if (NumFreeOps < Min) {
1094 Min = NumFreeOps;
1095 BestLane = Lane;
1096 }
1097 }
1098 return BestLane;
1099 }
1100
1101 /// \Returns the maximum number of operands that are allowed to be reordered
1102 /// for \p Lane. This is used as a heuristic for selecting the first lane to
1103 /// start operand reordering.
1104 unsigned getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
1105 unsigned CntTrue = 0;
1106 unsigned NumOperands = getNumOperands();
1107 // Operands with the same APO can be reordered. We therefore need to count
1108 // how many of them we have for each APO, like this: Cnt[APO] = x.
1109 // Since we only have two APOs, namely true and false, we can avoid using
1110 // a map. Instead we can simply count the number of operands that
1111 // correspond to one of them (in this case the 'true' APO), and calculate
1112 // the other by subtracting it from the total number of operands.
1113 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx)
1114 if (getData(OpIdx, Lane).APO)
1115 ++CntTrue;
1116 unsigned CntFalse = NumOperands - CntTrue;
1117 return std::max(CntTrue, CntFalse);
1118 }
1119
1120 /// Go through the instructions in VL and append their operands.
1121 void appendOperandsOfVL(ArrayRef<Value *> VL) {
1122 assert(!VL.empty() && "Bad VL")((!VL.empty() && "Bad VL") ? static_cast<void> (
0) : __assert_fail ("!VL.empty() && \"Bad VL\"", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1122, __PRETTY_FUNCTION__))
;
1123 assert((empty() || VL.size() == getNumLanes()) &&(((empty() || VL.size() == getNumLanes()) && "Expected same number of lanes"
) ? static_cast<void> (0) : __assert_fail ("(empty() || VL.size() == getNumLanes()) && \"Expected same number of lanes\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1124, __PRETTY_FUNCTION__))
1124 "Expected same number of lanes")(((empty() || VL.size() == getNumLanes()) && "Expected same number of lanes"
) ? static_cast<void> (0) : __assert_fail ("(empty() || VL.size() == getNumLanes()) && \"Expected same number of lanes\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1124, __PRETTY_FUNCTION__))
;
1125 assert(isa<Instruction>(VL[0]) && "Expected instruction")((isa<Instruction>(VL[0]) && "Expected instruction"
) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(VL[0]) && \"Expected instruction\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1125, __PRETTY_FUNCTION__))
;
1126 unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands();
1127 OpsVec.resize(NumOperands);
1128 unsigned NumLanes = VL.size();
1129 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1130 OpsVec[OpIdx].resize(NumLanes);
1131 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
1132 assert(isa<Instruction>(VL[Lane]) && "Expected instruction")((isa<Instruction>(VL[Lane]) && "Expected instruction"
) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(VL[Lane]) && \"Expected instruction\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1132, __PRETTY_FUNCTION__))
;
1133 // Our tree has just 3 nodes: the root and two operands.
1134 // It is therefore trivial to get the APO. We only need to check the
1135 // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or
1136 // RHS operand. The LHS operand of both add and sub is never attached
1137 // to an inversese operation in the linearized form, therefore its APO
1138 // is false. The RHS is true only if VL[Lane] is an inverse operation.
1139
1140 // Since operand reordering is performed on groups of commutative
1141 // operations or alternating sequences (e.g., +, -), we can safely
1142 // tell the inverse operations by checking commutativity.
1143 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane]));
1144 bool APO = (OpIdx == 0) ? false : IsInverseOperation;
1145 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx),
1146 APO, false};
1147 }
1148 }
1149 }
1150
1151 /// \returns the number of operands.
1152 unsigned getNumOperands() const { return OpsVec.size(); }
1153
1154 /// \returns the number of lanes.
1155 unsigned getNumLanes() const { return OpsVec[0].size(); }
1156
1157 /// \returns the operand value at \p OpIdx and \p Lane.
1158 Value *getValue(unsigned OpIdx, unsigned Lane) const {
1159 return getData(OpIdx, Lane).V;
1160 }
1161
1162 /// \returns true if the data structure is empty.
1163 bool empty() const { return OpsVec.empty(); }
1164
1165 /// Clears the data.
1166 void clear() { OpsVec.clear(); }
1167
1168 /// \Returns true if there are enough operands identical to \p Op to fill
1169 /// the whole vector.
1170 /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow.
1171 bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
1172 bool OpAPO = getData(OpIdx, Lane).APO;
1173 for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
1174 if (Ln == Lane)
1175 continue;
1176 // This is set to true if we found a candidate for broadcast at Lane.
1177 bool FoundCandidate = false;
1178 for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) {
1179 OperandData &Data = getData(OpI, Ln);
1180 if (Data.APO != OpAPO || Data.IsUsed)
1181 continue;
1182 if (Data.V == Op) {
1183 FoundCandidate = true;
1184 Data.IsUsed = true;
1185 break;
1186 }
1187 }
1188 if (!FoundCandidate)
1189 return false;
1190 }
1191 return true;
1192 }
1193
1194 public:
1195 /// Initialize with all the operands of the instruction vector \p RootVL.
1196 VLOperands(ArrayRef<Value *> RootVL, const DataLayout &DL,
1197 ScalarEvolution &SE, const BoUpSLP &R)
1198 : DL(DL), SE(SE), R(R) {
1199 // Append all the operands of RootVL.
1200 appendOperandsOfVL(RootVL);
1201 }
1202
1203 /// \Returns a value vector with the operands across all lanes for the
1204 /// opearnd at \p OpIdx.
1205 ValueList getVL(unsigned OpIdx) const {
1206 ValueList OpVL(OpsVec[OpIdx].size());
1207 assert(OpsVec[OpIdx].size() == getNumLanes() &&((OpsVec[OpIdx].size() == getNumLanes() && "Expected same num of lanes across all operands"
) ? static_cast<void> (0) : __assert_fail ("OpsVec[OpIdx].size() == getNumLanes() && \"Expected same num of lanes across all operands\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1208, __PRETTY_FUNCTION__))
1208 "Expected same num of lanes across all operands")((OpsVec[OpIdx].size() == getNumLanes() && "Expected same num of lanes across all operands"
) ? static_cast<void> (0) : __assert_fail ("OpsVec[OpIdx].size() == getNumLanes() && \"Expected same num of lanes across all operands\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1208, __PRETTY_FUNCTION__))
;
1209 for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane)
1210 OpVL[Lane] = OpsVec[OpIdx][Lane].V;
1211 return OpVL;
1212 }
1213
1214 // Performs operand reordering for 2 or more operands.
1215 // The original operands are in OrigOps[OpIdx][Lane].
1216 // The reordered operands are returned in 'SortedOps[OpIdx][Lane]'.
1217 void reorder() {
1218 unsigned NumOperands = getNumOperands();
1219 unsigned NumLanes = getNumLanes();
1220 // Each operand has its own mode. We are using this mode to help us select
1221 // the instructions for each lane, so that they match best with the ones
1222 // we have selected so far.
1223 SmallVector<ReorderingMode, 2> ReorderingModes(NumOperands);
1224
1225 // This is a greedy single-pass algorithm. We are going over each lane
1226 // once and deciding on the best order right away with no back-tracking.
1227 // However, in order to increase its effectiveness, we start with the lane
1228 // that has operands that can move the least. For example, given the
1229 // following lanes:
1230 // Lane 0 : A[0] = B[0] + C[0] // Visited 3rd
1231 // Lane 1 : A[1] = C[1] - B[1] // Visited 1st
1232 // Lane 2 : A[2] = B[2] + C[2] // Visited 2nd
1233 // Lane 3 : A[3] = C[3] - B[3] // Visited 4th
1234 // we will start at Lane 1, since the operands of the subtraction cannot
1235 // be reordered. Then we will visit the rest of the lanes in a circular
1236 // fashion. That is, Lanes 2, then Lane 0, and finally Lane 3.
1237
1238 // Find the first lane that we will start our search from.
1239 unsigned FirstLane = getBestLaneToStartReordering();
1240
1241 // Initialize the modes.
1242 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1243 Value *OpLane0 = getValue(OpIdx, FirstLane);
1244 // Keep track if we have instructions with all the same opcode on one
1245 // side.
1246 if (isa<LoadInst>(OpLane0))
1247 ReorderingModes[OpIdx] = ReorderingMode::Load;
1248 else if (isa<Instruction>(OpLane0)) {
1249 // Check if OpLane0 should be broadcast.
1250 if (shouldBroadcast(OpLane0, OpIdx, FirstLane))
1251 ReorderingModes[OpIdx] = ReorderingMode::Splat;
1252 else
1253 ReorderingModes[OpIdx] = ReorderingMode::Opcode;
1254 }
1255 else if (isa<Constant>(OpLane0))
1256 ReorderingModes[OpIdx] = ReorderingMode::Constant;
1257 else if (isa<Argument>(OpLane0))
1258 // Our best hope is a Splat. It may save some cost in some cases.
1259 ReorderingModes[OpIdx] = ReorderingMode::Splat;
1260 else
1261 // NOTE: This should be unreachable.
1262 ReorderingModes[OpIdx] = ReorderingMode::Failed;
1263 }
1264
1265 // If the initial strategy fails for any of the operand indexes, then we
1266 // perform reordering again in a second pass. This helps avoid assigning
1267 // high priority to the failed strategy, and should improve reordering for
1268 // the non-failed operand indexes.
1269 for (int Pass = 0; Pass != 2; ++Pass) {
1270 // Skip the second pass if the first pass did not fail.
1271 bool StrategyFailed = false;
1272 // Mark all operand data as free to use.
1273 clearUsed();
1274 // We keep the original operand order for the FirstLane, so reorder the
1275 // rest of the lanes. We are visiting the nodes in a circular fashion,
1276 // using FirstLane as the center point and increasing the radius
1277 // distance.
1278 for (unsigned Distance = 1; Distance != NumLanes; ++Distance) {
1279 // Visit the lane on the right and then the lane on the left.
1280 for (int Direction : {+1, -1}) {
1281 int Lane = FirstLane + Direction * Distance;
1282 if (Lane < 0 || Lane >= (int)NumLanes)
1283 continue;
1284 int LastLane = Lane - Direction;
1285 assert(LastLane >= 0 && LastLane < (int)NumLanes &&((LastLane >= 0 && LastLane < (int)NumLanes &&
"Out of bounds") ? static_cast<void> (0) : __assert_fail
("LastLane >= 0 && LastLane < (int)NumLanes && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1286, __PRETTY_FUNCTION__))
1286 "Out of bounds")((LastLane >= 0 && LastLane < (int)NumLanes &&
"Out of bounds") ? static_cast<void> (0) : __assert_fail
("LastLane >= 0 && LastLane < (int)NumLanes && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1286, __PRETTY_FUNCTION__))
;
1287 // Look for a good match for each operand.
1288 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1289 // Search for the operand that matches SortedOps[OpIdx][Lane-1].
1290 Optional<unsigned> BestIdx =
1291 getBestOperand(OpIdx, Lane, LastLane, ReorderingModes);
1292 // By not selecting a value, we allow the operands that follow to
1293 // select a better matching value. We will get a non-null value in
1294 // the next run of getBestOperand().
1295 if (BestIdx) {
1296 // Swap the current operand with the one returned by
1297 // getBestOperand().
1298 swap(OpIdx, BestIdx.getValue(), Lane);
1299 } else {
1300 // We failed to find a best operand, set mode to 'Failed'.
1301 ReorderingModes[OpIdx] = ReorderingMode::Failed;
1302 // Enable the second pass.
1303 StrategyFailed = true;
1304 }
1305 }
1306 }
1307 }
1308 // Skip second pass if the strategy did not fail.
1309 if (!StrategyFailed)
1310 break;
1311 }
1312 }
1313
1314#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1315 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) static StringRef getModeStr(ReorderingMode RMode) {
1316 switch (RMode) {
1317 case ReorderingMode::Load:
1318 return "Load";
1319 case ReorderingMode::Opcode:
1320 return "Opcode";
1321 case ReorderingMode::Constant:
1322 return "Constant";
1323 case ReorderingMode::Splat:
1324 return "Splat";
1325 case ReorderingMode::Failed:
1326 return "Failed";
1327 }
1328 llvm_unreachable("Unimplemented Reordering Type")::llvm::llvm_unreachable_internal("Unimplemented Reordering Type"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1328)
;
1329 }
1330
1331 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) static raw_ostream &printMode(ReorderingMode RMode,
1332 raw_ostream &OS) {
1333 return OS << getModeStr(RMode);
1334 }
1335
1336 /// Debug print.
1337 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) static void dumpMode(ReorderingMode RMode) {
1338 printMode(RMode, dbgs());
1339 }
1340
1341 friend raw_ostream &operator<<(raw_ostream &OS, ReorderingMode RMode) {
1342 return printMode(RMode, OS);
1343 }
1344
1345 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) raw_ostream &print(raw_ostream &OS) const {
1346 const unsigned Indent = 2;
1347 unsigned Cnt = 0;
1348 for (const OperandDataVec &OpDataVec : OpsVec) {
1349 OS << "Operand " << Cnt++ << "\n";
1350 for (const OperandData &OpData : OpDataVec) {
1351 OS.indent(Indent) << "{";
1352 if (Value *V = OpData.V)
1353 OS << *V;
1354 else
1355 OS << "null";
1356 OS << ", APO:" << OpData.APO << "}\n";
1357 }
1358 OS << "\n";
1359 }
1360 return OS;
1361 }
1362
1363 /// Debug print.
1364 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dump() const { print(dbgs()); }
1365#endif
1366 };
1367
1368 /// Checks if the instruction is marked for deletion.
1369 bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); }
1370
1371 /// Marks values operands for later deletion by replacing them with Undefs.
1372 void eraseInstructions(ArrayRef<Value *> AV);
1373
1374 ~BoUpSLP();
1375
1376private:
1377 /// Checks if all users of \p I are the part of the vectorization tree.
1378 bool areAllUsersVectorized(Instruction *I) const;
1379
1380 /// \returns the cost of the vectorizable entry.
1381 int getEntryCost(TreeEntry *E);
1382
1383 /// This is the recursive part of buildTree.
1384 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth,
1385 const EdgeInfo &EI);
1386
1387 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
1388 /// be vectorized to use the original vector (or aggregate "bitcast" to a
1389 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
1390 /// returns false, setting \p CurrentOrder to either an empty vector or a
1391 /// non-identity permutation that allows to reuse extract instructions.
1392 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1393 SmallVectorImpl<unsigned> &CurrentOrder) const;
1394
1395 /// Vectorize a single entry in the tree.
1396 Value *vectorizeTree(TreeEntry *E);
1397
1398 /// Vectorize a single entry in the tree, starting in \p VL.
1399 Value *vectorizeTree(ArrayRef<Value *> VL);
1400
1401 /// \returns the scalarization cost for this type. Scalarization in this
1402 /// context means the creation of vectors from a group of scalars.
1403 int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices) const;
1404
1405 /// \returns the scalarization cost for this list of values. Assuming that
1406 /// this subtree gets vectorized, we may need to extract the values from the
1407 /// roots. This method calculates the cost of extracting the values.
1408 int getGatherCost(ArrayRef<Value *> VL) const;
1409
1410 /// Set the Builder insert point to one after the last instruction in
1411 /// the bundle
1412 void setInsertPointAfterBundle(TreeEntry *E);
1413
1414 /// \returns a vector from a collection of scalars in \p VL.
1415 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
1416
1417 /// \returns whether the VectorizableTree is fully vectorizable and will
1418 /// be beneficial even the tree height is tiny.
1419 bool isFullyVectorizableTinyTree() const;
1420
1421 /// Reorder commutative or alt operands to get better probability of
1422 /// generating vectorized code.
1423 static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
1424 SmallVectorImpl<Value *> &Left,
1425 SmallVectorImpl<Value *> &Right,
1426 const DataLayout &DL,
1427 ScalarEvolution &SE,
1428 const BoUpSLP &R);
1429 struct TreeEntry {
1430 using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
1431 TreeEntry(VecTreeTy &Container) : Container(Container) {}
1432
1433 /// \returns true if the scalars in VL are equal to this entry.
1434 bool isSame(ArrayRef<Value *> VL) const {
1435 if (VL.size() == Scalars.size())
1436 return std::equal(VL.begin(), VL.end(), Scalars.begin());
1437 return VL.size() == ReuseShuffleIndices.size() &&
1438 std::equal(
1439 VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
1440 [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
1441 }
1442
1443 /// A vector of scalars.
1444 ValueList Scalars;
1445
1446 /// The Scalars are vectorized into this value. It is initialized to Null.
1447 Value *VectorizedValue = nullptr;
1448
1449 /// Do we need to gather this sequence ?
1450 enum EntryState { Vectorize, NeedToGather };
1451 EntryState State;
1452
1453 /// Does this sequence require some shuffling?
1454 SmallVector<unsigned, 4> ReuseShuffleIndices;
1455
1456 /// Does this entry require reordering?
1457 ArrayRef<unsigned> ReorderIndices;
1458
1459 /// Points back to the VectorizableTree.
1460 ///
1461 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
1462 /// to be a pointer and needs to be able to initialize the child iterator.
1463 /// Thus we need a reference back to the container to translate the indices
1464 /// to entries.
1465 VecTreeTy &Container;
1466
1467 /// The TreeEntry index containing the user of this entry. We can actually
1468 /// have multiple users so the data structure is not truly a tree.
1469 SmallVector<EdgeInfo, 1> UserTreeIndices;
1470
1471 /// The index of this treeEntry in VectorizableTree.
1472 int Idx = -1;
1473
1474 private:
1475 /// The operands of each instruction in each lane Operands[op_index][lane].
1476 /// Note: This helps avoid the replication of the code that performs the
1477 /// reordering of operands during buildTree_rec() and vectorizeTree().
1478 SmallVector<ValueList, 2> Operands;
1479
1480 /// The main/alternate instruction.
1481 Instruction *MainOp = nullptr;
1482 Instruction *AltOp = nullptr;
1483
1484 public:
1485 /// Set this bundle's \p OpIdx'th operand to \p OpVL.
1486 void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
1487 if (Operands.size() < OpIdx + 1)
1488 Operands.resize(OpIdx + 1);
1489 assert(Operands[OpIdx].size() == 0 && "Already resized?")((Operands[OpIdx].size() == 0 && "Already resized?") ?
static_cast<void> (0) : __assert_fail ("Operands[OpIdx].size() == 0 && \"Already resized?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1489, __PRETTY_FUNCTION__))
;
1490 Operands[OpIdx].resize(Scalars.size());
1491 for (unsigned Lane = 0, E = Scalars.size(); Lane != E; ++Lane)
1492 Operands[OpIdx][Lane] = OpVL[Lane];
1493 }
1494
1495 /// Set the operands of this bundle in their original order.
1496 void setOperandsInOrder() {
1497 assert(Operands.empty() && "Already initialized?")((Operands.empty() && "Already initialized?") ? static_cast
<void> (0) : __assert_fail ("Operands.empty() && \"Already initialized?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1497, __PRETTY_FUNCTION__))
;
1498 auto *I0 = cast<Instruction>(Scalars[0]);
1499 Operands.resize(I0->getNumOperands());
1500 unsigned NumLanes = Scalars.size();
1501 for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands();
1502 OpIdx != NumOperands; ++OpIdx) {
1503 Operands[OpIdx].resize(NumLanes);
1504 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
1505 auto *I = cast<Instruction>(Scalars[Lane]);
1506 assert(I->getNumOperands() == NumOperands &&((I->getNumOperands() == NumOperands && "Expected same number of operands"
) ? static_cast<void> (0) : __assert_fail ("I->getNumOperands() == NumOperands && \"Expected same number of operands\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1507, __PRETTY_FUNCTION__))
1507 "Expected same number of operands")((I->getNumOperands() == NumOperands && "Expected same number of operands"
) ? static_cast<void> (0) : __assert_fail ("I->getNumOperands() == NumOperands && \"Expected same number of operands\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1507, __PRETTY_FUNCTION__))
;
1508 Operands[OpIdx][Lane] = I->getOperand(OpIdx);
1509 }
1510 }
1511 }
1512
1513 /// \returns the \p OpIdx operand of this TreeEntry.
1514 ValueList &getOperand(unsigned OpIdx) {
1515 assert(OpIdx < Operands.size() && "Off bounds")((OpIdx < Operands.size() && "Off bounds") ? static_cast
<void> (0) : __assert_fail ("OpIdx < Operands.size() && \"Off bounds\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1515, __PRETTY_FUNCTION__))
;
1516 return Operands[OpIdx];
1517 }
1518
1519 /// \returns the number of operands.
1520 unsigned getNumOperands() const { return Operands.size(); }
1521
1522 /// \return the single \p OpIdx operand.
1523 Value *getSingleOperand(unsigned OpIdx) const {
1524 assert(OpIdx < Operands.size() && "Off bounds")((OpIdx < Operands.size() && "Off bounds") ? static_cast
<void> (0) : __assert_fail ("OpIdx < Operands.size() && \"Off bounds\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1524, __PRETTY_FUNCTION__))
;
1525 assert(!Operands[OpIdx].empty() && "No operand available")((!Operands[OpIdx].empty() && "No operand available")
? static_cast<void> (0) : __assert_fail ("!Operands[OpIdx].empty() && \"No operand available\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1525, __PRETTY_FUNCTION__))
;
1526 return Operands[OpIdx][0];
1527 }
1528
1529 /// Some of the instructions in the list have alternate opcodes.
1530 bool isAltShuffle() const {
1531 return getOpcode() != getAltOpcode();
1532 }
1533
1534 bool isOpcodeOrAlt(Instruction *I) const {
1535 unsigned CheckedOpcode = I->getOpcode();
1536 return (getOpcode() == CheckedOpcode ||
1537 getAltOpcode() == CheckedOpcode);
1538 }
1539
1540 /// Chooses the correct key for scheduling data. If \p Op has the same (or
1541 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is
1542 /// \p OpValue.
1543 Value *isOneOf(Value *Op) const {
1544 auto *I = dyn_cast<Instruction>(Op);
1545 if (I && isOpcodeOrAlt(I))
1546 return Op;
1547 return MainOp;
1548 }
1549
1550 void setOperations(const InstructionsState &S) {
1551 MainOp = S.MainOp;
1552 AltOp = S.AltOp;
1553 }
1554
1555 Instruction *getMainOp() const {
1556 return MainOp;
1557 }
1558
1559 Instruction *getAltOp() const {
1560 return AltOp;
1561 }
1562
1563 /// The main/alternate opcodes for the list of instructions.
1564 unsigned getOpcode() const {
1565 return MainOp ? MainOp->getOpcode() : 0;
1566 }
1567
1568 unsigned getAltOpcode() const {
1569 return AltOp ? AltOp->getOpcode() : 0;
1570 }
1571
1572 /// Update operations state of this entry if reorder occurred.
1573 bool updateStateIfReorder() {
1574 if (ReorderIndices.empty())
1575 return false;
1576 InstructionsState S = getSameOpcode(Scalars, ReorderIndices.front());
1577 setOperations(S);
1578 return true;
1579 }
1580
1581#ifndef NDEBUG
1582 /// Debug printer.
1583 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dump() const {
1584 dbgs() << Idx << ".\n";
1585 for (unsigned OpI = 0, OpE = Operands.size(); OpI != OpE; ++OpI) {
1586 dbgs() << "Operand " << OpI << ":\n";
1587 for (const Value *V : Operands[OpI])
1588 dbgs().indent(2) << *V << "\n";
1589 }
1590 dbgs() << "Scalars: \n";
1591 for (Value *V : Scalars)
1592 dbgs().indent(2) << *V << "\n";
1593 dbgs() << "State: ";
1594 switch (State) {
1595 case Vectorize:
1596 dbgs() << "Vectorize\n";
1597 break;
1598 case NeedToGather:
1599 dbgs() << "NeedToGather\n";
1600 break;
1601 }
1602 dbgs() << "MainOp: ";
1603 if (MainOp)
1604 dbgs() << *MainOp << "\n";
1605 else
1606 dbgs() << "NULL\n";
1607 dbgs() << "AltOp: ";
1608 if (AltOp)
1609 dbgs() << *AltOp << "\n";
1610 else
1611 dbgs() << "NULL\n";
1612 dbgs() << "VectorizedValue: ";
1613 if (VectorizedValue)
1614 dbgs() << *VectorizedValue << "\n";
1615 else
1616 dbgs() << "NULL\n";
1617 dbgs() << "ReuseShuffleIndices: ";
1618 if (ReuseShuffleIndices.empty())
1619 dbgs() << "Emtpy";
1620 else
1621 for (unsigned ReuseIdx : ReuseShuffleIndices)
1622 dbgs() << ReuseIdx << ", ";
1623 dbgs() << "\n";
1624 dbgs() << "ReorderIndices: ";
1625 for (unsigned ReorderIdx : ReorderIndices)
1626 dbgs() << ReorderIdx << ", ";
1627 dbgs() << "\n";
1628 dbgs() << "UserTreeIndices: ";
1629 for (const auto &EInfo : UserTreeIndices)
1630 dbgs() << EInfo << ", ";
1631 dbgs() << "\n";
1632 }
1633#endif
1634 };
1635
1636 /// Create a new VectorizableTree entry.
1637 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle,
1638 const InstructionsState &S,
1639 const EdgeInfo &UserTreeIdx,
1640 ArrayRef<unsigned> ReuseShuffleIndices = None,
1641 ArrayRef<unsigned> ReorderIndices = None) {
1642 bool Vectorized = (bool)Bundle;
1643 VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree));
1644 TreeEntry *Last = VectorizableTree.back().get();
1645 Last->Idx = VectorizableTree.size() - 1;
1646 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
1647 Last->State = Vectorized ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
1648 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
1649 ReuseShuffleIndices.end());
1650 Last->ReorderIndices = ReorderIndices;
1651 Last->setOperations(S);
1652 if (Vectorized) {
1653 for (int i = 0, e = VL.size(); i != e; ++i) {
1654 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!")((!getTreeEntry(VL[i]) && "Scalar already in tree!") ?
static_cast<void> (0) : __assert_fail ("!getTreeEntry(VL[i]) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1654, __PRETTY_FUNCTION__))
;
1655 ScalarToTreeEntry[VL[i]] = Last;
1656 }
1657 // Update the scheduler bundle to point to this TreeEntry.
1658 unsigned Lane = 0;
1659 for (ScheduleData *BundleMember = Bundle.getValue(); BundleMember;
1660 BundleMember = BundleMember->NextInBundle) {
1661 BundleMember->TE = Last;
1662 BundleMember->Lane = Lane;
1663 ++Lane;
1664 }
1665 assert((!Bundle.getValue() || Lane == VL.size()) &&(((!Bundle.getValue() || Lane == VL.size()) && "Bundle and VL out of sync"
) ? static_cast<void> (0) : __assert_fail ("(!Bundle.getValue() || Lane == VL.size()) && \"Bundle and VL out of sync\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1666, __PRETTY_FUNCTION__))
1666 "Bundle and VL out of sync")(((!Bundle.getValue() || Lane == VL.size()) && "Bundle and VL out of sync"
) ? static_cast<void> (0) : __assert_fail ("(!Bundle.getValue() || Lane == VL.size()) && \"Bundle and VL out of sync\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1666, __PRETTY_FUNCTION__))
;
1667 } else {
1668 MustGather.insert(VL.begin(), VL.end());
1669 }
1670
1671 if (UserTreeIdx.UserTE)
1672 Last->UserTreeIndices.push_back(UserTreeIdx);
1673
1674 return Last;
1675 }
1676
1677 /// -- Vectorization State --
1678 /// Holds all of the tree entries.
1679 TreeEntry::VecTreeTy VectorizableTree;
1680
1681#ifndef NDEBUG
1682 /// Debug printer.
1683 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dumpVectorizableTree() const {
1684 for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) {
1685 VectorizableTree[Id]->dump();
1686 dbgs() << "\n";
1687 }
1688 }
1689#endif
1690
1691 TreeEntry *getTreeEntry(Value *V) {
1692 auto I = ScalarToTreeEntry.find(V);
1693 if (I != ScalarToTreeEntry.end())
1694 return I->second;
1695 return nullptr;
1696 }
1697
1698 const TreeEntry *getTreeEntry(Value *V) const {
1699 auto I = ScalarToTreeEntry.find(V);
1700 if (I != ScalarToTreeEntry.end())
1701 return I->second;
1702 return nullptr;
1703 }
1704
1705 /// Maps a specific scalar to its tree entry.
1706 SmallDenseMap<Value*, TreeEntry *> ScalarToTreeEntry;
1707
1708 /// A list of scalars that we found that we need to keep as scalars.
1709 ValueSet MustGather;
1710
1711 /// This POD struct describes one external user in the vectorized tree.
1712 struct ExternalUser {
1713 ExternalUser(Value *S, llvm::User *U, int L)
1714 : Scalar(S), User(U), Lane(L) {}
1715
1716 // Which scalar in our function.
1717 Value *Scalar;
1718
1719 // Which user that uses the scalar.
1720 llvm::User *User;
1721
1722 // Which lane does the scalar belong to.
1723 int Lane;
1724 };
1725 using UserList = SmallVector<ExternalUser, 16>;
1726
1727 /// Checks if two instructions may access the same memory.
1728 ///
1729 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
1730 /// is invariant in the calling loop.
1731 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
1732 Instruction *Inst2) {
1733 // First check if the result is already in the cache.
1734 AliasCacheKey key = std::make_pair(Inst1, Inst2);
1735 Optional<bool> &result = AliasCache[key];
1736 if (result.hasValue()) {
1737 return result.getValue();
1738 }
1739 MemoryLocation Loc2 = getLocation(Inst2, AA);
1740 bool aliased = true;
1741 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
1742 // Do the alias check.
1743 aliased = AA->alias(Loc1, Loc2);
1744 }
1745 // Store the result in the cache.
1746 result = aliased;
1747 return aliased;
1748 }
1749
1750 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
1751
1752 /// Cache for alias results.
1753 /// TODO: consider moving this to the AliasAnalysis itself.
1754 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
1755
1756 /// Removes an instruction from its block and eventually deletes it.
1757 /// It's like Instruction::eraseFromParent() except that the actual deletion
1758 /// is delayed until BoUpSLP is destructed.
1759 /// This is required to ensure that there are no incorrect collisions in the
1760 /// AliasCache, which can happen if a new instruction is allocated at the
1761 /// same address as a previously deleted instruction.
1762 void eraseInstruction(Instruction *I, bool ReplaceOpsWithUndef = false) {
1763 auto It = DeletedInstructions.try_emplace(I, ReplaceOpsWithUndef).first;
1764 It->getSecond() = It->getSecond() && ReplaceOpsWithUndef;
1765 }
1766
1767 /// Temporary store for deleted instructions. Instructions will be deleted
1768 /// eventually when the BoUpSLP is destructed.
1769 DenseMap<Instruction *, bool> DeletedInstructions;
1770
1771 /// A list of values that need to extracted out of the tree.
1772 /// This list holds pairs of (Internal Scalar : External User). External User
1773 /// can be nullptr, it means that this Internal Scalar will be used later,
1774 /// after vectorization.
1775 UserList ExternalUses;
1776
1777 /// Values used only by @llvm.assume calls.
1778 SmallPtrSet<const Value *, 32> EphValues;
1779
1780 /// Holds all of the instructions that we gathered.
1781 SetVector<Instruction *> GatherSeq;
1782
1783 /// A list of blocks that we are going to CSE.
1784 SetVector<BasicBlock *> CSEBlocks;
1785
1786 /// Contains all scheduling relevant data for an instruction.
1787 /// A ScheduleData either represents a single instruction or a member of an
1788 /// instruction bundle (= a group of instructions which is combined into a
1789 /// vector instruction).
1790 struct ScheduleData {
1791 // The initial value for the dependency counters. It means that the
1792 // dependencies are not calculated yet.
1793 enum { InvalidDeps = -1 };
1794
1795 ScheduleData() = default;
1796
1797 void init(int BlockSchedulingRegionID, Value *OpVal) {
1798 FirstInBundle = this;
1799 NextInBundle = nullptr;
1800 NextLoadStore = nullptr;
1801 IsScheduled = false;
1802 SchedulingRegionID = BlockSchedulingRegionID;
1803 UnscheduledDepsInBundle = UnscheduledDeps;
1804 clearDependencies();
1805 OpValue = OpVal;
1806 TE = nullptr;
1807 Lane = -1;
1808 }
1809
1810 /// Returns true if the dependency information has been calculated.
1811 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
1812
1813 /// Returns true for single instructions and for bundle representatives
1814 /// (= the head of a bundle).
1815 bool isSchedulingEntity() const { return FirstInBundle == this; }
1816
1817 /// Returns true if it represents an instruction bundle and not only a
1818 /// single instruction.
1819 bool isPartOfBundle() const {
1820 return NextInBundle != nullptr || FirstInBundle != this;
1821 }
1822
1823 /// Returns true if it is ready for scheduling, i.e. it has no more
1824 /// unscheduled depending instructions/bundles.
1825 bool isReady() const {
1826 assert(isSchedulingEntity() &&((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1827, __PRETTY_FUNCTION__))
1827 "can't consider non-scheduling entity for ready list")((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1827, __PRETTY_FUNCTION__))
;
1828 return UnscheduledDepsInBundle == 0 && !IsScheduled;
1829 }
1830
1831 /// Modifies the number of unscheduled dependencies, also updating it for
1832 /// the whole bundle.
1833 int incrementUnscheduledDeps(int Incr) {
1834 UnscheduledDeps += Incr;
1835 return FirstInBundle->UnscheduledDepsInBundle += Incr;
1836 }
1837
1838 /// Sets the number of unscheduled dependencies to the number of
1839 /// dependencies.
1840 void resetUnscheduledDeps() {
1841 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
1842 }
1843
1844 /// Clears all dependency information.
1845 void clearDependencies() {
1846 Dependencies = InvalidDeps;
1847 resetUnscheduledDeps();
1848 MemoryDependencies.clear();
1849 }
1850
1851 void dump(raw_ostream &os) const {
1852 if (!isSchedulingEntity()) {
1853 os << "/ " << *Inst;
1854 } else if (NextInBundle) {
1855 os << '[' << *Inst;
1856 ScheduleData *SD = NextInBundle;
1857 while (SD) {
1858 os << ';' << *SD->Inst;
1859 SD = SD->NextInBundle;
1860 }
1861 os << ']';
1862 } else {
1863 os << *Inst;
1864 }
1865 }
1866
1867 Instruction *Inst = nullptr;
1868
1869 /// Points to the head in an instruction bundle (and always to this for
1870 /// single instructions).
1871 ScheduleData *FirstInBundle = nullptr;
1872
1873 /// Single linked list of all instructions in a bundle. Null if it is a
1874 /// single instruction.
1875 ScheduleData *NextInBundle = nullptr;
1876
1877 /// Single linked list of all memory instructions (e.g. load, store, call)
1878 /// in the block - until the end of the scheduling region.
1879 ScheduleData *NextLoadStore = nullptr;
1880
1881 /// The dependent memory instructions.
1882 /// This list is derived on demand in calculateDependencies().
1883 SmallVector<ScheduleData *, 4> MemoryDependencies;
1884
1885 /// This ScheduleData is in the current scheduling region if this matches
1886 /// the current SchedulingRegionID of BlockScheduling.
1887 int SchedulingRegionID = 0;
1888
1889 /// Used for getting a "good" final ordering of instructions.
1890 int SchedulingPriority = 0;
1891
1892 /// The number of dependencies. Constitutes of the number of users of the
1893 /// instruction plus the number of dependent memory instructions (if any).
1894 /// This value is calculated on demand.
1895 /// If InvalidDeps, the number of dependencies is not calculated yet.
1896 int Dependencies = InvalidDeps;
1897
1898 /// The number of dependencies minus the number of dependencies of scheduled
1899 /// instructions. As soon as this is zero, the instruction/bundle gets ready
1900 /// for scheduling.
1901 /// Note that this is negative as long as Dependencies is not calculated.
1902 int UnscheduledDeps = InvalidDeps;
1903
1904 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
1905 /// single instructions.
1906 int UnscheduledDepsInBundle = InvalidDeps;
1907
1908 /// True if this instruction is scheduled (or considered as scheduled in the
1909 /// dry-run).
1910 bool IsScheduled = false;
1911
1912 /// Opcode of the current instruction in the schedule data.
1913 Value *OpValue = nullptr;
1914
1915 /// The TreeEntry that this instruction corresponds to.
1916 TreeEntry *TE = nullptr;
1917
1918 /// The lane of this node in the TreeEntry.
1919 int Lane = -1;
1920 };
1921
1922#ifndef NDEBUG
1923 friend inline raw_ostream &operator<<(raw_ostream &os,
1924 const BoUpSLP::ScheduleData &SD) {
1925 SD.dump(os);
1926 return os;
1927 }
1928#endif
1929
1930 friend struct GraphTraits<BoUpSLP *>;
1931 friend struct DOTGraphTraits<BoUpSLP *>;
1932
1933 /// Contains all scheduling data for a basic block.
1934 struct BlockScheduling {
1935 BlockScheduling(BasicBlock *BB)
1936 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
1937
1938 void clear() {
1939 ReadyInsts.clear();
1940 ScheduleStart = nullptr;
1941 ScheduleEnd = nullptr;
1942 FirstLoadStoreInRegion = nullptr;
1943 LastLoadStoreInRegion = nullptr;
1944
1945 // Reduce the maximum schedule region size by the size of the
1946 // previous scheduling run.
1947 ScheduleRegionSizeLimit -= ScheduleRegionSize;
1948 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1949 ScheduleRegionSizeLimit = MinScheduleRegionSize;
1950 ScheduleRegionSize = 0;
1951
1952 // Make a new scheduling region, i.e. all existing ScheduleData is not
1953 // in the new region yet.
1954 ++SchedulingRegionID;
1955 }
1956
1957 ScheduleData *getScheduleData(Value *V) {
1958 ScheduleData *SD = ScheduleDataMap[V];
1959 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1960 return SD;
1961 return nullptr;
1962 }
1963
1964 ScheduleData *getScheduleData(Value *V, Value *Key) {
1965 if (V == Key)
1966 return getScheduleData(V);
1967 auto I = ExtraScheduleDataMap.find(V);
1968 if (I != ExtraScheduleDataMap.end()) {
1969 ScheduleData *SD = I->second[Key];
1970 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1971 return SD;
1972 }
1973 return nullptr;
1974 }
1975
1976 bool isInSchedulingRegion(ScheduleData *SD) const {
1977 return SD->SchedulingRegionID == SchedulingRegionID;
1978 }
1979
1980 /// Marks an instruction as scheduled and puts all dependent ready
1981 /// instructions into the ready-list.
1982 template <typename ReadyListType>
1983 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1984 SD->IsScheduled = true;
16
Access to field 'IsScheduled' results in a dereference of a null pointer (loaded from variable 'SD')
1985 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
1986
1987 ScheduleData *BundleMember = SD;
1988 while (BundleMember) {
1989 if (BundleMember->Inst != BundleMember->OpValue) {
1990 BundleMember = BundleMember->NextInBundle;
1991 continue;
1992 }
1993 // Handle the def-use chain dependencies.
1994
1995 // Decrement the unscheduled counter and insert to ready list if ready.
1996 auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
1997 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1998 if (OpDef && OpDef->hasValidDependencies() &&
1999 OpDef->incrementUnscheduledDeps(-1) == 0) {
2000 // There are no more unscheduled dependencies after
2001 // decrementing, so we can put the dependent instruction
2002 // into the ready list.
2003 ScheduleData *DepBundle = OpDef->FirstInBundle;
2004 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2005, __PRETTY_FUNCTION__))
2005 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2005, __PRETTY_FUNCTION__))
;
2006 ReadyList.insert(DepBundle);
2007 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
2008 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
2009 }
2010 });
2011 };
2012
2013 // If BundleMember is a vector bundle, its operands may have been
2014 // reordered duiring buildTree(). We therefore need to get its operands
2015 // through the TreeEntry.
2016 if (TreeEntry *TE = BundleMember->TE) {
2017 int Lane = BundleMember->Lane;
2018 assert(Lane >= 0 && "Lane not set")((Lane >= 0 && "Lane not set") ? static_cast<void
> (0) : __assert_fail ("Lane >= 0 && \"Lane not set\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2018, __PRETTY_FUNCTION__))
;
2019
2020 // Since vectorization tree is being built recursively this assertion
2021 // ensures that the tree entry has all operands set before reaching
2022 // this code. Couple of exceptions known at the moment are extracts
2023 // where their second (immediate) operand is not added. Since
2024 // immediates do not affect scheduler behavior this is considered
2025 // okay.
2026 auto *In = TE->getMainOp();
2027 assert(In &&((In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? static_cast<
void> (0) : __assert_fail ("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2030, __PRETTY_FUNCTION__))
2028 (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) ||((In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? static_cast<
void> (0) : __assert_fail ("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2030, __PRETTY_FUNCTION__))
2029 In->getNumOperands() == TE->getNumOperands()) &&((In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? static_cast<
void> (0) : __assert_fail ("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2030, __PRETTY_FUNCTION__))
2030 "Missed TreeEntry operands?")((In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? static_cast<
void> (0) : __assert_fail ("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2030, __PRETTY_FUNCTION__))
;
2031 (void)In; // fake use to avoid build failure when assertions disabled
2032
2033 for (unsigned OpIdx = 0, NumOperands = TE->getNumOperands();
2034 OpIdx != NumOperands; ++OpIdx)
2035 if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane]))
2036 DecrUnsched(I);
2037 } else {
2038 // If BundleMember is a stand-alone instruction, no operand reordering
2039 // has taken place, so we directly access its operands.
2040 for (Use &U : BundleMember->Inst->operands())
2041 if (auto *I = dyn_cast<Instruction>(U.get()))
2042 DecrUnsched(I);
2043 }
2044 // Handle the memory dependencies.
2045 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
2046 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
2047 // There are no more unscheduled dependencies after decrementing,
2048 // so we can put the dependent instruction into the ready list.
2049 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
2050 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2051, __PRETTY_FUNCTION__))
2051 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2051, __PRETTY_FUNCTION__))
;
2052 ReadyList.insert(DepBundle);
2053 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
2054 << "SLP: gets ready (mem): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
2055 }
2056 }
2057 BundleMember = BundleMember->NextInBundle;
2058 }
2059 }
2060
2061 void doForAllOpcodes(Value *V,
2062 function_ref<void(ScheduleData *SD)> Action) {
2063 if (ScheduleData *SD = getScheduleData(V))
2064 Action(SD);
2065 auto I = ExtraScheduleDataMap.find(V);
2066 if (I != ExtraScheduleDataMap.end())
2067 for (auto &P : I->second)
2068 if (P.second->SchedulingRegionID == SchedulingRegionID)
2069 Action(P.second);
2070 }
2071
2072 /// Put all instructions into the ReadyList which are ready for scheduling.
2073 template <typename ReadyListType>
2074 void initialFillReadyList(ReadyListType &ReadyList) {
2075 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
2076 doForAllOpcodes(I, [&](ScheduleData *SD) {
2077 if (SD->isSchedulingEntity() && SD->isReady()) {
2078 ReadyList.insert(SD);
2079 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
2080 << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
2081 }
2082 });
2083 }
2084 }
2085
2086 /// Checks if a bundle of instructions can be scheduled, i.e. has no
2087 /// cyclic dependencies. This is only a dry-run, no instructions are
2088 /// actually moved at this stage.
2089 /// \returns the scheduling bundle. The returned Optional value is non-None
2090 /// if \p VL is allowed to be scheduled.
2091 Optional<ScheduleData *>
2092 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
2093 const InstructionsState &S);
2094
2095 /// Un-bundles a group of instructions.
2096 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
2097
2098 /// Allocates schedule data chunk.
2099 ScheduleData *allocateScheduleDataChunks();
2100
2101 /// Extends the scheduling region so that V is inside the region.
2102 /// \returns true if the region size is within the limit.
2103 bool extendSchedulingRegion(Value *V, const InstructionsState &S);
2104
2105 /// Initialize the ScheduleData structures for new instructions in the
2106 /// scheduling region.
2107 void initScheduleData(Instruction *FromI, Instruction *ToI,
2108 ScheduleData *PrevLoadStore,
2109 ScheduleData *NextLoadStore);
2110
2111 /// Updates the dependency information of a bundle and of all instructions/
2112 /// bundles which depend on the original bundle.
2113 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
2114 BoUpSLP *SLP);
2115
2116 /// Sets all instruction in the scheduling region to un-scheduled.
2117 void resetSchedule();
2118
2119 BasicBlock *BB;
2120
2121 /// Simple memory allocation for ScheduleData.
2122 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
2123
2124 /// The size of a ScheduleData array in ScheduleDataChunks.
2125 int ChunkSize;
2126
2127 /// The allocator position in the current chunk, which is the last entry
2128 /// of ScheduleDataChunks.
2129 int ChunkPos;
2130
2131 /// Attaches ScheduleData to Instruction.
2132 /// Note that the mapping survives during all vectorization iterations, i.e.
2133 /// ScheduleData structures are recycled.
2134 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
2135
2136 /// Attaches ScheduleData to Instruction with the leading key.
2137 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
2138 ExtraScheduleDataMap;
2139
2140 struct ReadyList : SmallVector<ScheduleData *, 8> {
2141 void insert(ScheduleData *SD) { push_back(SD); }
2142 };
2143
2144 /// The ready-list for scheduling (only used for the dry-run).
2145 ReadyList ReadyInsts;
2146
2147 /// The first instruction of the scheduling region.
2148 Instruction *ScheduleStart = nullptr;
2149
2150 /// The first instruction _after_ the scheduling region.
2151 Instruction *ScheduleEnd = nullptr;
2152
2153 /// The first memory accessing instruction in the scheduling region
2154 /// (can be null).
2155 ScheduleData *FirstLoadStoreInRegion = nullptr;
2156
2157 /// The last memory accessing instruction in the scheduling region
2158 /// (can be null).
2159 ScheduleData *LastLoadStoreInRegion = nullptr;
2160
2161 /// The current size of the scheduling region.
2162 int ScheduleRegionSize = 0;
2163
2164 /// The maximum size allowed for the scheduling region.
2165 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
2166
2167 /// The ID of the scheduling region. For a new vectorization iteration this
2168 /// is incremented which "removes" all ScheduleData from the region.
2169 // Make sure that the initial SchedulingRegionID is greater than the
2170 // initial SchedulingRegionID in ScheduleData (which is 0).
2171 int SchedulingRegionID = 1;
2172 };
2173
2174 /// Attaches the BlockScheduling structures to basic blocks.
2175 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
2176
2177 /// Performs the "real" scheduling. Done before vectorization is actually
2178 /// performed in a basic block.
2179 void scheduleBlock(BlockScheduling *BS);
2180
2181 /// List of users to ignore during scheduling and that don't need extracting.
2182 ArrayRef<Value *> UserIgnoreList;
2183
2184 using OrdersType = SmallVector<unsigned, 4>;
2185 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
2186 /// sorted SmallVectors of unsigned.
2187 struct OrdersTypeDenseMapInfo {
2188 static OrdersType getEmptyKey() {
2189 OrdersType V;
2190 V.push_back(~1U);
2191 return V;
2192 }
2193
2194 static OrdersType getTombstoneKey() {
2195 OrdersType V;
2196 V.push_back(~2U);
2197 return V;
2198 }
2199
2200 static unsigned getHashValue(const OrdersType &V) {
2201 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
2202 }
2203
2204 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
2205 return LHS == RHS;
2206 }
2207 };
2208
2209 /// Contains orders of operations along with the number of bundles that have
2210 /// operations in this order. It stores only those orders that require
2211 /// reordering, if reordering is not required it is counted using \a
2212 /// NumOpsWantToKeepOriginalOrder.
2213 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder;
2214 /// Number of bundles that do not require reordering.
2215 unsigned NumOpsWantToKeepOriginalOrder = 0;
2216
2217 // Analysis and block reference.
2218 Function *F;
2219 ScalarEvolution *SE;
2220 TargetTransformInfo *TTI;
2221 TargetLibraryInfo *TLI;
2222 AliasAnalysis *AA;
2223 LoopInfo *LI;
2224 DominatorTree *DT;
2225 AssumptionCache *AC;
2226 DemandedBits *DB;
2227 const DataLayout *DL;
2228 OptimizationRemarkEmitter *ORE;
2229
2230 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
2231 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
2232
2233 /// Instruction builder to construct the vectorized tree.
2234 IRBuilder<> Builder;
2235
2236 /// A map of scalar integer values to the smallest bit width with which they
2237 /// can legally be represented. The values map to (width, signed) pairs,
2238 /// where "width" indicates the minimum bit width and "signed" is True if the
2239 /// value must be signed-extended, rather than zero-extended, back to its
2240 /// original width.
2241 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
2242};
2243
2244} // end namespace slpvectorizer
2245
2246template <> struct GraphTraits<BoUpSLP *> {
2247 using TreeEntry = BoUpSLP::TreeEntry;
2248
2249 /// NodeRef has to be a pointer per the GraphWriter.
2250 using NodeRef = TreeEntry *;
2251
2252 using ContainerTy = BoUpSLP::TreeEntry::VecTreeTy;
2253
2254 /// Add the VectorizableTree to the index iterator to be able to return
2255 /// TreeEntry pointers.
2256 struct ChildIteratorType
2257 : public iterator_adaptor_base<
2258 ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> {
2259 ContainerTy &VectorizableTree;
2260
2261 ChildIteratorType(SmallVector<BoUpSLP::EdgeInfo, 1>::iterator W,
2262 ContainerTy &VT)
2263 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
2264
2265 NodeRef operator*() { return I->UserTE; }
2266 };
2267
2268 static NodeRef getEntryNode(BoUpSLP &R) {
2269 return R.VectorizableTree[0].get();
2270 }
2271
2272 static ChildIteratorType child_begin(NodeRef N) {
2273 return {N->UserTreeIndices.begin(), N->Container};
2274 }
2275
2276 static ChildIteratorType child_end(NodeRef N) {
2277 return {N->UserTreeIndices.end(), N->Container};
2278 }
2279
2280 /// For the node iterator we just need to turn the TreeEntry iterator into a
2281 /// TreeEntry* iterator so that it dereferences to NodeRef.
2282 class nodes_iterator {
2283 using ItTy = ContainerTy::iterator;
2284 ItTy It;
2285
2286 public:
2287 nodes_iterator(const ItTy &It2) : It(It2) {}
2288 NodeRef operator*() { return It->get(); }
2289 nodes_iterator operator++() {
2290 ++It;
2291 return *this;
2292 }
2293 bool operator!=(const nodes_iterator &N2) const { return N2.It != It; }
2294 };
2295
2296 static nodes_iterator nodes_begin(BoUpSLP *R) {
2297 return nodes_iterator(R->VectorizableTree.begin());
2298 }
2299
2300 static nodes_iterator nodes_end(BoUpSLP *R) {
2301 return nodes_iterator(R->VectorizableTree.end());
2302 }
2303
2304 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
2305};
2306
2307template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
2308 using TreeEntry = BoUpSLP::TreeEntry;
2309
2310 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
2311
2312 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
2313 std::string Str;
2314 raw_string_ostream OS(Str);
2315 if (isSplat(Entry->Scalars)) {
2316 OS << "<splat> " << *Entry->Scalars[0];
2317 return Str;
2318 }
2319 for (auto V : Entry->Scalars) {
2320 OS << *V;
2321 if (std::any_of(
2322 R->ExternalUses.begin(), R->ExternalUses.end(),
2323 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
2324 OS << " <extract>";
2325 OS << "\n";
2326 }
2327 return Str;
2328 }
2329
2330 static std::string getNodeAttributes(const TreeEntry *Entry,
2331 const BoUpSLP *) {
2332 if (Entry->State == TreeEntry::NeedToGather)
2333 return "color=red";
2334 return "";
2335 }
2336};
2337
2338} // end namespace llvm
2339
2340BoUpSLP::~BoUpSLP() {
2341 for (const auto &Pair : DeletedInstructions) {
2342 // Replace operands of ignored instructions with Undefs in case if they were
2343 // marked for deletion.
2344 if (Pair.getSecond()) {
2345 Value *Undef = UndefValue::get(Pair.getFirst()->getType());
2346 Pair.getFirst()->replaceAllUsesWith(Undef);
2347 }
2348 Pair.getFirst()->dropAllReferences();
2349 }
2350 for (const auto &Pair : DeletedInstructions) {
2351 assert(Pair.getFirst()->use_empty() &&((Pair.getFirst()->use_empty() && "trying to erase instruction with users."
) ? static_cast<void> (0) : __assert_fail ("Pair.getFirst()->use_empty() && \"trying to erase instruction with users.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2352, __PRETTY_FUNCTION__))
2352 "trying to erase instruction with users.")((Pair.getFirst()->use_empty() && "trying to erase instruction with users."
) ? static_cast<void> (0) : __assert_fail ("Pair.getFirst()->use_empty() && \"trying to erase instruction with users.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2352, __PRETTY_FUNCTION__))
;
2353 Pair.getFirst()->eraseFromParent();
2354 }
2355}
2356
2357void BoUpSLP::eraseInstructions(ArrayRef<Value *> AV) {
2358 for (auto *V : AV) {
2359 if (auto *I = dyn_cast<Instruction>(V))
2360 eraseInstruction(I, /*ReplaceWithUndef=*/true);
2361 };
2362}
2363
2364void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
2365 ArrayRef<Value *> UserIgnoreLst) {
2366 ExtraValueToDebugLocsMap ExternallyUsedValues;
2367 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
2368}
2369
2370void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
2371 ExtraValueToDebugLocsMap &ExternallyUsedValues,
2372 ArrayRef<Value *> UserIgnoreLst) {
2373 deleteTree();
2374 UserIgnoreList = UserIgnoreLst;
2375 if (!allSameType(Roots))
2376 return;
2377 buildTree_rec(Roots, 0, EdgeInfo());
2378
2379 // Collect the values that we need to extract from the tree.
2380 for (auto &TEPtr : VectorizableTree) {
2381 TreeEntry *Entry = TEPtr.get();
2382
2383 // No need to handle users of gathered values.
2384 if (Entry->State == TreeEntry::NeedToGather)
2385 continue;
2386
2387 // For each lane:
2388 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
2389 Value *Scalar = Entry->Scalars[Lane];
2390 int FoundLane = Lane;
2391 if (!Entry->ReuseShuffleIndices.empty()) {
2392 FoundLane =
2393 std::distance(Entry->ReuseShuffleIndices.begin(),
2394 llvm::find(Entry->ReuseShuffleIndices, FoundLane));
2395 }
2396
2397 // Check if the scalar is externally used as an extra arg.
2398 auto ExtI = ExternallyUsedValues.find(Scalar);
2399 if (ExtI != ExternallyUsedValues.end()) {
2400 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
2401 << Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
2402 ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
2403 }
2404 for (User *U : Scalar->users()) {
2405 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
2406
2407 Instruction *UserInst = dyn_cast<Instruction>(U);
2408 if (!UserInst)
2409 continue;
2410
2411 // Skip in-tree scalars that become vectors
2412 if (TreeEntry *UseEntry = getTreeEntry(U)) {
2413 Value *UseScalar = UseEntry->Scalars[0];
2414 // Some in-tree scalars will remain as scalar in vectorized
2415 // instructions. If that is the case, the one in Lane 0 will
2416 // be used.
2417 if (UseScalar != U ||
2418 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
2419 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
2420 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
2421 assert(UseEntry->State != TreeEntry::NeedToGather && "Bad state")((UseEntry->State != TreeEntry::NeedToGather && "Bad state"
) ? static_cast<void> (0) : __assert_fail ("UseEntry->State != TreeEntry::NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2421, __PRETTY_FUNCTION__))
;
2422 continue;
2423 }
2424 }
2425
2426 // Ignore users in the user ignore list.
2427 if (is_contained(UserIgnoreList, UserInst))
2428 continue;
2429
2430 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
2431 << Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
2432 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
2433 }
2434 }
2435 }
2436}
2437
2438void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
2439 const EdgeInfo &UserTreeIdx) {
2440 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(((allConstant(VL) || allSameType(VL)) && "Invalid types!"
) ? static_cast<void> (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2440, __PRETTY_FUNCTION__))
;
2441
2442 InstructionsState S = getSameOpcode(VL);
2443 if (Depth == RecursionMaxDepth) {
2444 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
2445 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2446 return;
2447 }
2448
2449 // Don't handle vectors.
2450 if (S.OpValue->getType()->isVectorTy()) {
2451 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
2452 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2453 return;
2454 }
2455
2456 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2457 if (SI->getValueOperand()->getType()->isVectorTy()) {
2458 LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
2459 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2460 return;
2461 }
2462
2463 // If all of the operands are identical or constant we have a simple solution.
2464 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
2465 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
2466 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2467 return;
2468 }
2469
2470 // We now know that this is a vector of instructions of the same type from
2471 // the same block.
2472
2473 // Don't vectorize ephemeral values.
2474 for (Value *V : VL) {
2475 if (EphValues.count(V)) {
2476 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *Vdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is ephemeral.\n"; } } while (false)
2477 << ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is ephemeral.\n"; } } while (false)
;
2478 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2479 return;
2480 }
2481 }
2482
2483 // Check if this is a duplicate of another entry.
2484 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2485 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*S.OpValue << ".\n"; } } while (false)
;
2486 if (!E->isSame(VL)) {
2487 LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
2488 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2489 return;
2490 }
2491 // Record the reuse of the tree node. FIXME, currently this is only used to
2492 // properly draw the graph rather than for the actual vectorization.
2493 E->UserTreeIndices.push_back(UserTreeIdx);
2494 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValuedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
2495 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
;
2496 return;
2497 }
2498
2499 // Check that none of the instructions in the bundle are already in the tree.
2500 for (Value *V : VL) {
2501 auto *I = dyn_cast<Instruction>(V);
2502 if (!I)
2503 continue;
2504 if (getTreeEntry(I)) {
2505 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *Vdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is already in tree.\n"; } } while (false)
2506 << ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is already in tree.\n"; } } while (false)
;
2507 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2508 return;
2509 }
2510 }
2511
2512 // If any of the scalars is marked as a value that needs to stay scalar, then
2513 // we need to gather the scalars.
2514 // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
2515 for (Value *V : VL) {
2516 if (MustGather.count(V) || is_contained(UserIgnoreList, V)) {
2517 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
2518 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2519 return;
2520 }
2521 }
2522
2523 // Check that all of the users of the scalars that we want to vectorize are
2524 // schedulable.
2525 auto *VL0 = cast<Instruction>(S.OpValue);
2526 BasicBlock *BB = VL0->getParent();
2527
2528 if (!DT->isReachableFromEntry(BB)) {
2529 // Don't go into unreachable blocks. They may contain instructions with
2530 // dependency cycles which confuse the final scheduling.
2531 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
2532 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2533 return;
2534 }
2535
2536 // Check that every instruction appears once in this bundle.
2537 SmallVector<unsigned, 4> ReuseShuffleIndicies;
2538 SmallVector<Value *, 4> UniqueValues;
2539 DenseMap<Value *, unsigned> UniquePositions;
2540 for (Value *V : VL) {
2541 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2542 ReuseShuffleIndicies.emplace_back(Res.first->second);
2543 if (Res.second)
2544 UniqueValues.emplace_back(V);
2545 }
2546 size_t NumUniqueScalarValues = UniqueValues.size();
2547 if (NumUniqueScalarValues == VL.size()) {
2548 ReuseShuffleIndicies.clear();
2549 } else {
2550 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Shuffle for reused scalars.\n"
; } } while (false)
;
2551 if (NumUniqueScalarValues <= 1 ||
2552 !llvm::isPowerOf2_32(NumUniqueScalarValues)) {
2553 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
2554 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2555 return;
2556 }
2557 VL = UniqueValues;
2558 }
2559
2560 auto &BSRef = BlocksSchedules[BB];
2561 if (!BSRef)
2562 BSRef = std::make_unique<BlockScheduling>(BB);
2563
2564 BlockScheduling &BS = *BSRef.get();
2565
2566 Optional<ScheduleData *> Bundle = BS.tryScheduleBundle(VL, this, S);
2567 if (!Bundle) {
2568 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
2569 assert((!BS.getScheduleData(VL0) ||(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2571, __PRETTY_FUNCTION__))
2570 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2571, __PRETTY_FUNCTION__))
2571 "tryScheduleBundle should cancelScheduling on failure")(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2571, __PRETTY_FUNCTION__))
;
2572 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2573 ReuseShuffleIndicies);
2574 return;
2575 }
2576 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
2577
2578 unsigned ShuffleOrOp = S.isAltShuffle() ?
2579 (unsigned) Instruction::ShuffleVector : S.getOpcode();
2580 switch (ShuffleOrOp) {
2581 case Instruction::PHI: {
2582 auto *PH = cast<PHINode>(VL0);
2583
2584 // Check for terminator values (e.g. invoke).
2585 for (unsigned j = 0; j < VL.size(); ++j)
2586 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2587 Instruction *Term = dyn_cast<Instruction>(
2588 cast<PHINode>(VL[j])->getIncomingValueForBlock(
2589 PH->getIncomingBlock(i)));
2590 if (Term && Term->isTerminator()) {
2591 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (terminator use).\n"
; } } while (false)
2592 << "SLP: Need to swizzle PHINodes (terminator use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (terminator use).\n"
; } } while (false)
;
2593 BS.cancelScheduling(VL, VL0);
2594 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2595 ReuseShuffleIndicies);
2596 return;
2597 }
2598 }
2599
2600 TreeEntry *TE =
2601 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies);
2602 LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
2603
2604 // Keeps the reordered operands to avoid code duplication.
2605 SmallVector<ValueList, 2> OperandsVec;
2606 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2607 ValueList Operands;
2608 // Prepare the operand vector.
2609 for (Value *j : VL)
2610 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
2611 PH->getIncomingBlock(i)));
2612 TE->setOperand(i, Operands);
2613 OperandsVec.push_back(Operands);
2614 }
2615 for (unsigned OpIdx = 0, OpE = OperandsVec.size(); OpIdx != OpE; ++OpIdx)
2616 buildTree_rec(OperandsVec[OpIdx], Depth + 1, {TE, OpIdx});
2617 return;
2618 }
2619 case Instruction::ExtractValue:
2620 case Instruction::ExtractElement: {
2621 OrdersType CurrentOrder;
2622 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
2623 if (Reuse) {
2624 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing or shuffling extract sequence.\n"
; } } while (false)
;
2625 ++NumOpsWantToKeepOriginalOrder;
2626 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2627 ReuseShuffleIndicies);
2628 // This is a special case, as it does not gather, but at the same time
2629 // we are not extending buildTree_rec() towards the operands.
2630 ValueList Op0;
2631 Op0.assign(VL.size(), VL0->getOperand(0));
2632 VectorizableTree.back()->setOperand(0, Op0);
2633 return;
2634 }
2635 if (!CurrentOrder.empty()) {
2636 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2637 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2638 "with order";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2639 for (unsigned Idx : CurrentOrder)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2640 dbgs() << " " << Idx;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2641 dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2642 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
;
2643 // Insert new order with initial value 0, if it does not exist,
2644 // otherwise return the iterator to the existing one.
2645 auto StoredCurrentOrderAndNum =
2646 NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
2647 ++StoredCurrentOrderAndNum->getSecond();
2648 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2649 ReuseShuffleIndicies,
2650 StoredCurrentOrderAndNum->getFirst());
2651 // This is a special case, as it does not gather, but at the same time
2652 // we are not extending buildTree_rec() towards the operands.
2653 ValueList Op0;
2654 Op0.assign(VL.size(), VL0->getOperand(0));
2655 VectorizableTree.back()->setOperand(0, Op0);
2656 return;
2657 }
2658 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gather extract sequence.\n";
} } while (false)
;
2659 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2660 ReuseShuffleIndicies);
2661 BS.cancelScheduling(VL, VL0);
2662 return;
2663 }
2664 case Instruction::Load: {
2665 // Check that a vectorized load would load the same memory as a scalar
2666 // load. For example, we don't want to vectorize loads that are smaller
2667 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
2668 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
2669 // from such a struct, we read/write packed bits disagreeing with the
2670 // unvectorized version.
2671 Type *ScalarTy = VL0->getType();
2672
2673 if (DL->getTypeSizeInBits(ScalarTy) !=
2674 DL->getTypeAllocSizeInBits(ScalarTy)) {
2675 BS.cancelScheduling(VL, VL0);
2676 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2677 ReuseShuffleIndicies);
2678 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
2679 return;
2680 }
2681
2682 // Make sure all loads in the bundle are simple - we can't vectorize
2683 // atomic or volatile loads.
2684 SmallVector<Value *, 4> PointerOps(VL.size());
2685 auto POIter = PointerOps.begin();
2686 for (Value *V : VL) {
2687 auto *L = cast<LoadInst>(V);
2688 if (!L->isSimple()) {
2689 BS.cancelScheduling(VL, VL0);
2690 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2691 ReuseShuffleIndicies);
2692 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
2693 return;
2694 }
2695 *POIter = L->getPointerOperand();
2696 ++POIter;
2697 }
2698
2699 OrdersType CurrentOrder;
2700 // Check the order of pointer operands.
2701 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
2702 Value *Ptr0;
2703 Value *PtrN;
2704 if (CurrentOrder.empty()) {
2705 Ptr0 = PointerOps.front();
2706 PtrN = PointerOps.back();
2707 } else {
2708 Ptr0 = PointerOps[CurrentOrder.front()];
2709 PtrN = PointerOps[CurrentOrder.back()];
2710 }
2711 const SCEV *Scev0 = SE->getSCEV(Ptr0);
2712 const SCEV *ScevN = SE->getSCEV(PtrN);
2713 const auto *Diff =
2714 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
2715 uint64_t Size = DL->getTypeAllocSize(ScalarTy);
2716 // Check that the sorted loads are consecutive.
2717 if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) {
2718 if (CurrentOrder.empty()) {
2719 // Original loads are consecutive and does not require reordering.
2720 ++NumOpsWantToKeepOriginalOrder;
2721 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S,
2722 UserTreeIdx, ReuseShuffleIndicies);
2723 TE->setOperandsInOrder();
2724 LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
2725 } else {
2726 // Need to reorder.
2727 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
2728 ++I->getSecond();
2729 TreeEntry *TE =
2730 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2731 ReuseShuffleIndicies, I->getFirst());
2732 TE->setOperandsInOrder();
2733 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of jumbled loads.\n"
; } } while (false)
;
2734 }
2735 return;
2736 }
2737 }
2738
2739 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
2740 BS.cancelScheduling(VL, VL0);
2741 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2742 ReuseShuffleIndicies);
2743 return;
2744 }
2745 case Instruction::ZExt:
2746 case Instruction::SExt:
2747 case Instruction::FPToUI:
2748 case Instruction::FPToSI:
2749 case Instruction::FPExt:
2750 case Instruction::PtrToInt:
2751 case Instruction::IntToPtr:
2752 case Instruction::SIToFP:
2753 case Instruction::UIToFP:
2754 case Instruction::Trunc:
2755 case Instruction::FPTrunc:
2756 case Instruction::BitCast: {
2757 Type *SrcTy = VL0->getOperand(0)->getType();
2758 for (Value *V : VL) {
2759 Type *Ty = cast<Instruction>(V)->getOperand(0)->getType();
2760 if (Ty != SrcTy || !isValidElementType(Ty)) {
2761 BS.cancelScheduling(VL, VL0);
2762 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2763 ReuseShuffleIndicies);
2764 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
2765 << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
2766 return;
2767 }
2768 }
2769 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2770 ReuseShuffleIndicies);
2771 LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
2772
2773 TE->setOperandsInOrder();
2774 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
2775 ValueList Operands;
2776 // Prepare the operand vector.
2777 for (Value *V : VL)
2778 Operands.push_back(cast<Instruction>(V)->getOperand(i));
2779
2780 buildTree_rec(Operands, Depth + 1, {TE, i});
2781 }
2782 return;
2783 }
2784 case Instruction::ICmp:
2785 case Instruction::FCmp: {
2786 // Check that all of the compares have the same predicate.
2787 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2788 CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0);
2789 Type *ComparedTy = VL0->getOperand(0)->getType();
2790 for (Value *V : VL) {
2791 CmpInst *Cmp = cast<CmpInst>(V);
2792 if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) ||
2793 Cmp->getOperand(0)->getType() != ComparedTy) {
2794 BS.cancelScheduling(VL, VL0);
2795 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2796 ReuseShuffleIndicies);
2797 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
2798 << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
2799 return;
2800 }
2801 }
2802
2803 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2804 ReuseShuffleIndicies);
2805 LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
2806
2807 ValueList Left, Right;
2808 if (cast<CmpInst>(VL0)->isCommutative()) {
2809 // Commutative predicate - collect + sort operands of the instructions
2810 // so that each side is more likely to have the same opcode.
2811 assert(P0 == SwapP0 && "Commutative Predicate mismatch")((P0 == SwapP0 && "Commutative Predicate mismatch") ?
static_cast<void> (0) : __assert_fail ("P0 == SwapP0 && \"Commutative Predicate mismatch\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2811, __PRETTY_FUNCTION__))
;
2812 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
2813 } else {
2814 // Collect operands - commute if it uses the swapped predicate.
2815 for (Value *V : VL) {
2816 auto *Cmp = cast<CmpInst>(V);
2817 Value *LHS = Cmp->getOperand(0);
2818 Value *RHS = Cmp->getOperand(1);
2819 if (Cmp->getPredicate() != P0)
2820 std::swap(LHS, RHS);
2821 Left.push_back(LHS);
2822 Right.push_back(RHS);
2823 }
2824 }
2825 TE->setOperand(0, Left);
2826 TE->setOperand(1, Right);
2827 buildTree_rec(Left, Depth + 1, {TE, 0});
2828 buildTree_rec(Right, Depth + 1, {TE, 1});
2829 return;
2830 }
2831 case Instruction::Select:
2832 case Instruction::FNeg:
2833 case Instruction::Add:
2834 case Instruction::FAdd:
2835 case Instruction::Sub:
2836 case Instruction::FSub:
2837 case Instruction::Mul:
2838 case Instruction::FMul:
2839 case Instruction::UDiv:
2840 case Instruction::SDiv:
2841 case Instruction::FDiv:
2842 case Instruction::URem:
2843 case Instruction::SRem:
2844 case Instruction::FRem:
2845 case Instruction::Shl:
2846 case Instruction::LShr:
2847 case Instruction::AShr:
2848 case Instruction::And:
2849 case Instruction::Or:
2850 case Instruction::Xor: {
2851 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2852 ReuseShuffleIndicies);
2853 LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of un/bin op.\n"
; } } while (false)
;
2854
2855 // Sort operands of the instructions so that each side is more likely to
2856 // have the same opcode.
2857 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
2858 ValueList Left, Right;
2859 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
2860 TE->setOperand(0, Left);
2861 TE->setOperand(1, Right);
2862 buildTree_rec(Left, Depth + 1, {TE, 0});
2863 buildTree_rec(Right, Depth + 1, {TE, 1});
2864 return;
2865 }
2866
2867 TE->setOperandsInOrder();
2868 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
2869 ValueList Operands;
2870 // Prepare the operand vector.
2871 for (Value *j : VL)
2872 Operands.push_back(cast<Instruction>(j)->getOperand(i));
2873
2874 buildTree_rec(Operands, Depth + 1, {TE, i});
2875 }
2876 return;
2877 }
2878 case Instruction::GetElementPtr: {
2879 // We don't combine GEPs with complicated (nested) indexing.
2880 for (Value *V : VL) {
2881 if (cast<Instruction>(V)->getNumOperands() != 2) {
2882 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
2883 BS.cancelScheduling(VL, VL0);
2884 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2885 ReuseShuffleIndicies);
2886 return;
2887 }
2888 }
2889
2890 // We can't combine several GEPs into one vector if they operate on
2891 // different types.
2892 Type *Ty0 = VL0->getOperand(0)->getType();
2893 for (Value *V : VL) {
2894 Type *CurTy = cast<Instruction>(V)->getOperand(0)->getType();
2895 if (Ty0 != CurTy) {
2896 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
2897 << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
2898 BS.cancelScheduling(VL, VL0);
2899 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2900 ReuseShuffleIndicies);
2901 return;
2902 }
2903 }
2904
2905 // We don't combine GEPs with non-constant indexes.
2906 Type *Ty1 = VL0->getOperand(1)->getType();
2907 for (Value *V : VL) {
2908 auto Op = cast<Instruction>(V)->getOperand(1);
2909 if (!isa<ConstantInt>(Op) ||
2910 (Op->getType() != Ty1 &&
2911 Op->getType()->getScalarSizeInBits() >
2912 DL->getIndexSizeInBits(
2913 V->getType()->getPointerAddressSpace()))) {
2914 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
2915 << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
2916 BS.cancelScheduling(VL, VL0);
2917 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2918 ReuseShuffleIndicies);
2919 return;
2920 }
2921 }
2922
2923 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2924 ReuseShuffleIndicies);
2925 LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
2926 TE->setOperandsInOrder();
2927 for (unsigned i = 0, e = 2; i < e; ++i) {
2928 ValueList Operands;
2929 // Prepare the operand vector.
2930 for (Value *V : VL)
2931 Operands.push_back(cast<Instruction>(V)->getOperand(i));
2932
2933 buildTree_rec(Operands, Depth + 1, {TE, i});
2934 }
2935 return;
2936 }
2937 case Instruction::Store: {
2938 // Check if the stores are consecutive or if we need to swizzle them.
2939 llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType();
2940 // Make sure all stores in the bundle are simple - we can't vectorize
2941 // atomic or volatile stores.
2942 SmallVector<Value *, 4> PointerOps(VL.size());
2943 ValueList Operands(VL.size());
2944 auto POIter = PointerOps.begin();
2945 auto OIter = Operands.begin();
2946 for (Value *V : VL) {
2947 auto *SI = cast<StoreInst>(V);
2948 if (!SI->isSimple()) {
2949 BS.cancelScheduling(VL, VL0);
2950 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2951 ReuseShuffleIndicies);
2952 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple stores.\n"
; } } while (false)
;
2953 return;
2954 }
2955 *POIter = SI->getPointerOperand();
2956 *OIter = SI->getValueOperand();
2957 ++POIter;
2958 ++OIter;
2959 }
2960
2961 OrdersType CurrentOrder;
2962 // Check the order of pointer operands.
2963 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
2964 Value *Ptr0;
2965 Value *PtrN;
2966 if (CurrentOrder.empty()) {
2967 Ptr0 = PointerOps.front();
2968 PtrN = PointerOps.back();
2969 } else {
2970 Ptr0 = PointerOps[CurrentOrder.front()];
2971 PtrN = PointerOps[CurrentOrder.back()];
2972 }
2973 const SCEV *Scev0 = SE->getSCEV(Ptr0);
2974 const SCEV *ScevN = SE->getSCEV(PtrN);
2975 const auto *Diff =
2976 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
2977 uint64_t Size = DL->getTypeAllocSize(ScalarTy);
2978 // Check that the sorted pointer operands are consecutive.
2979 if (Diff && Diff->getAPInt() == (VL.size() - 1) * Size) {
2980 if (CurrentOrder.empty()) {
2981 // Original stores are consecutive and does not require reordering.
2982 ++NumOpsWantToKeepOriginalOrder;
2983 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S,
2984 UserTreeIdx, ReuseShuffleIndicies);
2985 TE->setOperandsInOrder();
2986 buildTree_rec(Operands, Depth + 1, {TE, 0});
2987 LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
2988 } else {
2989 // Need to reorder.
2990 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
2991 ++(I->getSecond());
2992 TreeEntry *TE =
2993 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2994 ReuseShuffleIndicies, I->getFirst());
2995 TE->setOperandsInOrder();
2996 buildTree_rec(Operands, Depth + 1, {TE, 0});
2997 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of jumbled stores.\n"
; } } while (false)
;
2998 }
2999 return;
3000 }
3001 }
3002
3003 BS.cancelScheduling(VL, VL0);
3004 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3005 ReuseShuffleIndicies);
3006 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
3007 return;
3008 }
3009 case Instruction::Call: {
3010 // Check if the calls are all to the same vectorizable intrinsic.
3011 CallInst *CI = cast<CallInst>(VL0);
3012 // Check if this is an Intrinsic call or something that can be
3013 // represented by an intrinsic call
3014 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3015 if (!isTriviallyVectorizable(ID)) {
3016 BS.cancelScheduling(VL, VL0);
3017 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3018 ReuseShuffleIndicies);
3019 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
3020 return;
3021 }
3022 Function *Int = CI->getCalledFunction();
3023 unsigned NumArgs = CI->getNumArgOperands();
3024 SmallVector<Value*, 4> ScalarArgs(NumArgs, nullptr);
3025 for (unsigned j = 0; j != NumArgs; ++j)
3026 if (hasVectorInstrinsicScalarOpd(ID, j))
3027 ScalarArgs[j] = CI->getArgOperand(j);
3028 for (Value *V : VL) {
3029 CallInst *CI2 = dyn_cast<CallInst>(V);
3030 if (!CI2 || CI2->getCalledFunction() != Int ||
3031 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
3032 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
3033 BS.cancelScheduling(VL, VL0);
3034 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3035 ReuseShuffleIndicies);
3036 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *Vdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *V << "\n"; } } while (false)
3037 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *V << "\n"; } } while (false)
;
3038 return;
3039 }
3040 // Some intrinsics have scalar arguments and should be same in order for
3041 // them to be vectorized.
3042 for (unsigned j = 0; j != NumArgs; ++j) {
3043 if (hasVectorInstrinsicScalarOpd(ID, j)) {
3044 Value *A1J = CI2->getArgOperand(j);
3045 if (ScalarArgs[j] != A1J) {
3046 BS.cancelScheduling(VL, VL0);
3047 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3048 ReuseShuffleIndicies);
3049 LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << ScalarArgs[j] <<
"!=" << A1J << "\n"; } } while (false)
3050 << " argument " << ScalarArgs[j] << "!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << ScalarArgs[j] <<
"!=" << A1J << "\n"; } } while (false)
3051 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << ScalarArgs[j] <<
"!=" << A1J << "\n"; } } while (false)
;
3052 return;
3053 }
3054 }
3055 }
3056 // Verify that the bundle operands are identical between the two calls.
3057 if (CI->hasOperandBundles() &&
3058 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
3059 CI->op_begin() + CI->getBundleOperandsEndIndex(),
3060 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
3061 BS.cancelScheduling(VL, VL0);
3062 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3063 ReuseShuffleIndicies);
3064 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *V << '\n'; } } while
(false)
3065 << *CI << "!=" << *V << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *V << '\n'; } } while
(false)
;
3066 return;
3067 }
3068 }
3069
3070 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3071 ReuseShuffleIndicies);
3072 TE->setOperandsInOrder();
3073 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
3074 ValueList Operands;
3075 // Prepare the operand vector.
3076 for (Value *V : VL) {
3077 auto *CI2 = cast<CallInst>(V);
3078 Operands.push_back(CI2->getArgOperand(i));
3079 }
3080 buildTree_rec(Operands, Depth + 1, {TE, i});
3081 }
3082 return;
3083 }
3084 case Instruction::ShuffleVector: {
3085 // If this is not an alternate sequence of opcode like add-sub
3086 // then do not vectorize this instruction.
3087 if (!S.isAltShuffle()) {
3088 BS.cancelScheduling(VL, VL0);
3089 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3090 ReuseShuffleIndicies);
3091 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
3092 return;
3093 }
3094 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3095 ReuseShuffleIndicies);
3096 LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
3097
3098 // Reorder operands if reordering would enable vectorization.
3099 if (isa<BinaryOperator>(VL0)) {
3100 ValueList Left, Right;
3101 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3102 TE->setOperand(0, Left);
3103 TE->setOperand(1, Right);
3104 buildTree_rec(Left, Depth + 1, {TE, 0});
3105 buildTree_rec(Right, Depth + 1, {TE, 1});
3106 return;
3107 }
3108
3109 TE->setOperandsInOrder();
3110 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3111 ValueList Operands;
3112 // Prepare the operand vector.
3113 for (Value *V : VL)
3114 Operands.push_back(cast<Instruction>(V)->getOperand(i));
3115
3116 buildTree_rec(Operands, Depth + 1, {TE, i});
3117 }
3118 return;
3119 }
3120 default:
3121 BS.cancelScheduling(VL, VL0);
3122 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3123 ReuseShuffleIndicies);
3124 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
3125 return;
3126 }
3127}
3128
3129unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
3130 unsigned N = 1;
3131 Type *EltTy = T;
3132
3133 while (isa<CompositeType>(EltTy)) {
3134 if (auto *ST = dyn_cast<StructType>(EltTy)) {
3135 // Check that struct is homogeneous.
3136 for (const auto *Ty : ST->elements())
3137 if (Ty != *ST->element_begin())
3138 return 0;
3139 N *= ST->getNumElements();
3140 EltTy = *ST->element_begin();
3141 } else {
3142 auto *SeqT = cast<SequentialType>(EltTy);
3143 N *= SeqT->getNumElements();
3144 EltTy = SeqT->getElementType();
3145 }
3146 }
3147
3148 if (!isValidElementType(EltTy))
3149 return 0;
3150 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
3151 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
3152 return 0;
3153 return N;
3154}
3155
3156bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
3157 SmallVectorImpl<unsigned> &CurrentOrder) const {
3158 Instruction *E0 = cast<Instruction>(OpValue);
3159 assert(E0->getOpcode() == Instruction::ExtractElement ||((E0->getOpcode() == Instruction::ExtractElement || E0->
getOpcode() == Instruction::ExtractValue) ? static_cast<void
> (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3160, __PRETTY_FUNCTION__))
3160 E0->getOpcode() == Instruction::ExtractValue)((E0->getOpcode() == Instruction::ExtractElement || E0->
getOpcode() == Instruction::ExtractValue) ? static_cast<void
> (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3160, __PRETTY_FUNCTION__))
;
3161 assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode")((E0->getOpcode() == getSameOpcode(VL).getOpcode() &&
"Invalid opcode") ? static_cast<void> (0) : __assert_fail
("E0->getOpcode() == getSameOpcode(VL).getOpcode() && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3161, __PRETTY_FUNCTION__))
;
3162 // Check if all of the extracts come from the same vector and from the
3163 // correct offset.
3164 Value *Vec = E0->getOperand(0);
3165
3166 CurrentOrder.clear();
3167
3168 // We have to extract from a vector/aggregate with the same number of elements.
3169 unsigned NElts;
3170 if (E0->getOpcode() == Instruction::ExtractValue) {
3171 const DataLayout &DL = E0->getModule()->getDataLayout();
3172 NElts = canMapToVector(Vec->getType(), DL);
3173 if (!NElts)
3174 return false;
3175 // Check if load can be rewritten as load of vector.
3176 LoadInst *LI = dyn_cast<LoadInst>(Vec);
3177 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
3178 return false;
3179 } else {
3180 NElts = Vec->getType()->getVectorNumElements();
3181 }
3182
3183 if (NElts != VL.size())
3184 return false;
3185
3186 // Check that all of the indices extract from the correct offset.
3187 bool ShouldKeepOrder = true;
3188 unsigned E = VL.size();
3189 // Assign to all items the initial value E + 1 so we can check if the extract
3190 // instruction index was used already.
3191 // Also, later we can check that all the indices are used and we have a
3192 // consecutive access in the extract instructions, by checking that no
3193 // element of CurrentOrder still has value E + 1.
3194 CurrentOrder.assign(E, E + 1);
3195 unsigned I = 0;
3196 for (; I < E; ++I) {
3197 auto *Inst = cast<Instruction>(VL[I]);
3198 if (Inst->getOperand(0) != Vec)
3199 break;
3200 Optional<unsigned> Idx = getExtractIndex(Inst);
3201 if (!Idx)
3202 break;
3203 const unsigned ExtIdx = *Idx;
3204 if (ExtIdx != I) {
3205 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
3206 break;
3207 ShouldKeepOrder = false;
3208 CurrentOrder[ExtIdx] = I;
3209 } else {
3210 if (CurrentOrder[I] != E + 1)
3211 break;
3212 CurrentOrder[I] = I;
3213 }
3214 }
3215 if (I < E) {
3216 CurrentOrder.clear();
3217 return false;
3218 }
3219
3220 return ShouldKeepOrder;
3221}
3222
3223bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
3224 return I->hasOneUse() ||
3225 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
3226 return ScalarToTreeEntry.count(U) > 0;
3227 });
3228}
3229
3230int BoUpSLP::getEntryCost(TreeEntry *E) {
3231 ArrayRef<Value*> VL = E->Scalars;
3232
3233 Type *ScalarTy = VL[0]->getType();
3234 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
3235 ScalarTy = SI->getValueOperand()->getType();
3236 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
3237 ScalarTy = CI->getOperand(0)->getType();
3238 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3239
3240 // If we have computed a smaller type for the expression, update VecTy so
3241 // that the costs will be accurate.
3242 if (MinBWs.count(VL[0]))
3243 VecTy = VectorType::get(
3244 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
3245
3246 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
3247 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3248 int ReuseShuffleCost = 0;
3249 if (NeedToShuffleReuses) {
3250 ReuseShuffleCost =
3251 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
3252 }
3253 if (E->State == TreeEntry::NeedToGather) {
3254 if (allConstant(VL))
3255 return 0;
3256 if (isSplat(VL)) {
3257 return ReuseShuffleCost +
3258 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
3259 }
3260 if (E->getOpcode() == Instruction::ExtractElement &&
3261 allSameType(VL) && allSameBlock(VL)) {
3262 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
3263 if (ShuffleKind.hasValue()) {
3264 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
3265 for (auto *V : VL) {
3266 // If all users of instruction are going to be vectorized and this
3267 // instruction itself is not going to be vectorized, consider this
3268 // instruction as dead and remove its cost from the final cost of the
3269 // vectorized tree.
3270 if (areAllUsersVectorized(cast<Instruction>(V)) &&
3271 !ScalarToTreeEntry.count(V)) {
3272 auto *IO = cast<ConstantInt>(
3273 cast<ExtractElementInst>(V)->getIndexOperand());
3274 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
3275 IO->getZExtValue());
3276 }
3277 }
3278 return ReuseShuffleCost + Cost;
3279 }
3280 }
3281 return ReuseShuffleCost + getGatherCost(VL);
3282 }
3283 assert(E->State == TreeEntry::Vectorize && "Unhandled state")((E->State == TreeEntry::Vectorize && "Unhandled state"
) ? static_cast<void> (0) : __assert_fail ("E->State == TreeEntry::Vectorize && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3283, __PRETTY_FUNCTION__))
;
3284 assert(E->getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL")((E->getOpcode() && allSameType(VL) && allSameBlock
(VL) && "Invalid VL") ? static_cast<void> (0) :
__assert_fail ("E->getOpcode() && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3284, __PRETTY_FUNCTION__))
;
3285 Instruction *VL0 = E->getMainOp();
3286 unsigned ShuffleOrOp =
3287 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
3288 switch (ShuffleOrOp) {
3289 case Instruction::PHI:
3290 return 0;
3291
3292 case Instruction::ExtractValue:
3293 case Instruction::ExtractElement: {
3294 if (NeedToShuffleReuses) {
3295 unsigned Idx = 0;
3296 for (unsigned I : E->ReuseShuffleIndices) {
3297 if (ShuffleOrOp == Instruction::ExtractElement) {
3298 auto *IO = cast<ConstantInt>(
3299 cast<ExtractElementInst>(VL[I])->getIndexOperand());
3300 Idx = IO->getZExtValue();
3301 ReuseShuffleCost -= TTI->getVectorInstrCost(
3302 Instruction::ExtractElement, VecTy, Idx);
3303 } else {
3304 ReuseShuffleCost -= TTI->getVectorInstrCost(
3305 Instruction::ExtractElement, VecTy, Idx);
3306 ++Idx;
3307 }
3308 }
3309 Idx = ReuseShuffleNumbers;
3310 for (Value *V : VL) {
3311 if (ShuffleOrOp == Instruction::ExtractElement) {
3312 auto *IO = cast<ConstantInt>(
3313 cast<ExtractElementInst>(V)->getIndexOperand());
3314 Idx = IO->getZExtValue();
3315 } else {
3316 --Idx;
3317 }
3318 ReuseShuffleCost +=
3319 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
3320 }
3321 }
3322 int DeadCost = ReuseShuffleCost;
3323 if (!E->ReorderIndices.empty()) {
3324 // TODO: Merge this shuffle with the ReuseShuffleCost.
3325 DeadCost += TTI->getShuffleCost(
3326 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
3327 }
3328 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
3329 Instruction *E = cast<Instruction>(VL[i]);
3330 // If all users are going to be vectorized, instruction can be
3331 // considered as dead.
3332 // The same, if have only one user, it will be vectorized for sure.
3333 if (areAllUsersVectorized(E)) {
3334 // Take credit for instruction that will become dead.
3335 if (E->hasOneUse()) {
3336 Instruction *Ext = E->user_back();
3337 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
3338 all_of(Ext->users(),
3339 [](User *U) { return isa<GetElementPtrInst>(U); })) {
3340 // Use getExtractWithExtendCost() to calculate the cost of
3341 // extractelement/ext pair.
3342 DeadCost -= TTI->getExtractWithExtendCost(
3343 Ext->getOpcode(), Ext->getType(), VecTy, i);
3344 // Add back the cost of s|zext which is subtracted separately.
3345 DeadCost += TTI->getCastInstrCost(
3346 Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
3347 continue;
3348 }
3349 }
3350 DeadCost -=
3351 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
3352 }
3353 }
3354 return DeadCost;
3355 }
3356 case Instruction::ZExt:
3357 case Instruction::SExt:
3358 case Instruction::FPToUI:
3359 case Instruction::FPToSI:
3360 case Instruction::FPExt:
3361 case Instruction::PtrToInt:
3362 case Instruction::IntToPtr:
3363 case Instruction::SIToFP:
3364 case Instruction::UIToFP:
3365 case Instruction::Trunc:
3366 case Instruction::FPTrunc:
3367 case Instruction::BitCast: {
3368 Type *SrcTy = VL0->getOperand(0)->getType();
3369 int ScalarEltCost =
3370 TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy, VL0);
3371 if (NeedToShuffleReuses) {
3372 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3373 }
3374
3375 // Calculate the cost of this instruction.
3376 int ScalarCost = VL.size() * ScalarEltCost;
3377
3378 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
3379 int VecCost = 0;
3380 // Check if the values are candidates to demote.
3381 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
3382 VecCost = ReuseShuffleCost +
3383 TTI->getCastInstrCost(E->getOpcode(), VecTy, SrcVecTy, VL0);
3384 }
3385 return VecCost - ScalarCost;
3386 }
3387 case Instruction::FCmp:
3388 case Instruction::ICmp:
3389 case Instruction::Select: {
3390 // Calculate the cost of this instruction.
3391 int ScalarEltCost = TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy,
3392 Builder.getInt1Ty(), VL0);
3393 if (NeedToShuffleReuses) {
3394 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3395 }
3396 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
3397 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
3398 int VecCost = TTI->getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy, VL0);
3399 return ReuseShuffleCost + VecCost - ScalarCost;
3400 }
3401 case Instruction::FNeg:
3402 case Instruction::Add:
3403 case Instruction::FAdd:
3404 case Instruction::Sub:
3405 case Instruction::FSub:
3406 case Instruction::Mul:
3407 case Instruction::FMul:
3408 case Instruction::UDiv:
3409 case Instruction::SDiv:
3410 case Instruction::FDiv:
3411 case Instruction::URem:
3412 case Instruction::SRem:
3413 case Instruction::FRem:
3414 case Instruction::Shl:
3415 case Instruction::LShr:
3416 case Instruction::AShr:
3417 case Instruction::And:
3418 case Instruction::Or:
3419 case Instruction::Xor: {
3420 // Certain instructions can be cheaper to vectorize if they have a
3421 // constant second vector operand.
3422 TargetTransformInfo::OperandValueKind Op1VK =
3423 TargetTransformInfo::OK_AnyValue;
3424 TargetTransformInfo::OperandValueKind Op2VK =
3425 TargetTransformInfo::OK_UniformConstantValue;
3426 TargetTransformInfo::OperandValueProperties Op1VP =
3427 TargetTransformInfo::OP_None;
3428 TargetTransformInfo::OperandValueProperties Op2VP =
3429 TargetTransformInfo::OP_PowerOf2;
3430
3431 // If all operands are exactly the same ConstantInt then set the
3432 // operand kind to OK_UniformConstantValue.
3433 // If instead not all operands are constants, then set the operand kind
3434 // to OK_AnyValue. If all operands are constants but not the same,
3435 // then set the operand kind to OK_NonUniformConstantValue.
3436 ConstantInt *CInt0 = nullptr;
3437 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
3438 const Instruction *I = cast<Instruction>(VL[i]);
3439 unsigned OpIdx = isa<BinaryOperator>(I) ? 1 : 0;
3440 ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(OpIdx));
3441 if (!CInt) {
3442 Op2VK = TargetTransformInfo::OK_AnyValue;
3443 Op2VP = TargetTransformInfo::OP_None;
3444 break;
3445 }
3446 if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
3447 !CInt->getValue().isPowerOf2())
3448 Op2VP = TargetTransformInfo::OP_None;
3449 if (i == 0) {
3450 CInt0 = CInt;
3451 continue;
3452 }
3453 if (CInt0 != CInt)
3454 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
3455 }
3456
3457 SmallVector<const Value *, 4> Operands(VL0->operand_values());
3458 int ScalarEltCost = TTI->getArithmeticInstrCost(
3459 E->getOpcode(), ScalarTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands, VL0);
3460 if (NeedToShuffleReuses) {
3461 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3462 }
3463 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
3464 int VecCost = TTI->getArithmeticInstrCost(
3465 E->getOpcode(), VecTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands, VL0);
3466 return ReuseShuffleCost + VecCost - ScalarCost;
3467 }
3468 case Instruction::GetElementPtr: {
3469 TargetTransformInfo::OperandValueKind Op1VK =
3470 TargetTransformInfo::OK_AnyValue;
3471 TargetTransformInfo::OperandValueKind Op2VK =
3472 TargetTransformInfo::OK_UniformConstantValue;
3473
3474 int ScalarEltCost =
3475 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
3476 if (NeedToShuffleReuses) {
3477 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3478 }
3479 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
3480 int VecCost =
3481 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
3482 return ReuseShuffleCost + VecCost - ScalarCost;
3483 }
3484 case Instruction::Load: {
3485 // Cost of wide load - cost of scalar loads.
3486 MaybeAlign alignment(cast<LoadInst>(VL0)->getAlignment());
3487 int ScalarEltCost =
3488 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
3489 if (NeedToShuffleReuses) {
3490 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3491 }
3492 int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
3493 int VecLdCost =
3494 TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, VL0);
3495 if (!E->ReorderIndices.empty()) {
3496 // TODO: Merge this shuffle with the ReuseShuffleCost.
3497 VecLdCost += TTI->getShuffleCost(
3498 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
3499 }
3500 return ReuseShuffleCost + VecLdCost - ScalarLdCost;
3501 }
3502 case Instruction::Store: {
3503 // We know that we can merge the stores. Calculate the cost.
3504 bool IsReorder = !E->ReorderIndices.empty();
3505 auto *SI =
3506 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
3507 MaybeAlign Alignment(SI->getAlignment());
3508 int ScalarEltCost =
3509 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, Alignment, 0, VL0);
3510 if (NeedToShuffleReuses)
3511 ReuseShuffleCost = -(ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3512 int ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
3513 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
3514 VecTy, Alignment, 0, VL0);
3515 if (IsReorder) {
3516 // TODO: Merge this shuffle with the ReuseShuffleCost.
3517 VecStCost += TTI->getShuffleCost(
3518 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
3519 }
3520 return ReuseShuffleCost + VecStCost - ScalarStCost;
3521 }
3522 case Instruction::Call: {
3523 CallInst *CI = cast<CallInst>(VL0);
3524 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3525
3526 // Calculate the cost of the scalar and vector calls.
3527 SmallVector<Type *, 4> ScalarTys;
3528 for (unsigned op = 0, opc = CI->getNumArgOperands(); op != opc; ++op)
3529 ScalarTys.push_back(CI->getArgOperand(op)->getType());
3530
3531 FastMathFlags FMF;
3532 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
3533 FMF = FPMO->getFastMathFlags();
3534
3535 int ScalarEltCost =
3536 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
3537 if (NeedToShuffleReuses) {
3538 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3539 }
3540 int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
3541
3542 SmallVector<Value *, 4> Args(CI->arg_operands());
3543 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
3544 VecTy->getNumElements());
3545
3546 LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
3547 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
3548 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
3549
3550 return ReuseShuffleCost + VecCallCost - ScalarCallCost;
3551 }
3552 case Instruction::ShuffleVector: {
3553 assert(E->isAltShuffle() &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3558, __PRETTY_FUNCTION__))
3554 ((Instruction::isBinaryOp(E->getOpcode()) &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3558, __PRETTY_FUNCTION__))
3555 Instruction::isBinaryOp(E->getAltOpcode())) ||((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3558, __PRETTY_FUNCTION__))
3556 (Instruction::isCast(E->getOpcode()) &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3558, __PRETTY_FUNCTION__))
3557 Instruction::isCast(E->getAltOpcode()))) &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3558, __PRETTY_FUNCTION__))
3558 "Invalid Shuffle Vector Operand")((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3558, __PRETTY_FUNCTION__))
;
3559 int ScalarCost = 0;
3560 if (NeedToShuffleReuses) {
3561 for (unsigned Idx : E->ReuseShuffleIndices) {
3562 Instruction *I = cast<Instruction>(VL[Idx]);
3563 ReuseShuffleCost -= TTI->getInstructionCost(
3564 I, TargetTransformInfo::TCK_RecipThroughput);
3565 }
3566 for (Value *V : VL) {
3567 Instruction *I = cast<Instruction>(V);
3568 ReuseShuffleCost += TTI->getInstructionCost(
3569 I, TargetTransformInfo::TCK_RecipThroughput);
3570 }
3571 }
3572 for (Value *V : VL) {
3573 Instruction *I = cast<Instruction>(V);
3574 assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode")((E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"
) ? static_cast<void> (0) : __assert_fail ("E->isOpcodeOrAlt(I) && \"Unexpected main/alternate opcode\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3574, __PRETTY_FUNCTION__))
;
3575 ScalarCost += TTI->getInstructionCost(
3576 I, TargetTransformInfo::TCK_RecipThroughput);
3577 }
3578 // VecCost is equal to sum of the cost of creating 2 vectors
3579 // and the cost of creating shuffle.
3580 int VecCost = 0;
3581 if (Instruction::isBinaryOp(E->getOpcode())) {
3582 VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy);
3583 VecCost += TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy);
3584 } else {
3585 Type *Src0SclTy = E->getMainOp()->getOperand(0)->getType();
3586 Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType();
3587 VectorType *Src0Ty = VectorType::get(Src0SclTy, VL.size());
3588 VectorType *Src1Ty = VectorType::get(Src1SclTy, VL.size());
3589 VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty);
3590 VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty);
3591 }
3592 VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
3593 return ReuseShuffleCost + VecCost - ScalarCost;
3594 }
3595 default:
3596 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3596)
;
3597 }
3598}
3599
3600bool BoUpSLP::isFullyVectorizableTinyTree() const {
3601 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
3602 << VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
3603
3604 // We only handle trees of heights 1 and 2.
3605 if (VectorizableTree.size() == 1 &&
3606 VectorizableTree[0]->State == TreeEntry::Vectorize)
3607 return true;
3608
3609 if (VectorizableTree.size() != 2)
3610 return false;
3611
3612 // Handle splat and all-constants stores.
3613 if (VectorizableTree[0]->State == TreeEntry::Vectorize &&
3614 (allConstant(VectorizableTree[1]->Scalars) ||
3615 isSplat(VectorizableTree[1]->Scalars)))
3616 return true;
3617
3618 // Gathering cost would be too much for tiny trees.
3619 if (VectorizableTree[0]->State == TreeEntry::NeedToGather ||
3620 VectorizableTree[1]->State == TreeEntry::NeedToGather)
3621 return false;
3622
3623 return true;
3624}
3625
3626bool BoUpSLP::isLoadCombineReductionCandidate(unsigned RdxOpcode) const {
3627 if (RdxOpcode != Instruction::Or)
3628 return false;
3629
3630 unsigned NumElts = VectorizableTree[0]->Scalars.size();
3631 Value *FirstReduced = VectorizableTree[0]->Scalars[0];
3632
3633 // Look past the reduction to find a source value. Arbitrarily follow the
3634 // path through operand 0 of any 'or'. Also, peek through optional
3635 // shift-left-by-constant.
3636 Value *ZextLoad = FirstReduced;
3637 while (match(ZextLoad, m_Or(m_Value(), m_Value())) ||
3638 match(ZextLoad, m_Shl(m_Value(), m_Constant())))
3639 ZextLoad = cast<BinaryOperator>(ZextLoad)->getOperand(0);
3640
3641 // Check if the input to the reduction is an extended load.
3642 Value *LoadPtr;
3643 if (!match(ZextLoad, m_ZExt(m_Load(m_Value(LoadPtr)))))
3644 return false;
3645
3646 // Require that the total load bit width is a legal integer type.
3647 // For example, <8 x i8> --> i64 is a legal integer on a 64-bit target.
3648 // But <16 x i8> --> i128 is not, so the backend probably can't reduce it.
3649 Type *SrcTy = LoadPtr->getType()->getPointerElementType();
3650 unsigned LoadBitWidth = SrcTy->getIntegerBitWidth() * NumElts;
3651 LLVMContext &Context = FirstReduced->getContext();
3652 if (!TTI->isTypeLegal(IntegerType::get(Context, LoadBitWidth)))
3653 return false;
3654
3655 // Everything matched - assume that we can fold the whole sequence using
3656 // load combining.
3657 LLVM_DEBUG(dbgs() << "SLP: Assume load combining for scalar reduction of "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Assume load combining for scalar reduction of "
<< *(cast<Instruction>(FirstReduced)) << "\n"
; } } while (false)
3658 << *(cast<Instruction>(FirstReduced)) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Assume load combining for scalar reduction of "
<< *(cast<Instruction>(FirstReduced)) << "\n"
; } } while (false)
;
3659
3660 return true;
3661}
3662
3663bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() const {
3664 // We can vectorize the tree if its size is greater than or equal to the
3665 // minimum size specified by the MinTreeSize command line option.
3666 if (VectorizableTree.size() >= MinTreeSize)
3667 return false;
3668
3669 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
3670 // can vectorize it if we can prove it fully vectorizable.
3671 if (isFullyVectorizableTinyTree())
3672 return false;
3673
3674 assert(VectorizableTree.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3676, __PRETTY_FUNCTION__))
3675 ? ExternalUses.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3676, __PRETTY_FUNCTION__))
3676 : true && "We shouldn't have any external users")((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3676, __PRETTY_FUNCTION__))
;
3677
3678 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
3679 // vectorizable.
3680 return true;
3681}
3682
3683int BoUpSLP::getSpillCost() const {
3684 // Walk from the bottom of the tree to the top, tracking which values are
3685 // live. When we see a call instruction that is not part of our tree,
3686 // query TTI to see if there is a cost to keeping values live over it
3687 // (for example, if spills and fills are required).
3688 unsigned BundleWidth = VectorizableTree.front()->Scalars.size();
3689 int Cost = 0;
3690
3691 SmallPtrSet<Instruction*, 4> LiveValues;
3692 Instruction *PrevInst = nullptr;
3693
3694 for (const auto &TEPtr : VectorizableTree) {
3695 Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]);
3696 if (!Inst)
3697 continue;
3698
3699 if (!PrevInst) {
3700 PrevInst = Inst;
3701 continue;
3702 }
3703
3704 // Update LiveValues.
3705 LiveValues.erase(PrevInst);
3706 for (auto &J : PrevInst->operands()) {
3707 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
3708 LiveValues.insert(cast<Instruction>(&*J));
3709 }
3710
3711 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
3712 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
3713 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
3714 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
3715 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
3716 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
3717 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
;
3718
3719 // Now find the sequence of instructions between PrevInst and Inst.
3720 unsigned NumCalls = 0;
3721 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
3722 PrevInstIt =
3723 PrevInst->getIterator().getReverse();
3724 while (InstIt != PrevInstIt) {
3725 if (PrevInstIt == PrevInst->getParent()->rend()) {
3726 PrevInstIt = Inst->getParent()->rbegin();
3727 continue;
3728 }
3729
3730 // Debug information does not impact spill cost.
3731 if ((isa<CallInst>(&*PrevInstIt) &&
3732 !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
3733 &*PrevInstIt != PrevInst)
3734 NumCalls++;
3735
3736 ++PrevInstIt;
3737 }
3738
3739 if (NumCalls) {
3740 SmallVector<Type*, 4> V;
3741 for (auto *II : LiveValues)
3742 V.push_back(VectorType::get(II->getType(), BundleWidth));
3743 Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(V);
3744 }
3745
3746 PrevInst = Inst;
3747 }
3748
3749 return Cost;
3750}
3751
3752int BoUpSLP::getTreeCost() {
3753 int Cost = 0;
3754 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
3755 << VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
3756
3757 unsigned BundleWidth = VectorizableTree[0]->Scalars.size();
3758
3759 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
3760 TreeEntry &TE = *VectorizableTree[I].get();
3761
3762 // We create duplicate tree entries for gather sequences that have multiple
3763 // uses. However, we should not compute the cost of duplicate sequences.
3764 // For example, if we have a build vector (i.e., insertelement sequence)
3765 // that is used by more than one vector instruction, we only need to
3766 // compute the cost of the insertelement instructions once. The redundant
3767 // instructions will be eliminated by CSE.
3768 //
3769 // We should consider not creating duplicate tree entries for gather
3770 // sequences, and instead add additional edges to the tree representing
3771 // their uses. Since such an approach results in fewer total entries,
3772 // existing heuristics based on tree size may yield different results.
3773 //
3774 if (TE.State == TreeEntry::NeedToGather &&
3775 std::any_of(std::next(VectorizableTree.begin(), I + 1),
3776 VectorizableTree.end(),
3777 [TE](const std::unique_ptr<TreeEntry> &EntryPtr) {
3778 return EntryPtr->State == TreeEntry::NeedToGather &&
3779 EntryPtr->isSame(TE.Scalars);
3780 }))
3781 continue;
3782
3783 int C = getEntryCost(&TE);
3784 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
3785 << " for bundle that starts with " << *TE.Scalars[0]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
3786 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
3787 Cost += C;
3788 }
3789
3790 SmallPtrSet<Value *, 16> ExtractCostCalculated;
3791 int ExtractCost = 0;
3792 for (ExternalUser &EU : ExternalUses) {
3793 // We only add extract cost once for the same scalar.
3794 if (!ExtractCostCalculated.insert(EU.Scalar).second)
3795 continue;
3796
3797 // Uses by ephemeral values are free (because the ephemeral value will be
3798 // removed prior to code generation, and so the extraction will be
3799 // removed as well).
3800 if (EphValues.count(EU.User))
3801 continue;
3802
3803 // If we plan to rewrite the tree in a smaller type, we will need to sign
3804 // extend the extracted value back to the original type. Here, we account
3805 // for the extract and the added cost of the sign extend if needed.
3806 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
3807 auto *ScalarRoot = VectorizableTree[0]->Scalars[0];
3808 if (MinBWs.count(ScalarRoot)) {
3809 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3810 auto Extend =
3811 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
3812 VecTy = VectorType::get(MinTy, BundleWidth);
3813 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
3814 VecTy, EU.Lane);
3815 } else {
3816 ExtractCost +=
3817 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
3818 }
3819 }
3820
3821 int SpillCost = getSpillCost();
3822 Cost += SpillCost + ExtractCost;
3823
3824 std::string Str;
3825 {
3826 raw_string_ostream OS(Str);
3827 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
3828 << "SLP: Extract Cost = " << ExtractCost << ".\n"
3829 << "SLP: Total Cost = " << Cost << ".\n";
3830 }
3831 LLVM_DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
3832
3833 if (ViewSLPTree)
3834 ViewGraph(this, "SLP" + F->getName(), false, Str);
3835
3836 return Cost;
3837}
3838
3839int BoUpSLP::getGatherCost(Type *Ty,
3840 const DenseSet<unsigned> &ShuffledIndices) const {
3841 int Cost = 0;
3842 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
3843 if (!ShuffledIndices.count(i))
3844 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
3845 if (!ShuffledIndices.empty())
3846 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
3847 return Cost;
3848}
3849
3850int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) const {
3851 // Find the type of the operands in VL.
3852 Type *ScalarTy = VL[0]->getType();
3853 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
3854 ScalarTy = SI->getValueOperand()->getType();
3855 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3856 // Find the cost of inserting/extracting values from the vector.
3857 // Check if the same elements are inserted several times and count them as
3858 // shuffle candidates.
3859 DenseSet<unsigned> ShuffledElements;
3860 DenseSet<Value *> UniqueElements;
3861 // Iterate in reverse order to consider insert elements with the high cost.
3862 for (unsigned I = VL.size(); I > 0; --I) {
3863 unsigned Idx = I - 1;
3864 if (!UniqueElements.insert(VL[Idx]).second)
3865 ShuffledElements.insert(Idx);
3866 }
3867 return getGatherCost(VecTy, ShuffledElements);
3868}
3869
3870// Perform operand reordering on the instructions in VL and return the reordered
3871// operands in Left and Right.
3872void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
3873 SmallVectorImpl<Value *> &Left,
3874 SmallVectorImpl<Value *> &Right,
3875 const DataLayout &DL,
3876 ScalarEvolution &SE,
3877 const BoUpSLP &R) {
3878 if (VL.empty())
3879 return;
3880 VLOperands Ops(VL, DL, SE, R);
3881 // Reorder the operands in place.
3882 Ops.reorder();
3883 Left = Ops.getVL(0);
3884 Right = Ops.getVL(1);
3885}
3886
3887void BoUpSLP::setInsertPointAfterBundle(TreeEntry *E) {
3888 // Get the basic block this bundle is in. All instructions in the bundle
3889 // should be in this block.
3890 auto *Front = E->getMainOp();
3891 auto *BB = Front->getParent();
3892 assert(llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()),((llvm::all_of(make_range(E->Scalars.begin(), E->Scalars
.end()), [=](Value *V) -> bool { auto *I = cast<Instruction
>(V); return !E->isOpcodeOrAlt(I) || I->getParent() ==
BB; })) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3896, __PRETTY_FUNCTION__))
3893 [=](Value *V) -> bool {((llvm::all_of(make_range(E->Scalars.begin(), E->Scalars
.end()), [=](Value *V) -> bool { auto *I = cast<Instruction
>(V); return !E->isOpcodeOrAlt(I) || I->getParent() ==
BB; })) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3896, __PRETTY_FUNCTION__))
3894 auto *I = cast<Instruction>(V);((llvm::all_of(make_range(E->Scalars.begin(), E->Scalars
.end()), [=](Value *V) -> bool { auto *I = cast<Instruction
>(V); return !E->isOpcodeOrAlt(I) || I->getParent() ==
BB; })) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3896, __PRETTY_FUNCTION__))
3895 return !E->isOpcodeOrAlt(I) || I->getParent() == BB;((llvm::all_of(make_range(E->Scalars.begin(), E->Scalars
.end()), [=](Value *V) -> bool { auto *I = cast<Instruction
>(V); return !E->isOpcodeOrAlt(I) || I->getParent() ==
BB; })) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3896, __PRETTY_FUNCTION__))
3896 }))((llvm::all_of(make_range(E->Scalars.begin(), E->Scalars
.end()), [=](Value *V) -> bool { auto *I = cast<Instruction
>(V); return !E->isOpcodeOrAlt(I) || I->getParent() ==
BB; })) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(make_range(E->Scalars.begin(), E->Scalars.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3896, __PRETTY_FUNCTION__))
;
3897
3898 // The last instruction in the bundle in program order.
3899 Instruction *LastInst = nullptr;
3900
3901 // Find the last instruction. The common case should be that BB has been
3902 // scheduled, and the last instruction is VL.back(). So we start with
3903 // VL.back() and iterate over schedule data until we reach the end of the
3904 // bundle. The end of the bundle is marked by null ScheduleData.
3905 if (BlocksSchedules.count(BB)) {
3906 auto *Bundle =
3907 BlocksSchedules[BB]->getScheduleData(E->isOneOf(E->Scalars.back()));
3908 if (Bundle && Bundle->isPartOfBundle())
3909 for (; Bundle; Bundle = Bundle->NextInBundle)
3910 if (Bundle->OpValue == Bundle->Inst)
3911 LastInst = Bundle->Inst;
3912 }
3913
3914 // LastInst can still be null at this point if there's either not an entry
3915 // for BB in BlocksSchedules or there's no ScheduleData available for
3916 // VL.back(). This can be the case if buildTree_rec aborts for various
3917 // reasons (e.g., the maximum recursion depth is reached, the maximum region
3918 // size is reached, etc.). ScheduleData is initialized in the scheduling
3919 // "dry-run".
3920 //
3921 // If this happens, we can still find the last instruction by brute force. We
3922 // iterate forwards from Front (inclusive) until we either see all
3923 // instructions in the bundle or reach the end of the block. If Front is the
3924 // last instruction in program order, LastInst will be set to Front, and we
3925 // will visit all the remaining instructions in the block.
3926 //
3927 // One of the reasons we exit early from buildTree_rec is to place an upper
3928 // bound on compile-time. Thus, taking an additional compile-time hit here is
3929 // not ideal. However, this should be exceedingly rare since it requires that
3930 // we both exit early from buildTree_rec and that the bundle be out-of-order
3931 // (causing us to iterate all the way to the end of the block).
3932 if (!LastInst) {
3933 SmallPtrSet<Value *, 16> Bundle(E->Scalars.begin(), E->Scalars.end());
3934 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
3935 if (Bundle.erase(&I) && E->isOpcodeOrAlt(&I))
3936 LastInst = &I;
3937 if (Bundle.empty())
3938 break;
3939 }
3940 }
3941 assert(LastInst && "Failed to find last instruction in bundle")((LastInst && "Failed to find last instruction in bundle"
) ? static_cast<void> (0) : __assert_fail ("LastInst && \"Failed to find last instruction in bundle\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3941, __PRETTY_FUNCTION__))
;
3942
3943 // Set the insertion point after the last instruction in the bundle. Set the
3944 // debug location to Front.
3945 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
3946 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
3947}
3948
3949Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
3950 Value *Vec = UndefValue::get(Ty);
3951 // Generate the 'InsertElement' instruction.
3952 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
3953 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
3954 if (auto *Insrt = dyn_cast<InsertElementInst>(Vec)) {
3955 GatherSeq.insert(Insrt);
3956 CSEBlocks.insert(Insrt->getParent());
3957
3958 // Add to our 'need-to-extract' list.
3959 if (TreeEntry *E = getTreeEntry(VL[i])) {
3960 // Find which lane we need to extract.
3961 int FoundLane = -1;
3962 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
3963 // Is this the lane of the scalar that we are looking for ?
3964 if (E->Scalars[Lane] == VL[i]) {
3965 FoundLane = Lane;
3966 break;
3967 }
3968 }
3969 assert(FoundLane >= 0 && "Could not find the correct lane")((FoundLane >= 0 && "Could not find the correct lane"
) ? static_cast<void> (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3969, __PRETTY_FUNCTION__))
;
3970 if (!E->ReuseShuffleIndices.empty()) {
3971 FoundLane =
3972 std::distance(E->ReuseShuffleIndices.begin(),
3973 llvm::find(E->ReuseShuffleIndices, FoundLane));
3974 }
3975 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
3976 }
3977 }
3978 }
3979
3980 return Vec;
3981}
3982
3983Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
3984 InstructionsState S = getSameOpcode(VL);
3985 if (S.getOpcode()) {
3986 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
3987 if (E->isSame(VL)) {
3988 Value *V = vectorizeTree(E);
3989 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
3990 // We need to get the vectorized value but without shuffle.
3991 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
3992 V = SV->getOperand(0);
3993 } else {
3994 // Reshuffle to get only unique values.
3995 SmallVector<unsigned, 4> UniqueIdxs;
3996 SmallSet<unsigned, 4> UsedIdxs;
3997 for(unsigned Idx : E->ReuseShuffleIndices)
3998 if (UsedIdxs.insert(Idx).second)
3999 UniqueIdxs.emplace_back(Idx);
4000 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
4001 UniqueIdxs);
4002 }
4003 }
4004 return V;
4005 }
4006 }
4007 }
4008
4009 Type *ScalarTy = S.OpValue->getType();
4010 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
4011 ScalarTy = SI->getValueOperand()->getType();
4012
4013 // Check that every instruction appears once in this bundle.
4014 SmallVector<unsigned, 4> ReuseShuffleIndicies;
4015 SmallVector<Value *, 4> UniqueValues;
4016 if (VL.size() > 2) {
4017 DenseMap<Value *, unsigned> UniquePositions;
4018 for (Value *V : VL) {
4019 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
4020 ReuseShuffleIndicies.emplace_back(Res.first->second);
4021 if (Res.second || isa<Constant>(V))
4022 UniqueValues.emplace_back(V);
4023 }
4024 // Do not shuffle single element or if number of unique values is not power
4025 // of 2.
4026 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
4027 !llvm::isPowerOf2_32(UniqueValues.size()))
4028 ReuseShuffleIndicies.clear();
4029 else
4030 VL = UniqueValues;
4031 }
4032 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
4033
4034 Value *V = Gather(VL, VecTy);
4035 if (!ReuseShuffleIndicies.empty()) {
4036 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4037 ReuseShuffleIndicies, "shuffle");
4038 if (auto *I = dyn_cast<Instruction>(V)) {
4039 GatherSeq.insert(I);
4040 CSEBlocks.insert(I->getParent());
4041 }
4042 }
4043 return V;
4044}
4045
4046static void inversePermutation(ArrayRef<unsigned> Indices,
4047 SmallVectorImpl<unsigned> &Mask) {
4048 Mask.clear();
4049 const unsigned E = Indices.size();
4050 Mask.resize(E);
4051 for (unsigned I = 0; I < E; ++I)
4052 Mask[Indices[I]] = I;
4053}
4054
4055Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
4056 IRBuilder<>::InsertPointGuard Guard(Builder);
4057
4058 if (E->VectorizedValue) {
4059 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
4060 return E->VectorizedValue;
4061 }
4062
4063 Instruction *VL0 = E->getMainOp();
4064 Type *ScalarTy = VL0->getType();
4065 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
4066 ScalarTy = SI->getValueOperand()->getType();
4067 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
4068
4069 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
4070
4071 if (E->State == TreeEntry::NeedToGather) {
4072 setInsertPointAfterBundle(E);
4073 auto *V = Gather(E->Scalars, VecTy);
4074 if (NeedToShuffleReuses) {
4075 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4076 E->ReuseShuffleIndices, "shuffle");
4077 if (auto *I = dyn_cast<Instruction>(V)) {
4078 GatherSeq.insert(I);
4079 CSEBlocks.insert(I->getParent());
4080 }
4081 }
4082 E->VectorizedValue = V;
4083 return V;
4084 }
4085
4086 assert(E->State == TreeEntry::Vectorize && "Unhandled state")((E->State == TreeEntry::Vectorize && "Unhandled state"
) ? static_cast<void> (0) : __assert_fail ("E->State == TreeEntry::Vectorize && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4086, __PRETTY_FUNCTION__))
;
4087 unsigned ShuffleOrOp =
4088 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
4089 switch (ShuffleOrOp) {
4090 case Instruction::PHI: {
4091 auto *PH = cast<PHINode>(VL0);
4092 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
4093 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
4094 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
4095 Value *V = NewPhi;
4096 if (NeedToShuffleReuses) {
4097 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4098 E->ReuseShuffleIndices, "shuffle");
4099 }
4100 E->VectorizedValue = V;
4101
4102 // PHINodes may have multiple entries from the same block. We want to
4103 // visit every block once.
4104 SmallPtrSet<BasicBlock*, 4> VisitedBBs;
4105
4106 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
4107 ValueList Operands;
4108 BasicBlock *IBB = PH->getIncomingBlock(i);
4109
4110 if (!VisitedBBs.insert(IBB).second) {
4111 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
4112 continue;
4113 }
4114
4115 Builder.SetInsertPoint(IBB->getTerminator());
4116 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
4117 Value *Vec = vectorizeTree(E->getOperand(i));
4118 NewPhi->addIncoming(Vec, IBB);
4119 }
4120
4121 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4122, __PRETTY_FUNCTION__))
4122 "Invalid number of incoming values")((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4122, __PRETTY_FUNCTION__))
;
4123 return V;
4124 }
4125
4126 case Instruction::ExtractElement: {
4127 Value *V = E->getSingleOperand(0);
4128 if (!E->ReorderIndices.empty()) {
4129 OrdersType Mask;
4130 inversePermutation(E->ReorderIndices, Mask);
4131 Builder.SetInsertPoint(VL0);
4132 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
4133 "reorder_shuffle");
4134 }
4135 if (NeedToShuffleReuses) {
4136 // TODO: Merge this shuffle with the ReorderShuffleMask.
4137 if (E->ReorderIndices.empty())
4138 Builder.SetInsertPoint(VL0);
4139 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4140 E->ReuseShuffleIndices, "shuffle");
4141 }
4142 E->VectorizedValue = V;
4143 return V;
4144 }
4145 case Instruction::ExtractValue: {
4146 LoadInst *LI = cast<LoadInst>(E->getSingleOperand(0));
4147 Builder.SetInsertPoint(LI);
4148 PointerType *PtrTy =
4149 PointerType::get(VecTy, LI->getPointerAddressSpace());
4150 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
4151 LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlign());
4152 Value *NewV = propagateMetadata(V, E->Scalars);
4153 if (!E->ReorderIndices.empty()) {
4154 OrdersType Mask;
4155 inversePermutation(E->ReorderIndices, Mask);
4156 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
4157 "reorder_shuffle");
4158 }
4159 if (NeedToShuffleReuses) {
4160 // TODO: Merge this shuffle with the ReorderShuffleMask.
4161 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy),
4162 E->ReuseShuffleIndices, "shuffle");
4163 }
4164 E->VectorizedValue = NewV;
4165 return NewV;
4166 }
4167 case Instruction::ZExt:
4168 case Instruction::SExt:
4169 case Instruction::FPToUI:
4170 case Instruction::FPToSI:
4171 case Instruction::FPExt:
4172 case Instruction::PtrToInt:
4173 case Instruction::IntToPtr:
4174 case Instruction::SIToFP:
4175 case Instruction::UIToFP:
4176 case Instruction::Trunc:
4177 case Instruction::FPTrunc:
4178 case Instruction::BitCast: {
4179 setInsertPointAfterBundle(E);
4180
4181 Value *InVec = vectorizeTree(E->getOperand(0));
4182
4183 if (E->VectorizedValue) {
4184 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
4185 return E->VectorizedValue;
4186 }
4187
4188 auto *CI = cast<CastInst>(VL0);
4189 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
4190 if (NeedToShuffleReuses) {
4191 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4192 E->ReuseShuffleIndices, "shuffle");
4193 }
4194 E->VectorizedValue = V;
4195 ++NumVectorInstructions;
4196 return V;
4197 }
4198 case Instruction::FCmp:
4199 case Instruction::ICmp: {
4200 setInsertPointAfterBundle(E);
4201
4202 Value *L = vectorizeTree(E->getOperand(0));
4203 Value *R = vectorizeTree(E->getOperand(1));
4204
4205 if (E->VectorizedValue) {
4206 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
4207 return E->VectorizedValue;
4208 }
4209
4210 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
4211 Value *V;
4212 if (E->getOpcode() == Instruction::FCmp)
4213 V = Builder.CreateFCmp(P0, L, R);
4214 else
4215 V = Builder.CreateICmp(P0, L, R);
4216
4217 propagateIRFlags(V, E->Scalars, VL0);
4218 if (NeedToShuffleReuses) {
4219 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4220 E->ReuseShuffleIndices, "shuffle");
4221 }
4222 E->VectorizedValue = V;
4223 ++NumVectorInstructions;
4224 return V;
4225 }
4226 case Instruction::Select: {
4227 setInsertPointAfterBundle(E);
4228
4229 Value *Cond = vectorizeTree(E->getOperand(0));
4230 Value *True = vectorizeTree(E->getOperand(1));
4231 Value *False = vectorizeTree(E->getOperand(2));
4232
4233 if (E->VectorizedValue) {
4234 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
4235 return E->VectorizedValue;
4236 }
4237
4238 Value *V = Builder.CreateSelect(Cond, True, False);
4239 if (NeedToShuffleReuses) {
4240 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4241 E->ReuseShuffleIndices, "shuffle");
4242 }
4243 E->VectorizedValue = V;
4244 ++NumVectorInstructions;
4245 return V;
4246 }
4247 case Instruction::FNeg: {
4248 setInsertPointAfterBundle(E);
4249
4250 Value *Op = vectorizeTree(E->getOperand(0));
4251
4252 if (E->VectorizedValue) {
4253 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
4254 return E->VectorizedValue;
4255 }
4256
4257 Value *V = Builder.CreateUnOp(
4258 static_cast<Instruction::UnaryOps>(E->getOpcode()), Op);
4259 propagateIRFlags(V, E->Scalars, VL0);
4260 if (auto *I = dyn_cast<Instruction>(V))
4261 V = propagateMetadata(I, E->Scalars);
4262
4263 if (NeedToShuffleReuses) {
4264 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4265 E->ReuseShuffleIndices, "shuffle");
4266 }
4267 E->VectorizedValue = V;
4268 ++NumVectorInstructions;
4269
4270 return V;
4271 }
4272 case Instruction::Add:
4273 case Instruction::FAdd:
4274 case Instruction::Sub:
4275 case Instruction::FSub:
4276 case Instruction::Mul:
4277 case Instruction::FMul:
4278 case Instruction::UDiv:
4279 case Instruction::SDiv:
4280 case Instruction::FDiv:
4281 case Instruction::URem:
4282 case Instruction::SRem:
4283 case Instruction::FRem:
4284 case Instruction::Shl:
4285 case Instruction::LShr:
4286 case Instruction::AShr:
4287 case Instruction::And:
4288 case Instruction::Or:
4289 case Instruction::Xor: {
4290 setInsertPointAfterBundle(E);
4291
4292 Value *LHS = vectorizeTree(E->getOperand(0));
4293 Value *RHS = vectorizeTree(E->getOperand(1));
4294
4295 if (E->VectorizedValue) {
4296 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
4297 return E->VectorizedValue;
4298 }
4299
4300 Value *V = Builder.CreateBinOp(
4301 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS,
4302 RHS);
4303 propagateIRFlags(V, E->Scalars, VL0);
4304 if (auto *I = dyn_cast<Instruction>(V))
4305 V = propagateMetadata(I, E->Scalars);
4306
4307 if (NeedToShuffleReuses) {
4308 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4309 E->ReuseShuffleIndices, "shuffle");
4310 }
4311 E->VectorizedValue = V;
4312 ++NumVectorInstructions;
4313
4314 return V;
4315 }
4316 case Instruction::Load: {
4317 // Loads are inserted at the head of the tree because we don't want to
4318 // sink them all the way down past store instructions.
4319 bool IsReorder = E->updateStateIfReorder();
4320 if (IsReorder)
4321 VL0 = E->getMainOp();
4322 setInsertPointAfterBundle(E);
4323
4324 LoadInst *LI = cast<LoadInst>(VL0);
4325 Type *ScalarLoadTy = LI->getType();
4326 unsigned AS = LI->getPointerAddressSpace();
4327
4328 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
4329 VecTy->getPointerTo(AS));
4330
4331 // The pointer operand uses an in-tree scalar so we add the new BitCast to
4332 // ExternalUses list to make sure that an extract will be generated in the
4333 // future.
4334 Value *PO = LI->getPointerOperand();
4335 if (getTreeEntry(PO))
4336 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
4337
4338 MaybeAlign Alignment = MaybeAlign(LI->getAlignment());
4339 LI = Builder.CreateLoad(VecTy, VecPtr);
4340 if (!Alignment)
4341 Alignment = MaybeAlign(DL->getABITypeAlignment(ScalarLoadTy));
4342 LI->setAlignment(Alignment);
4343 Value *V = propagateMetadata(LI, E->Scalars);
4344 if (IsReorder) {
4345 OrdersType Mask;
4346 inversePermutation(E->ReorderIndices, Mask);
4347 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
4348 Mask, "reorder_shuffle");
4349 }
4350 if (NeedToShuffleReuses) {
4351 // TODO: Merge this shuffle with the ReorderShuffleMask.
4352 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4353 E->ReuseShuffleIndices, "shuffle");
4354 }
4355 E->VectorizedValue = V;
4356 ++NumVectorInstructions;
4357 return V;
4358 }
4359 case Instruction::Store: {
4360 bool IsReorder = !E->ReorderIndices.empty();
4361 auto *SI = cast<StoreInst>(
4362 IsReorder ? E->Scalars[E->ReorderIndices.front()] : VL0);
4363 unsigned Alignment = SI->getAlignment();
4364 unsigned AS = SI->getPointerAddressSpace();
4365
4366 setInsertPointAfterBundle(E);
4367
4368 Value *VecValue = vectorizeTree(E->getOperand(0));
4369 if (IsReorder) {
4370 OrdersType Mask;
4371 inversePermutation(E->ReorderIndices, Mask);
4372 VecValue = Builder.CreateShuffleVector(
4373 VecValue, UndefValue::get(VecValue->getType()), E->ReorderIndices,
4374 "reorder_shuffle");
4375 }
4376 Value *ScalarPtr = SI->getPointerOperand();
4377 Value *VecPtr = Builder.CreateBitCast(
4378 ScalarPtr, VecValue->getType()->getPointerTo(AS));
4379 StoreInst *ST = Builder.CreateStore(VecValue, VecPtr);
4380
4381 // The pointer operand uses an in-tree scalar, so add the new BitCast to
4382 // ExternalUses to make sure that an extract will be generated in the
4383 // future.
4384 if (getTreeEntry(ScalarPtr))
4385 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
4386
4387 if (!Alignment)
4388 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
4389
4390 ST->setAlignment(Align(Alignment));
4391 Value *V = propagateMetadata(ST, E->Scalars);
4392 if (NeedToShuffleReuses) {
4393 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4394 E->ReuseShuffleIndices, "shuffle");
4395 }
4396 E->VectorizedValue = V;
4397 ++NumVectorInstructions;
4398 return V;
4399 }
4400 case Instruction::GetElementPtr: {
4401 setInsertPointAfterBundle(E);
4402
4403 Value *Op0 = vectorizeTree(E->getOperand(0));
4404
4405 std::vector<Value *> OpVecs;
4406 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
4407 ++j) {
4408 ValueList &VL = E->getOperand(j);
4409 // Need to cast all elements to the same type before vectorization to
4410 // avoid crash.
4411 Type *VL0Ty = VL0->getOperand(j)->getType();
4412 Type *Ty = llvm::all_of(
4413 VL, [VL0Ty](Value *V) { return VL0Ty == V->getType(); })
4414 ? VL0Ty
4415 : DL->getIndexType(cast<GetElementPtrInst>(VL0)
4416 ->getPointerOperandType()
4417 ->getScalarType());
4418 for (Value *&V : VL) {
4419 auto *CI = cast<ConstantInt>(V);
4420 V = ConstantExpr::getIntegerCast(CI, Ty,
4421 CI->getValue().isSignBitSet());
4422 }
4423 Value *OpVec = vectorizeTree(VL);
4424 OpVecs.push_back(OpVec);
4425 }
4426
4427 Value *V = Builder.CreateGEP(
4428 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
4429 if (Instruction *I = dyn_cast<Instruction>(V))
4430 V = propagateMetadata(I, E->Scalars);
4431
4432 if (NeedToShuffleReuses) {
4433 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4434 E->ReuseShuffleIndices, "shuffle");
4435 }
4436 E->VectorizedValue = V;
4437 ++NumVectorInstructions;
4438
4439 return V;
4440 }
4441 case Instruction::Call: {
4442 CallInst *CI = cast<CallInst>(VL0);
4443 setInsertPointAfterBundle(E);
4444
4445 Intrinsic::ID IID = Intrinsic::not_intrinsic;
4446 if (Function *FI = CI->getCalledFunction())
4447 IID = FI->getIntrinsicID();
4448
4449 Value *ScalarArg = nullptr;
4450 std::vector<Value *> OpVecs;
4451 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
4452 ValueList OpVL;
4453 // Some intrinsics have scalar arguments. This argument should not be
4454 // vectorized.
4455 if (hasVectorInstrinsicScalarOpd(IID, j)) {
4456 CallInst *CEI = cast<CallInst>(VL0);
4457 ScalarArg = CEI->getArgOperand(j);
4458 OpVecs.push_back(CEI->getArgOperand(j));
4459 continue;
4460 }
4461
4462 Value *OpVec = vectorizeTree(E->getOperand(j));
4463 LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
4464 OpVecs.push_back(OpVec);
4465 }
4466
4467 Module *M = F->getParent();
4468 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4469 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
4470 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
4471 SmallVector<OperandBundleDef, 1> OpBundles;
4472 CI->getOperandBundlesAsDefs(OpBundles);
4473 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
4474
4475 // The scalar argument uses an in-tree scalar so we add the new vectorized
4476 // call to ExternalUses list to make sure that an extract will be
4477 // generated in the future.
4478 if (ScalarArg && getTreeEntry(ScalarArg))
4479 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
4480
4481 propagateIRFlags(V, E->Scalars, VL0);
4482 if (NeedToShuffleReuses) {
4483 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4484 E->ReuseShuffleIndices, "shuffle");
4485 }
4486 E->VectorizedValue = V;
4487 ++NumVectorInstructions;
4488 return V;
4489 }
4490 case Instruction::ShuffleVector: {
4491 assert(E->isAltShuffle() &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
4492 ((Instruction::isBinaryOp(E->getOpcode()) &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
4493 Instruction::isBinaryOp(E->getAltOpcode())) ||((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
4494 (Instruction::isCast(E->getOpcode()) &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
4495 Instruction::isCast(E->getAltOpcode()))) &&((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
4496 "Invalid Shuffle Vector Operand")((E->isAltShuffle() && ((Instruction::isBinaryOp(E
->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode
())) || (Instruction::isCast(E->getOpcode()) && Instruction
::isCast(E->getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
;
4497
4498 Value *LHS = nullptr, *RHS = nullptr;
4499 if (Instruction::isBinaryOp(E->getOpcode())) {
4500 setInsertPointAfterBundle(E);
4501 LHS = vectorizeTree(E->getOperand(0));
4502 RHS = vectorizeTree(E->getOperand(1));
4503 } else {
4504 setInsertPointAfterBundle(E);
4505 LHS = vectorizeTree(E->getOperand(0));
4506 }
4507
4508 if (E->VectorizedValue) {
4509 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
4510 return E->VectorizedValue;
4511 }
4512
4513 Value *V0, *V1;
4514 if (Instruction::isBinaryOp(E->getOpcode())) {
4515 V0 = Builder.CreateBinOp(
4516 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS);
4517 V1 = Builder.CreateBinOp(
4518 static_cast<Instruction::BinaryOps>(E->getAltOpcode()), LHS, RHS);
4519 } else {
4520 V0 = Builder.CreateCast(
4521 static_cast<Instruction::CastOps>(E->getOpcode()), LHS, VecTy);
4522 V1 = Builder.CreateCast(
4523 static_cast<Instruction::CastOps>(E->getAltOpcode()), LHS, VecTy);
4524 }
4525
4526 // Create shuffle to take alternate operations from the vector.
4527 // Also, gather up main and alt scalar ops to propagate IR flags to
4528 // each vector operation.
4529 ValueList OpScalars, AltScalars;
4530 unsigned e = E->Scalars.size();
4531 SmallVector<Constant *, 8> Mask(e);
4532 for (unsigned i = 0; i < e; ++i) {
4533 auto *OpInst = cast<Instruction>(E->Scalars[i]);
4534 assert(E->isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode")((E->isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode"
) ? static_cast<void> (0) : __assert_fail ("E->isOpcodeOrAlt(OpInst) && \"Unexpected main/alternate opcode\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4534, __PRETTY_FUNCTION__))
;
4535 if (OpInst->getOpcode() == E->getAltOpcode()) {
4536 Mask[i] = Builder.getInt32(e + i);
4537 AltScalars.push_back(E->Scalars[i]);
4538 } else {
4539 Mask[i] = Builder.getInt32(i);
4540 OpScalars.push_back(E->Scalars[i]);
4541 }
4542 }
4543
4544 Value *ShuffleMask = ConstantVector::get(Mask);
4545 propagateIRFlags(V0, OpScalars);
4546 propagateIRFlags(V1, AltScalars);
4547
4548 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
4549 if (Instruction *I = dyn_cast<Instruction>(V))
4550 V = propagateMetadata(I, E->Scalars);
4551 if (NeedToShuffleReuses) {
4552 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
4553 E->ReuseShuffleIndices, "shuffle");
4554 }
4555 E->VectorizedValue = V;
4556 ++NumVectorInstructions;
4557
4558 return V;
4559 }
4560 default:
4561 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4561)
;
4562 }
4563 return nullptr;
4564}
4565
4566Value *BoUpSLP::vectorizeTree() {
4567 ExtraValueToDebugLocsMap ExternallyUsedValues;
4568 return vectorizeTree(ExternallyUsedValues);
1
Calling 'BoUpSLP::vectorizeTree'
4569}
4570
4571Value *
4572BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
4573 // All blocks must be scheduled before any instructions are inserted.
4574 for (auto &BSIter : BlocksSchedules) {
4575 scheduleBlock(BSIter.second.get());
2
Calling 'BoUpSLP::scheduleBlock'
4576 }
4577
4578 Builder.SetInsertPoint(&F->getEntryBlock().front());
4579 auto *VectorRoot = vectorizeTree(VectorizableTree[0].get());
4580
4581 // If the vectorized tree can be rewritten in a smaller type, we truncate the
4582 // vectorized root. InstCombine will then rewrite the entire expression. We
4583 // sign extend the extracted values below.
4584 auto *ScalarRoot = VectorizableTree[0]->Scalars[0];
4585 if (MinBWs.count(ScalarRoot)) {
4586 if (auto *I = dyn_cast<Instruction>(VectorRoot))
4587 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
4588 auto BundleWidth = VectorizableTree[0]->Scalars.size();
4589 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
4590 auto *VecTy = VectorType::get(MinTy, BundleWidth);
4591 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
4592 VectorizableTree[0]->VectorizedValue = Trunc;
4593 }
4594
4595 LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
4596 << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
4597
4598 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
4599 // specified by ScalarType.
4600 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
4601 if (!MinBWs.count(ScalarRoot))
4602 return Ex;
4603 if (MinBWs[ScalarRoot].second)
4604 return Builder.CreateSExt(Ex, ScalarType);
4605 return Builder.CreateZExt(Ex, ScalarType);
4606 };
4607
4608 // Extract all of the elements with the external uses.
4609 for (const auto &ExternalUse : ExternalUses) {
4610 Value *Scalar = ExternalUse.Scalar;
4611 llvm::User *User = ExternalUse.User;
4612
4613 // Skip users that we already RAUW. This happens when one instruction
4614 // has multiple uses of the same value.
4615 if (User && !is_contained(Scalar->users(), User))
4616 continue;
4617 TreeEntry *E = getTreeEntry(Scalar);
4618 assert(E && "Invalid scalar")((E && "Invalid scalar") ? static_cast<void> (0
) : __assert_fail ("E && \"Invalid scalar\"", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4618, __PRETTY_FUNCTION__))
;
4619 assert(E->State == TreeEntry::Vectorize && "Extracting from a gather list")((E->State == TreeEntry::Vectorize && "Extracting from a gather list"
) ? static_cast<void> (0) : __assert_fail ("E->State == TreeEntry::Vectorize && \"Extracting from a gather list\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4619, __PRETTY_FUNCTION__))
;
4620
4621 Value *Vec = E->VectorizedValue;
4622 assert(Vec && "Can't find vectorizable value")((Vec && "Can't find vectorizable value") ? static_cast
<void> (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4622, __PRETTY_FUNCTION__))
;
4623
4624 Value *Lane = Builder.getInt32(ExternalUse.Lane);
4625 // If User == nullptr, the Scalar is used as extra arg. Generate
4626 // ExtractElement instruction and update the record for this scalar in
4627 // ExternallyUsedValues.
4628 if (!User) {
4629 assert(ExternallyUsedValues.count(Scalar) &&((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4631, __PRETTY_FUNCTION__))
4630 "Scalar with nullptr as an external user must be registered in "((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4631, __PRETTY_FUNCTION__))
4631 "ExternallyUsedValues map")((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4631, __PRETTY_FUNCTION__))
;
4632 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
4633 Builder.SetInsertPoint(VecI->getParent(),
4634 std::next(VecI->getIterator()));
4635 } else {
4636 Builder.SetInsertPoint(&F->getEntryBlock().front());
4637 }
4638 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
4639 Ex = extend(ScalarRoot, Ex, Scalar->getType());
4640 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
4641 auto &Locs = ExternallyUsedValues[Scalar];
4642 ExternallyUsedValues.insert({Ex, Locs});
4643 ExternallyUsedValues.erase(Scalar);
4644 // Required to update internally referenced instructions.
4645 Scalar->replaceAllUsesWith(Ex);
4646 continue;
4647 }
4648
4649 // Generate extracts for out-of-tree users.
4650 // Find the insertion point for the extractelement lane.
4651 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
4652 if (PHINode *PH = dyn_cast<PHINode>(User)) {
4653 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
4654 if (PH->getIncomingValue(i) == Scalar) {
4655 Instruction *IncomingTerminator =
4656 PH->getIncomingBlock(i)->getTerminator();
4657 if (isa<CatchSwitchInst>(IncomingTerminator)) {
4658 Builder.SetInsertPoint(VecI->getParent(),
4659 std::next(VecI->getIterator()));
4660 } else {
4661 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
4662 }
4663 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
4664 Ex = extend(ScalarRoot, Ex, Scalar->getType());
4665 CSEBlocks.insert(PH->getIncomingBlock(i));
4666 PH->setOperand(i, Ex);
4667 }
4668 }
4669 } else {
4670 Builder.SetInsertPoint(cast<Instruction>(User));
4671 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
4672 Ex = extend(ScalarRoot, Ex, Scalar->getType());
4673 CSEBlocks.insert(cast<Instruction>(User)->getParent());
4674 User->replaceUsesOfWith(Scalar, Ex);
4675 }
4676 } else {
4677 Builder.SetInsertPoint(&F->getEntryBlock().front());
4678 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
4679 Ex = extend(ScalarRoot, Ex, Scalar->getType());
4680 CSEBlocks.insert(&F->getEntryBlock());
4681 User->replaceUsesOfWith(Scalar, Ex);
4682 }
4683
4684 LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
4685 }
4686
4687 // For each vectorized value:
4688 for (auto &TEPtr : VectorizableTree) {
4689 TreeEntry *Entry = TEPtr.get();
4690
4691 // No need to handle users of gathered values.
4692 if (Entry->State == TreeEntry::NeedToGather)
4693 continue;
4694
4695 assert(Entry->VectorizedValue && "Can't find vectorizable value")((Entry->VectorizedValue && "Can't find vectorizable value"
) ? static_cast<void> (0) : __assert_fail ("Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4695, __PRETTY_FUNCTION__))
;
4696
4697 // For each lane:
4698 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
4699 Value *Scalar = Entry->Scalars[Lane];
4700
4701#ifndef NDEBUG
4702 Type *Ty = Scalar->getType();
4703 if (!Ty->isVoidTy()) {
4704 for (User *U : Scalar->users()) {
4705 LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
4706
4707 // It is legal to delete users in the ignorelist.
4708 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&(((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
"Deleting out-of-tree value") ? static_cast<void> (0) :
__assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Deleting out-of-tree value\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4709, __PRETTY_FUNCTION__))
4709 "Deleting out-of-tree value")(((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
"Deleting out-of-tree value") ? static_cast<void> (0) :
__assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Deleting out-of-tree value\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4709, __PRETTY_FUNCTION__))
;
4710 }
4711 }
4712#endif
4713 LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
4714 eraseInstruction(cast<Instruction>(Scalar));
4715 }
4716 }
4717
4718 Builder.ClearInsertionPoint();
4719
4720 return VectorizableTree[0]->VectorizedValue;
4721}
4722
4723void BoUpSLP::optimizeGatherSequence() {
4724 LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
4725 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
4726 // LICM InsertElementInst sequences.
4727 for (Instruction *I : GatherSeq) {
4728 if (isDeleted(I))
4729 continue;
4730
4731 // Check if this block is inside a loop.
4732 Loop *L = LI->getLoopFor(I->getParent());
4733 if (!L)
4734 continue;
4735
4736 // Check if it has a preheader.
4737 BasicBlock *PreHeader = L->getLoopPreheader();
4738 if (!PreHeader)
4739 continue;
4740
4741 // If the vector or the element that we insert into it are
4742 // instructions that are defined in this basic block then we can't
4743 // hoist this instruction.
4744 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4745 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4746 if (Op0 && L->contains(Op0))
4747 continue;
4748 if (Op1 && L->contains(Op1))
4749 continue;
4750
4751 // We can hoist this instruction. Move it to the pre-header.
4752 I->moveBefore(PreHeader->getTerminator());
4753 }
4754
4755 // Make a list of all reachable blocks in our CSE queue.
4756 SmallVector<const DomTreeNode *, 8> CSEWorkList;
4757 CSEWorkList.reserve(CSEBlocks.size());
4758 for (BasicBlock *BB : CSEBlocks)
4759 if (DomTreeNode *N = DT->getNode(BB)) {
4760 assert(DT->isReachableFromEntry(N))((DT->isReachableFromEntry(N)) ? static_cast<void> (
0) : __assert_fail ("DT->isReachableFromEntry(N)", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4760, __PRETTY_FUNCTION__))
;
4761 CSEWorkList.push_back(N);
4762 }
4763
4764 // Sort blocks by domination. This ensures we visit a block after all blocks
4765 // dominating it are visited.
4766 llvm::stable_sort(CSEWorkList,
4767 [this](const DomTreeNode *A, const DomTreeNode *B) {
4768 return DT->properlyDominates(A, B);
4769 });
4770
4771 // Perform O(N^2) search over the gather sequences and merge identical
4772 // instructions. TODO: We can further optimize this scan if we split the
4773 // instructions into different buckets based on the insert lane.
4774 SmallVector<Instruction *, 16> Visited;
4775 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
4776 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4777, __PRETTY_FUNCTION__))
4777 "Worklist not sorted properly!")(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4777, __PRETTY_FUNCTION__))
;
4778 BasicBlock *BB = (*I)->getBlock();
4779 // For all instructions in blocks containing gather sequences:
4780 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
4781 Instruction *In = &*it++;
4782 if (isDeleted(In))
4783 continue;
4784 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
4785 continue;
4786
4787 // Check if we can replace this instruction with any of the
4788 // visited instructions.
4789 for (Instruction *v : Visited) {
4790 if (In->isIdenticalTo(v) &&
4791 DT->dominates(v->getParent(), In->getParent())) {
4792 In->replaceAllUsesWith(v);
4793 eraseInstruction(In);
4794 In = nullptr;
4795 break;
4796 }
4797 }
4798 if (In) {
4799 assert(!is_contained(Visited, In))((!is_contained(Visited, In)) ? static_cast<void> (0) :
__assert_fail ("!is_contained(Visited, In)", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4799, __PRETTY_FUNCTION__))
;
4800 Visited.push_back(In);
4801 }
4802 }
4803 }
4804 CSEBlocks.clear();
4805 GatherSeq.clear();
4806}
4807
4808// Groups the instructions to a bundle (which is then a single scheduling entity)
4809// and schedules instructions until the bundle gets ready.
4810Optional<BoUpSLP::ScheduleData *>
4811BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
4812 const InstructionsState &S) {
4813 if (isa<PHINode>(S.OpValue))
4814 return nullptr;
4815
4816 // Initialize the instruction bundle.
4817 Instruction *OldScheduleEnd = ScheduleEnd;
4818 ScheduleData *PrevInBundle = nullptr;
4819 ScheduleData *Bundle = nullptr;
4820 bool ReSchedule = false;
4821 LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *S.OpValue
<< "\n"; } } while (false)
;
4822
4823 // Make sure that the scheduling region contains all
4824 // instructions of the bundle.
4825 for (Value *V : VL) {
4826 if (!extendSchedulingRegion(V, S))
4827 return None;
4828 }
4829
4830 for (Value *V : VL) {
4831 ScheduleData *BundleMember = getScheduleData(V);
4832 assert(BundleMember &&((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4833, __PRETTY_FUNCTION__))
4833 "no ScheduleData for bundle member (maybe not in same basic block)")((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4833, __PRETTY_FUNCTION__))
;
4834 if (BundleMember->IsScheduled) {
4835 // A bundle member was scheduled as single instruction before and now
4836 // needs to be scheduled as part of the bundle. We just get rid of the
4837 // existing schedule.
4838 LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
4839 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
4840 ReSchedule = true;
4841 }
4842 assert(BundleMember->isSchedulingEntity() &&((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4843, __PRETTY_FUNCTION__))
4843 "bundle member already part of other bundle")((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4843, __PRETTY_FUNCTION__))
;
4844 if (PrevInBundle) {
4845 PrevInBundle->NextInBundle = BundleMember;
4846 } else {
4847 Bundle = BundleMember;
4848 }
4849 BundleMember->UnscheduledDepsInBundle = 0;
4850 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
4851
4852 // Group the instructions to a bundle.
4853 BundleMember->FirstInBundle = Bundle;
4854 PrevInBundle = BundleMember;
4855 }
4856 if (ScheduleEnd != OldScheduleEnd) {
4857 // The scheduling region got new instructions at the lower end (or it is a
4858 // new region for the first bundle). This makes it necessary to
4859 // recalculate all dependencies.
4860 // It is seldom that this needs to be done a second time after adding the
4861 // initial bundle to the region.
4862 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4863 doForAllOpcodes(I, [](ScheduleData *SD) {
4864 SD->clearDependencies();
4865 });
4866 }
4867 ReSchedule = true;
4868 }
4869 if (ReSchedule) {
4870 resetSchedule();
4871 initialFillReadyList(ReadyInsts);
4872 }
4873 assert(Bundle && "Failed to find schedule bundle")((Bundle && "Failed to find schedule bundle") ? static_cast
<void> (0) : __assert_fail ("Bundle && \"Failed to find schedule bundle\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4873, __PRETTY_FUNCTION__))
;
4874
4875 LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
4876 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
4877
4878 calculateDependencies(Bundle, true, SLP);
4879
4880 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
4881 // means that there are no cyclic dependencies and we can schedule it.
4882 // Note that's important that we don't "schedule" the bundle yet (see
4883 // cancelScheduling).
4884 while (!Bundle->isReady() && !ReadyInsts.empty()) {
4885
4886 ScheduleData *pickedSD = ReadyInsts.back();
4887 ReadyInsts.pop_back();
4888
4889 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
4890 schedule(pickedSD, ReadyInsts);
4891 }
4892 }
4893 if (!Bundle->isReady()) {
4894 cancelScheduling(VL, S.OpValue);
4895 return None;
4896 }
4897 return Bundle;
4898}
4899
4900void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
4901 Value *OpValue) {
4902 if (isa<PHINode>(OpValue))
4903 return;
4904
4905 ScheduleData *Bundle = getScheduleData(OpValue);
4906 LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
4907 assert(!Bundle->IsScheduled &&((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4908, __PRETTY_FUNCTION__))
4908 "Can't cancel bundle which is already scheduled")((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4908, __PRETTY_FUNCTION__))
;
4909 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4910, __PRETTY_FUNCTION__))
4910 "tried to unbundle something which is not a bundle")((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4910, __PRETTY_FUNCTION__))
;
4911
4912 // Un-bundle: make single instructions out of the bundle.
4913 ScheduleData *BundleMember = Bundle;
4914 while (BundleMember) {
4915 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")((BundleMember->FirstInBundle == Bundle && "corrupt bundle links"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4915, __PRETTY_FUNCTION__))
;
4916 BundleMember->FirstInBundle = BundleMember;
4917 ScheduleData *Next = BundleMember->NextInBundle;
4918 BundleMember->NextInBundle = nullptr;
4919 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
4920 if (BundleMember->UnscheduledDepsInBundle == 0) {
4921 ReadyInsts.insert(BundleMember);
4922 }
4923 BundleMember = Next;
4924 }
4925}
4926
4927BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
4928 // Allocate a new ScheduleData for the instruction.
4929 if (ChunkPos >= ChunkSize) {
4930 ScheduleDataChunks.push_back(std::make_unique<ScheduleData[]>(ChunkSize));
4931 ChunkPos = 0;
4932 }
4933 return &(ScheduleDataChunks.back()[ChunkPos++]);
4934}
4935
4936bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
4937 const InstructionsState &S) {
4938 if (getScheduleData(V, isOneOf(S, V)))
4939 return true;
4940 Instruction *I = dyn_cast<Instruction>(V);
4941 assert(I && "bundle member must be an instruction")((I && "bundle member must be an instruction") ? static_cast
<void> (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4941, __PRETTY_FUNCTION__))
;
4942 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")((!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4942, __PRETTY_FUNCTION__))
;
4943 auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool {
4944 ScheduleData *ISD = getScheduleData(I);
4945 if (!ISD)
4946 return false;
4947 assert(isInSchedulingRegion(ISD) &&((isInSchedulingRegion(ISD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4948, __PRETTY_FUNCTION__))
4948 "ScheduleData not in scheduling region")((isInSchedulingRegion(ISD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4948, __PRETTY_FUNCTION__))
;
4949 ScheduleData *SD = allocateScheduleDataChunks();
4950 SD->Inst = I;
4951 SD->init(SchedulingRegionID, S.OpValue);
4952 ExtraScheduleDataMap[I][S.OpValue] = SD;
4953 return true;
4954 };
4955 if (CheckSheduleForI(I))
4956 return true;
4957 if (!ScheduleStart) {
4958 // It's the first instruction in the new region.
4959 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
4960 ScheduleStart = I;
4961 ScheduleEnd = I->getNextNode();
4962 if (isOneOf(S, I) != I)
4963 CheckSheduleForI(I);
4964 assert(ScheduleEnd && "tried to vectorize a terminator?")((ScheduleEnd && "tried to vectorize a terminator?") ?
static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a terminator?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4964, __PRETTY_FUNCTION__))
;
4965 LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
4966 return true;
4967 }
4968 // Search up and down at the same time, because we don't know if the new
4969 // instruction is above or below the existing scheduling region.
4970 BasicBlock::reverse_iterator UpIter =
4971 ++ScheduleStart->getIterator().getReverse();
4972 BasicBlock::reverse_iterator UpperEnd = BB->rend();
4973 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
4974 BasicBlock::iterator LowerEnd = BB->end();
4975 while (true) {
4976 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
4977 LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
4978 return false;
4979 }
4980
4981 if (UpIter != UpperEnd) {
4982 if (&*UpIter == I) {
4983 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
4984 ScheduleStart = I;
4985 if (isOneOf(S, I) != I)
4986 CheckSheduleForI(I);
4987 LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
4988 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
4989 return true;
4990 }
4991 ++UpIter;
4992 }
4993 if (DownIter != LowerEnd) {
4994 if (&*DownIter == I) {
4995 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
4996 nullptr);
4997 ScheduleEnd = I->getNextNode();
4998 if (isOneOf(S, I) != I)
4999 CheckSheduleForI(I);
5000 assert(ScheduleEnd && "tried to vectorize a terminator?")((ScheduleEnd && "tried to vectorize a terminator?") ?
static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a terminator?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5000, __PRETTY_FUNCTION__))
;
5001 LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
5002 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
5003 return true;
5004 }
5005 ++DownIter;
5006 }
5007 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5008, __PRETTY_FUNCTION__))
5008 "instruction not found in block")(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5008, __PRETTY_FUNCTION__))
;
5009 }
5010 return true;
5011}
5012
5013void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
5014 Instruction *ToI,
5015 ScheduleData *PrevLoadStore,
5016 ScheduleData *NextLoadStore) {
5017 ScheduleData *CurrentLoadStore = PrevLoadStore;
5018 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
5019 ScheduleData *SD = ScheduleDataMap[I];
5020 if (!SD) {
5021 SD = allocateScheduleDataChunks();
5022 ScheduleDataMap[I] = SD;
5023 SD->Inst = I;
5024 }
5025 assert(!isInSchedulingRegion(SD) &&((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5026, __PRETTY_FUNCTION__))
5026 "new ScheduleData already in scheduling region")((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5026, __PRETTY_FUNCTION__))
;
5027 SD->init(SchedulingRegionID, I);
5028
5029 if (I->mayReadOrWriteMemory() &&
5030 (!isa<IntrinsicInst>(I) ||
5031 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
5032 // Update the linked list of memory accessing instructions.
5033 if (CurrentLoadStore) {
5034 CurrentLoadStore->NextLoadStore = SD;
5035 } else {
5036 FirstLoadStoreInRegion = SD;
5037 }
5038 CurrentLoadStore = SD;
5039 }
5040 }
5041 if (NextLoadStore) {
5042 if (CurrentLoadStore)
5043 CurrentLoadStore->NextLoadStore = NextLoadStore;
5044 } else {
5045 LastLoadStoreInRegion = CurrentLoadStore;
5046 }
5047}
5048
5049void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
5050 bool InsertInReadyList,
5051 BoUpSLP *SLP) {
5052 assert(SD->isSchedulingEntity())((SD->isSchedulingEntity()) ? static_cast<void> (0) :
__assert_fail ("SD->isSchedulingEntity()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5052, __PRETTY_FUNCTION__))
;
5053
5054 SmallVector<ScheduleData *, 10> WorkList;
5055 WorkList.push_back(SD);
5056
5057 while (!WorkList.empty()) {
5058 ScheduleData *SD = WorkList.back();
5059 WorkList.pop_back();
5060
5061 ScheduleData *BundleMember = SD;
5062 while (BundleMember) {
5063 assert(isInSchedulingRegion(BundleMember))((isInSchedulingRegion(BundleMember)) ? static_cast<void>
(0) : __assert_fail ("isInSchedulingRegion(BundleMember)", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5063, __PRETTY_FUNCTION__))
;
5064 if (!BundleMember->hasValidDependencies()) {
5065
5066 LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
5067 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
5068 BundleMember->Dependencies = 0;
5069 BundleMember->resetUnscheduledDeps();
5070
5071 // Handle def-use chain dependencies.
5072 if (BundleMember->OpValue != BundleMember->Inst) {
5073 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
5074 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
5075 BundleMember->Dependencies++;
5076 ScheduleData *DestBundle = UseSD->FirstInBundle;
5077 if (!DestBundle->IsScheduled)
5078 BundleMember->incrementUnscheduledDeps(1);
5079 if (!DestBundle->hasValidDependencies())
5080 WorkList.push_back(DestBundle);
5081 }
5082 } else {
5083 for (User *U : BundleMember->Inst->users()) {
5084 if (isa<Instruction>(U)) {
5085 ScheduleData *UseSD = getScheduleData(U);
5086 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
5087 BundleMember->Dependencies++;
5088 ScheduleData *DestBundle = UseSD->FirstInBundle;
5089 if (!DestBundle->IsScheduled)
5090 BundleMember->incrementUnscheduledDeps(1);
5091 if (!DestBundle->hasValidDependencies())
5092 WorkList.push_back(DestBundle);
5093 }
5094 } else {
5095 // I'm not sure if this can ever happen. But we need to be safe.
5096 // This lets the instruction/bundle never be scheduled and
5097 // eventually disable vectorization.
5098 BundleMember->Dependencies++;
5099 BundleMember->incrementUnscheduledDeps(1);
5100 }
5101 }
5102 }
5103
5104 // Handle the memory dependencies.
5105 ScheduleData *DepDest = BundleMember->NextLoadStore;
5106 if (DepDest) {
5107 Instruction *SrcInst = BundleMember->Inst;
5108 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
5109 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
5110 unsigned numAliased = 0;
5111 unsigned DistToSrc = 1;
5112
5113 while (DepDest) {
5114 assert(isInSchedulingRegion(DepDest))((isInSchedulingRegion(DepDest)) ? static_cast<void> (0
) : __assert_fail ("isInSchedulingRegion(DepDest)", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5114, __PRETTY_FUNCTION__))
;
5115
5116 // We have two limits to reduce the complexity:
5117 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
5118 // SLP->isAliased (which is the expensive part in this loop).
5119 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
5120 // the whole loop (even if the loop is fast, it's quadratic).
5121 // It's important for the loop break condition (see below) to
5122 // check this limit even between two read-only instructions.
5123 if (DistToSrc >= MaxMemDepDistance ||
5124 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
5125 (numAliased >= AliasedCheckLimit ||
5126 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
5127
5128 // We increment the counter only if the locations are aliased
5129 // (instead of counting all alias checks). This gives a better
5130 // balance between reduced runtime and accurate dependencies.
5131 numAliased++;
5132
5133 DepDest->MemoryDependencies.push_back(BundleMember);
5134 BundleMember->Dependencies++;
5135 ScheduleData *DestBundle = DepDest->FirstInBundle;
5136 if (!DestBundle->IsScheduled) {
5137 BundleMember->incrementUnscheduledDeps(1);
5138 }
5139 if (!DestBundle->hasValidDependencies()) {
5140 WorkList.push_back(DestBundle);
5141 }
5142 }
5143 DepDest = DepDest->NextLoadStore;
5144
5145 // Example, explaining the loop break condition: Let's assume our
5146 // starting instruction is i0 and MaxMemDepDistance = 3.
5147 //
5148 // +--------v--v--v
5149 // i0,i1,i2,i3,i4,i5,i6,i7,i8
5150 // +--------^--^--^
5151 //
5152 // MaxMemDepDistance let us stop alias-checking at i3 and we add
5153 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
5154 // Previously we already added dependencies from i3 to i6,i7,i8
5155 // (because of MaxMemDepDistance). As we added a dependency from
5156 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
5157 // and we can abort this loop at i6.
5158 if (DistToSrc >= 2 * MaxMemDepDistance)
5159 break;
5160 DistToSrc++;
5161 }
5162 }
5163 }
5164 BundleMember = BundleMember->NextInBundle;
5165 }
5166 if (InsertInReadyList && SD->isReady()) {
5167 ReadyInsts.push_back(SD);
5168 LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
5169 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
5170 }
5171 }
5172}
5173
5174void BoUpSLP::BlockScheduling::resetSchedule() {
5175 assert(ScheduleStart &&((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5176, __PRETTY_FUNCTION__))
5176 "tried to reset schedule on block which has not been scheduled")((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5176, __PRETTY_FUNCTION__))
;
5177 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
5178 doForAllOpcodes(I, [&](ScheduleData *SD) {
5179 assert(isInSchedulingRegion(SD) &&((isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5180, __PRETTY_FUNCTION__))
5180 "ScheduleData not in scheduling region")((isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5180, __PRETTY_FUNCTION__))
;
5181 SD->IsScheduled = false;
5182 SD->resetUnscheduledDeps();
5183 });
5184 }
5185 ReadyInsts.clear();
5186}
5187
5188void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
5189 if (!BS->ScheduleStart)
3
Assuming field 'ScheduleStart' is non-null
4
Taking false branch
5190 return;
5191
5192 LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
5
Assuming 'DebugFlag' is false
6
Loop condition is false. Exiting loop
5193
5194 BS->resetSchedule();
5195
5196 // For the real scheduling we use a more sophisticated ready-list: it is
5197 // sorted by the original instruction location. This lets the final schedule
5198 // be as close as possible to the original instruction order.
5199 struct ScheduleDataCompare {
5200 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
5201 return SD2->SchedulingPriority < SD1->SchedulingPriority;
5202 }
5203 };
5204 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
5205
5206 // Ensure that all dependency data is updated and fill the ready-list with
5207 // initial instructions.
5208 int Idx = 0;
5209 int NumToSchedule = 0;
5210 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
7
Assuming 'I' is equal to field 'ScheduleEnd'
8
Loop condition is false. Execution continues on line 5223
5211 I = I->getNextNode()) {
5212 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
5213 assert(SD->isPartOfBundle() ==((SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr
) && "scheduler and vectorizer bundle mismatch") ? static_cast
<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5215, __PRETTY_FUNCTION__))
5214 (getTreeEntry(SD->Inst) != nullptr) &&((SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr
) && "scheduler and vectorizer bundle mismatch") ? static_cast
<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5215, __PRETTY_FUNCTION__))
5215 "scheduler and vectorizer bundle mismatch")((SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr
) && "scheduler and vectorizer bundle mismatch") ? static_cast
<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5215, __PRETTY_FUNCTION__))
;
5216 SD->FirstInBundle->SchedulingPriority = Idx++;
5217 if (SD->isSchedulingEntity()) {
5218 BS->calculateDependencies(SD, false, this);
5219 NumToSchedule++;
5220 }
5221 });
5222 }
5223 BS->initialFillReadyList(ReadyInsts);
5224
5225 Instruction *LastScheduledInst = BS->ScheduleEnd;
5226
5227 // Do the "real" scheduling.
5228 while (!ReadyInsts.empty()) {
9
Assuming the condition is true
10
Loop condition is true. Entering loop body
5229 ScheduleData *picked = *ReadyInsts.begin();
11
'picked' initialized here
5230 ReadyInsts.erase(ReadyInsts.begin());
5231
5232 // Move the scheduled instruction(s) to their dedicated places, if not
5233 // there yet.
5234 ScheduleData *BundleMember = picked;
5235 while (BundleMember) {
12
Assuming pointer value is null
13
Loop condition is false. Execution continues on line 5246
5236 Instruction *pickedInst = BundleMember->Inst;
5237 if (LastScheduledInst->getNextNode() != pickedInst) {
5238 BS->BB->getInstList().remove(pickedInst);
5239 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
5240 pickedInst);
5241 }
5242 LastScheduledInst = pickedInst;
5243 BundleMember = BundleMember->NextInBundle;
5244 }
5245
5246 BS->schedule(picked, ReadyInsts);
14
Passing null pointer value via 1st parameter 'SD'
15
Calling 'BlockScheduling::schedule'
5247 NumToSchedule--;
5248 }
5249 assert(NumToSchedule == 0 && "could not schedule all instructions")((NumToSchedule == 0 && "could not schedule all instructions"
) ? static_cast<void> (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5249, __PRETTY_FUNCTION__))
;
5250
5251 // Avoid duplicate scheduling of the block.
5252 BS->ScheduleStart = nullptr;
5253}
5254
5255unsigned BoUpSLP::getVectorElementSize(Value *V) const {
5256 // If V is a store, just return the width of the stored value without
5257 // traversing the expression tree. This is the common case.
5258 if (auto *Store = dyn_cast<StoreInst>(V))
5259 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
5260
5261 // If V is not a store, we can traverse the expression tree to find loads
5262 // that feed it. The type of the loaded value may indicate a more suitable
5263 // width than V's type. We want to base the vector element size on the width
5264 // of memory operations where possible.
5265 SmallVector<Instruction *, 16> Worklist;
5266 SmallPtrSet<Instruction *, 16> Visited;
5267 if (auto *I = dyn_cast<Instruction>(V))
5268 Worklist.push_back(I);
5269
5270 // Traverse the expression tree in bottom-up order looking for loads. If we
5271 // encounter an instruction we don't yet handle, we give up.
5272 auto MaxWidth = 0u;
5273 auto FoundUnknownInst = false;
5274 while (!Worklist.empty() && !FoundUnknownInst) {
5275 auto *I = Worklist.pop_back_val();
5276 Visited.insert(I);
5277
5278 // We should only be looking at scalar instructions here. If the current
5279 // instruction has a vector type, give up.
5280 auto *Ty = I->getType();
5281 if (isa<VectorType>(Ty))
5282 FoundUnknownInst = true;
5283
5284 // If the current instruction is a load, update MaxWidth to reflect the
5285 // width of the loaded value.
5286 else if (isa<LoadInst>(I))
5287 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
5288
5289 // Otherwise, we need to visit the operands of the instruction. We only
5290 // handle the interesting cases from buildTree here. If an operand is an
5291 // instruction we haven't yet visited, we add it to the worklist.
5292 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
5293 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
5294 for (Use &U : I->operands())
5295 if (auto *J = dyn_cast<Instruction>(U.get()))
5296 if (!Visited.count(J))
5297 Worklist.push_back(J);
5298 }
5299
5300 // If we don't yet handle the instruction, give up.
5301 else
5302 FoundUnknownInst = true;
5303 }
5304
5305 // If we didn't encounter a memory access in the expression tree, or if we
5306 // gave up for some reason, just return the width of V.
5307 if (!MaxWidth || FoundUnknownInst)
5308 return DL->getTypeSizeInBits(V->getType());
5309
5310 // Otherwise, return the maximum width we found.
5311 return MaxWidth;
5312}
5313
5314// Determine if a value V in a vectorizable expression Expr can be demoted to a
5315// smaller type with a truncation. We collect the values that will be demoted
5316// in ToDemote and additional roots that require investigating in Roots.
5317static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
5318 SmallVectorImpl<Value *> &ToDemote,
5319 SmallVectorImpl<Value *> &Roots) {
5320 // We can always demote constants.
5321 if (isa<Constant>(V)) {
5322 ToDemote.push_back(V);
5323 return true;
5324 }
5325
5326 // If the value is not an instruction in the expression with only one use, it
5327 // cannot be demoted.
5328 auto *I = dyn_cast<Instruction>(V);
5329 if (!I || !I->hasOneUse() || !Expr.count(I))
5330 return false;
5331
5332 switch (I->getOpcode()) {
5333
5334 // We can always demote truncations and extensions. Since truncations can
5335 // seed additional demotion, we save the truncated value.
5336 case Instruction::Trunc:
5337 Roots.push_back(I->getOperand(0));
5338 break;
5339 case Instruction::ZExt:
5340 case Instruction::SExt:
5341 break;
5342
5343 // We can demote certain binary operations if we can demote both of their
5344 // operands.
5345 case Instruction::Add:
5346 case Instruction::Sub:
5347 case Instruction::Mul:
5348 case Instruction::And:
5349 case Instruction::Or:
5350 case Instruction::Xor:
5351 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
5352 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
5353 return false;
5354 break;
5355
5356 // We can demote selects if we can demote their true and false values.
5357 case Instruction::Select: {
5358 SelectInst *SI = cast<SelectInst>(I);
5359 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
5360 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
5361 return false;
5362 break;
5363 }
5364
5365 // We can demote phis if we can demote all their incoming operands. Note that
5366 // we don't need to worry about cycles since we ensure single use above.
5367 case Instruction::PHI: {
5368 PHINode *PN = cast<PHINode>(I);
5369 for (Value *IncValue : PN->incoming_values())
5370 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
5371 return false;
5372 break;
5373 }
5374
5375 // Otherwise, conservatively give up.
5376 default:
5377 return false;
5378 }
5379
5380 // Record the value that we can demote.
5381 ToDemote.push_back(V);
5382 return true;
5383}
5384
5385void BoUpSLP::computeMinimumValueSizes() {
5386 // If there are no external uses, the expression tree must be rooted by a
5387 // store. We can't demote in-memory values, so there is nothing to do here.
5388 if (ExternalUses.empty())
5389 return;
5390
5391 // We only attempt to truncate integer expressions.
5392 auto &TreeRoot = VectorizableTree[0]->Scalars;
5393 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
5394 if (!TreeRootIT)
5395 return;
5396
5397 // If the expression is not rooted by a store, these roots should have
5398 // external uses. We will rely on InstCombine to rewrite the expression in
5399 // the narrower type. However, InstCombine only rewrites single-use values.
5400 // This means that if a tree entry other than a root is used externally, it
5401 // must have multiple uses and InstCombine will not rewrite it. The code
5402 // below ensures that only the roots are used externally.
5403 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
5404 for (auto &EU : ExternalUses)
5405 if (!Expr.erase(EU.Scalar))
5406 return;
5407 if (!Expr.empty())
5408 return;
5409
5410 // Collect the scalar values of the vectorizable expression. We will use this
5411 // context to determine which values can be demoted. If we see a truncation,
5412 // we mark it as seeding another demotion.
5413 for (auto &EntryPtr : VectorizableTree)
5414 Expr.insert(EntryPtr->Scalars.begin(), EntryPtr->Scalars.end());
5415
5416 // Ensure the roots of the vectorizable tree don't form a cycle. They must
5417 // have a single external user that is not in the vectorizable tree.
5418 for (auto *Root : TreeRoot)
5419 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
5420 return;
5421
5422 // Conservatively determine if we can actually truncate the roots of the
5423 // expression. Collect the values that can be demoted in ToDemote and
5424 // additional roots that require investigating in Roots.
5425 SmallVector<Value *, 32> ToDemote;
5426 SmallVector<Value *, 4> Roots;
5427 for (auto *Root : TreeRoot)
5428 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
5429 return;
5430
5431 // The maximum bit width required to represent all the values that can be
5432 // demoted without loss of precision. It would be safe to truncate the roots
5433 // of the expression to this width.
5434 auto MaxBitWidth = 8u;
5435
5436 // We first check if all the bits of the roots are demanded. If they're not,
5437 // we can truncate the roots to this narrower type.
5438 for (auto *Root : TreeRoot) {
5439 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
5440 MaxBitWidth = std::max<unsigned>(
5441 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
5442 }
5443
5444 // True if the roots can be zero-extended back to their original type, rather
5445 // than sign-extended. We know that if the leading bits are not demanded, we
5446 // can safely zero-extend. So we initialize IsKnownPositive to True.
5447 bool IsKnownPositive = true;
5448
5449 // If all the bits of the roots are demanded, we can try a little harder to
5450 // compute a narrower type. This can happen, for example, if the roots are
5451 // getelementptr indices. InstCombine promotes these indices to the pointer
5452 // width. Thus, all their bits are technically demanded even though the
5453 // address computation might be vectorized in a smaller type.
5454 //
5455 // We start by looking at each entry that can be demoted. We compute the
5456 // maximum bit width required to store the scalar by using ValueTracking to
5457 // compute the number of high-order bits we can truncate.
5458 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
5459 llvm::all_of(TreeRoot, [](Value *R) {
5460 assert(R->hasOneUse() && "Root should have only one use!")((R->hasOneUse() && "Root should have only one use!"
) ? static_cast<void> (0) : __assert_fail ("R->hasOneUse() && \"Root should have only one use!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5460, __PRETTY_FUNCTION__))
;
5461 return isa<GetElementPtrInst>(R->user_back());
5462 })) {
5463 MaxBitWidth = 8u;
5464
5465 // Determine if the sign bit of all the roots is known to be zero. If not,
5466 // IsKnownPositive is set to False.
5467 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
5468 KnownBits Known = computeKnownBits(R, *DL);
5469 return Known.isNonNegative();
5470 });
5471
5472 // Determine the maximum number of bits required to store the scalar
5473 // values.
5474 for (auto *Scalar : ToDemote) {
5475 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
5476 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
5477 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
5478 }
5479
5480 // If we can't prove that the sign bit is zero, we must add one to the
5481 // maximum bit width to account for the unknown sign bit. This preserves
5482 // the existing sign bit so we can safely sign-extend the root back to the
5483 // original type. Otherwise, if we know the sign bit is zero, we will
5484 // zero-extend the root instead.
5485 //
5486 // FIXME: This is somewhat suboptimal, as there will be cases where adding
5487 // one to the maximum bit width will yield a larger-than-necessary
5488 // type. In general, we need to add an extra bit only if we can't
5489 // prove that the upper bit of the original type is equal to the
5490 // upper bit of the proposed smaller type. If these two bits are the
5491 // same (either zero or one) we know that sign-extending from the
5492 // smaller type will result in the same value. Here, since we can't
5493 // yet prove this, we are just making the proposed smaller type
5494 // larger to ensure correctness.
5495 if (!IsKnownPositive)
5496 ++MaxBitWidth;
5497 }
5498
5499 // Round MaxBitWidth up to the next power-of-two.
5500 if (!isPowerOf2_64(MaxBitWidth))
5501 MaxBitWidth = NextPowerOf2(MaxBitWidth);
5502
5503 // If the maximum bit width we compute is less than the with of the roots'
5504 // type, we can proceed with the narrowing. Otherwise, do nothing.
5505 if (MaxBitWidth >= TreeRootIT->getBitWidth())
5506 return;
5507
5508 // If we can truncate the root, we must collect additional values that might
5509 // be demoted as a result. That is, those seeded by truncations we will
5510 // modify.
5511 while (!Roots.empty())
5512 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
5513
5514 // Finally, map the values we can demote to the maximum bit with we computed.
5515 for (auto *Scalar : ToDemote)
5516 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
5517}
5518
5519namespace {
5520
5521/// The SLPVectorizer Pass.
5522struct SLPVectorizer : public FunctionPass {
5523 SLPVectorizerPass Impl;
5524
5525 /// Pass identification, replacement for typeid
5526 static char ID;
5527
5528 explicit SLPVectorizer() : FunctionPass(ID) {
5529 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
5530 }
5531
5532 bool doInitialization(Module &M) override {
5533 return false;
5534 }
5535
5536 bool runOnFunction(Function &F) override {
5537 if (skipFunction(F))
5538 return false;
5539
5540 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
5541 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
5542 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
5543 auto *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
5544 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
5545 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
5546 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
5547 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
5548 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
5549 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
5550
5551 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
5552 }
5553
5554