Bug Summary

File:llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 6251, column 30
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SLPVectorizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/build-llvm/lib/Transforms/Vectorize -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-06-10-162512-48765-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10// stores that can be put together into vector-stores. Next, it attempts to
11// construct vectorizable tree using the use-def chains. If a profitable tree
12// was found, the SLP vectorizer performs vectorization on the tree.
13//
14// The pass is inspired by the work described in the paper:
15// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/Optional.h"
23#include "llvm/ADT/PostOrderIterator.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SetOperations.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallBitVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/SmallString.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/iterator.h"
33#include "llvm/ADT/iterator_range.h"
34#include "llvm/Analysis/AliasAnalysis.h"
35#include "llvm/Analysis/AssumptionCache.h"
36#include "llvm/Analysis/CodeMetrics.h"
37#include "llvm/Analysis/DemandedBits.h"
38#include "llvm/Analysis/GlobalsModRef.h"
39#include "llvm/Analysis/IVDescriptors.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PatternMatch.h"
69#include "llvm/IR/Type.h"
70#include "llvm/IR/Use.h"
71#include "llvm/IR/User.h"
72#include "llvm/IR/Value.h"
73#include "llvm/IR/ValueHandle.h"
74#include "llvm/IR/Verifier.h"
75#include "llvm/InitializePasses.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/DOTGraphTraits.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/ErrorHandling.h"
83#include "llvm/Support/GraphWriter.h"
84#include "llvm/Support/InstructionCost.h"
85#include "llvm/Support/KnownBits.h"
86#include "llvm/Support/MathExtras.h"
87#include "llvm/Support/raw_ostream.h"
88#include "llvm/Transforms/Utils/InjectTLIMappings.h"
89#include "llvm/Transforms/Utils/LoopUtils.h"
90#include "llvm/Transforms/Vectorize.h"
91#include <algorithm>
92#include <cassert>
93#include <cstdint>
94#include <iterator>
95#include <memory>
96#include <set>
97#include <string>
98#include <tuple>
99#include <utility>
100#include <vector>
101
102using namespace llvm;
103using namespace llvm::PatternMatch;
104using namespace slpvectorizer;
105
106#define SV_NAME"slp-vectorizer" "slp-vectorizer"
107#define DEBUG_TYPE"SLP" "SLP"
108
109STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated"}
;
110
111cl::opt<bool> RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden,
112 cl::desc("Run the SLP vectorization passes"));
113
114static cl::opt<int>
115 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
116 cl::desc("Only vectorize if you gain more than this "
117 "number "));
118
119static cl::opt<bool>
120ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
121 cl::desc("Attempt to vectorize horizontal reductions"));
122
123static cl::opt<bool> ShouldStartVectorizeHorAtStore(
124 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
125 cl::desc(
126 "Attempt to vectorize horizontal reductions feeding into a store"));
127
128static cl::opt<int>
129MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
130 cl::desc("Attempt to vectorize for this register size in bits"));
131
132static cl::opt<unsigned>
133MaxVFOption("slp-max-vf", cl::init(0), cl::Hidden,
134 cl::desc("Maximum SLP vectorization factor (0=unlimited)"));
135
136static cl::opt<int>
137MaxStoreLookup("slp-max-store-lookup", cl::init(32), cl::Hidden,
138 cl::desc("Maximum depth of the lookup for consecutive stores."));
139
140/// Limits the size of scheduling regions in a block.
141/// It avoid long compile times for _very_ large blocks where vector
142/// instructions are spread over a wide range.
143/// This limit is way higher than needed by real-world functions.
144static cl::opt<int>
145ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
146 cl::desc("Limit the size of the SLP scheduling region per block"));
147
148static cl::opt<int> MinVectorRegSizeOption(
149 "slp-min-reg-size", cl::init(128), cl::Hidden,
150 cl::desc("Attempt to vectorize for this register size in bits"));
151
152static cl::opt<unsigned> RecursionMaxDepth(
153 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
154 cl::desc("Limit the recursion depth when building a vectorizable tree"));
155
156static cl::opt<unsigned> MinTreeSize(
157 "slp-min-tree-size", cl::init(3), cl::Hidden,
158 cl::desc("Only vectorize small trees if they are fully vectorizable"));
159
160// The maximum depth that the look-ahead score heuristic will explore.
161// The higher this value, the higher the compilation time overhead.
162static cl::opt<int> LookAheadMaxDepth(
163 "slp-max-look-ahead-depth", cl::init(2), cl::Hidden,
164 cl::desc("The maximum look-ahead depth for operand reordering scores"));
165
166// The Look-ahead heuristic goes through the users of the bundle to calculate
167// the users cost in getExternalUsesCost(). To avoid compilation time increase
168// we limit the number of users visited to this value.
169static cl::opt<unsigned> LookAheadUsersBudget(
170 "slp-look-ahead-users-budget", cl::init(2), cl::Hidden,
171 cl::desc("The maximum number of users to visit while visiting the "
172 "predecessors. This prevents compilation time increase."));
173
174static cl::opt<bool>
175 ViewSLPTree("view-slp-tree", cl::Hidden,
176 cl::desc("Display the SLP trees with Graphviz"));
177
178// Limit the number of alias checks. The limit is chosen so that
179// it has no negative effect on the llvm benchmarks.
180static const unsigned AliasedCheckLimit = 10;
181
182// Another limit for the alias checks: The maximum distance between load/store
183// instructions where alias checks are done.
184// This limit is useful for very large basic blocks.
185static const unsigned MaxMemDepDistance = 160;
186
187/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
188/// regions to be handled.
189static const int MinScheduleRegionSize = 16;
190
191/// Predicate for the element types that the SLP vectorizer supports.
192///
193/// The most important thing to filter here are types which are invalid in LLVM
194/// vectors. We also filter target specific types which have absolutely no
195/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
196/// avoids spending time checking the cost model and realizing that they will
197/// be inevitably scalarized.
198static bool isValidElementType(Type *Ty) {
199 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
200 !Ty->isPPC_FP128Ty();
201}
202
203/// \returns true if all of the instructions in \p VL are in the same block or
204/// false otherwise.
205static bool allSameBlock(ArrayRef<Value *> VL) {
206 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
207 if (!I0)
208 return false;
209 BasicBlock *BB = I0->getParent();
210 for (int I = 1, E = VL.size(); I < E; I++) {
211 auto *II = dyn_cast<Instruction>(VL[I]);
212 if (!II)
213 return false;
214
215 if (BB != II->getParent())
216 return false;
217 }
218 return true;
219}
220
221/// \returns True if the value is a constant (but not globals/constant
222/// expressions).
223static bool isConstant(Value *V) {
224 return isa<Constant>(V) && !isa<ConstantExpr>(V) && !isa<GlobalValue>(V);
225}
226
227/// \returns True if all of the values in \p VL are constants (but not
228/// globals/constant expressions).
229static bool allConstant(ArrayRef<Value *> VL) {
230 // Constant expressions and globals can't be vectorized like normal integer/FP
231 // constants.
232 return all_of(VL, isConstant);
233}
234
235/// \returns True if all of the values in \p VL are identical.
236static bool isSplat(ArrayRef<Value *> VL) {
237 for (unsigned i = 1, e = VL.size(); i < e; ++i)
238 if (VL[i] != VL[0])
239 return false;
240 return true;
241}
242
243/// \returns True if \p I is commutative, handles CmpInst and BinaryOperator.
244static bool isCommutative(Instruction *I) {
245 if (auto *Cmp = dyn_cast<CmpInst>(I))
246 return Cmp->isCommutative();
247 if (auto *BO = dyn_cast<BinaryOperator>(I))
248 return BO->isCommutative();
249 // TODO: This should check for generic Instruction::isCommutative(), but
250 // we need to confirm that the caller code correctly handles Intrinsics
251 // for example (does not have 2 operands).
252 return false;
253}
254
255/// Checks if the vector of instructions can be represented as a shuffle, like:
256/// %x0 = extractelement <4 x i8> %x, i32 0
257/// %x3 = extractelement <4 x i8> %x, i32 3
258/// %y1 = extractelement <4 x i8> %y, i32 1
259/// %y2 = extractelement <4 x i8> %y, i32 2
260/// %x0x0 = mul i8 %x0, %x0
261/// %x3x3 = mul i8 %x3, %x3
262/// %y1y1 = mul i8 %y1, %y1
263/// %y2y2 = mul i8 %y2, %y2
264/// %ins1 = insertelement <4 x i8> poison, i8 %x0x0, i32 0
265/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
266/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
267/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
268/// ret <4 x i8> %ins4
269/// can be transformed into:
270/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
271/// i32 6>
272/// %2 = mul <4 x i8> %1, %1
273/// ret <4 x i8> %2
274/// We convert this initially to something like:
275/// %x0 = extractelement <4 x i8> %x, i32 0
276/// %x3 = extractelement <4 x i8> %x, i32 3
277/// %y1 = extractelement <4 x i8> %y, i32 1
278/// %y2 = extractelement <4 x i8> %y, i32 2
279/// %1 = insertelement <4 x i8> poison, i8 %x0, i32 0
280/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
281/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
282/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
283/// %5 = mul <4 x i8> %4, %4
284/// %6 = extractelement <4 x i8> %5, i32 0
285/// %ins1 = insertelement <4 x i8> poison, i8 %6, i32 0
286/// %7 = extractelement <4 x i8> %5, i32 1
287/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
288/// %8 = extractelement <4 x i8> %5, i32 2
289/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
290/// %9 = extractelement <4 x i8> %5, i32 3
291/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
292/// ret <4 x i8> %ins4
293/// InstCombiner transforms this into a shuffle and vector mul
294/// Mask will return the Shuffle Mask equivalent to the extracted elements.
295/// TODO: Can we split off and reuse the shuffle mask detection from
296/// TargetTransformInfo::getInstructionThroughput?
297static Optional<TargetTransformInfo::ShuffleKind>
298isShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask) {
299 auto *EI0 = cast<ExtractElementInst>(VL[0]);
300 unsigned Size =
301 cast<FixedVectorType>(EI0->getVectorOperandType())->getNumElements();
302 Value *Vec1 = nullptr;
303 Value *Vec2 = nullptr;
304 enum ShuffleMode { Unknown, Select, Permute };
305 ShuffleMode CommonShuffleMode = Unknown;
306 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
307 auto *EI = cast<ExtractElementInst>(VL[I]);
308 auto *Vec = EI->getVectorOperand();
309 // All vector operands must have the same number of vector elements.
310 if (cast<FixedVectorType>(Vec->getType())->getNumElements() != Size)
311 return None;
312 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
313 if (!Idx)
314 return None;
315 // Undefined behavior if Idx is negative or >= Size.
316 if (Idx->getValue().uge(Size)) {
317 Mask.push_back(UndefMaskElem);
318 continue;
319 }
320 unsigned IntIdx = Idx->getValue().getZExtValue();
321 Mask.push_back(IntIdx);
322 // We can extractelement from undef or poison vector.
323 if (isa<UndefValue>(Vec))
324 continue;
325 // For correct shuffling we have to have at most 2 different vector operands
326 // in all extractelement instructions.
327 if (!Vec1 || Vec1 == Vec)
328 Vec1 = Vec;
329 else if (!Vec2 || Vec2 == Vec)
330 Vec2 = Vec;
331 else
332 return None;
333 if (CommonShuffleMode == Permute)
334 continue;
335 // If the extract index is not the same as the operation number, it is a
336 // permutation.
337 if (IntIdx != I) {
338 CommonShuffleMode = Permute;
339 continue;
340 }
341 CommonShuffleMode = Select;
342 }
343 // If we're not crossing lanes in different vectors, consider it as blending.
344 if (CommonShuffleMode == Select && Vec2)
345 return TargetTransformInfo::SK_Select;
346 // If Vec2 was never used, we have a permutation of a single vector, otherwise
347 // we have permutation of 2 vectors.
348 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
349 : TargetTransformInfo::SK_PermuteSingleSrc;
350}
351
352namespace {
353
354/// Main data required for vectorization of instructions.
355struct InstructionsState {
356 /// The very first instruction in the list with the main opcode.
357 Value *OpValue = nullptr;
358
359 /// The main/alternate instruction.
360 Instruction *MainOp = nullptr;
361 Instruction *AltOp = nullptr;
362
363 /// The main/alternate opcodes for the list of instructions.
364 unsigned getOpcode() const {
365 return MainOp ? MainOp->getOpcode() : 0;
366 }
367
368 unsigned getAltOpcode() const {
369 return AltOp ? AltOp->getOpcode() : 0;
370 }
371
372 /// Some of the instructions in the list have alternate opcodes.
373 bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
374
375 bool isOpcodeOrAlt(Instruction *I) const {
376 unsigned CheckedOpcode = I->getOpcode();
377 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
378 }
379
380 InstructionsState() = delete;
381 InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
382 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
383};
384
385} // end anonymous namespace
386
387/// Chooses the correct key for scheduling data. If \p Op has the same (or
388/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
389/// OpValue.
390static Value *isOneOf(const InstructionsState &S, Value *Op) {
391 auto *I = dyn_cast<Instruction>(Op);
392 if (I && S.isOpcodeOrAlt(I))
393 return Op;
394 return S.OpValue;
395}
396
397/// \returns true if \p Opcode is allowed as part of of the main/alternate
398/// instruction for SLP vectorization.
399///
400/// Example of unsupported opcode is SDIV that can potentially cause UB if the
401/// "shuffled out" lane would result in division by zero.
402static bool isValidForAlternation(unsigned Opcode) {
403 if (Instruction::isIntDivRem(Opcode))
404 return false;
405
406 return true;
407}
408
409/// \returns analysis of the Instructions in \p VL described in
410/// InstructionsState, the Opcode that we suppose the whole list
411/// could be vectorized even if its structure is diverse.
412static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
413 unsigned BaseIndex = 0) {
414 // Make sure these are all Instructions.
415 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
416 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
417
418 bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
419 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
420 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
421 unsigned AltOpcode = Opcode;
422 unsigned AltIndex = BaseIndex;
423
424 // Check for one alternate opcode from another BinaryOperator.
425 // TODO - generalize to support all operators (types, calls etc.).
426 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
427 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
428 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
429 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
430 continue;
431 if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) &&
432 isValidForAlternation(Opcode)) {
433 AltOpcode = InstOpcode;
434 AltIndex = Cnt;
435 continue;
436 }
437 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
438 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
439 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
440 if (Ty0 == Ty1) {
441 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
442 continue;
443 if (Opcode == AltOpcode) {
444 assert(isValidForAlternation(Opcode) &&(static_cast <bool> (isValidForAlternation(Opcode) &&
isValidForAlternation(InstOpcode) && "Cast isn't safe for alternation, logic needs to be updated!"
) ? void (0) : __assert_fail ("isValidForAlternation(Opcode) && isValidForAlternation(InstOpcode) && \"Cast isn't safe for alternation, logic needs to be updated!\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 446, __extension__ __PRETTY_FUNCTION__))
445 isValidForAlternation(InstOpcode) &&(static_cast <bool> (isValidForAlternation(Opcode) &&
isValidForAlternation(InstOpcode) && "Cast isn't safe for alternation, logic needs to be updated!"
) ? void (0) : __assert_fail ("isValidForAlternation(Opcode) && isValidForAlternation(InstOpcode) && \"Cast isn't safe for alternation, logic needs to be updated!\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 446, __extension__ __PRETTY_FUNCTION__))
446 "Cast isn't safe for alternation, logic needs to be updated!")(static_cast <bool> (isValidForAlternation(Opcode) &&
isValidForAlternation(InstOpcode) && "Cast isn't safe for alternation, logic needs to be updated!"
) ? void (0) : __assert_fail ("isValidForAlternation(Opcode) && isValidForAlternation(InstOpcode) && \"Cast isn't safe for alternation, logic needs to be updated!\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 446, __extension__ __PRETTY_FUNCTION__))
;
447 AltOpcode = InstOpcode;
448 AltIndex = Cnt;
449 continue;
450 }
451 }
452 } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
453 continue;
454 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
455 }
456
457 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
458 cast<Instruction>(VL[AltIndex]));
459}
460
461/// \returns true if all of the values in \p VL have the same type or false
462/// otherwise.
463static bool allSameType(ArrayRef<Value *> VL) {
464 Type *Ty = VL[0]->getType();
465 for (int i = 1, e = VL.size(); i < e; i++)
466 if (VL[i]->getType() != Ty)
467 return false;
468
469 return true;
470}
471
472/// \returns True if Extract{Value,Element} instruction extracts element Idx.
473static Optional<unsigned> getExtractIndex(Instruction *E) {
474 unsigned Opcode = E->getOpcode();
475 assert((Opcode == Instruction::ExtractElement ||(static_cast <bool> ((Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? void (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 477, __extension__ __PRETTY_FUNCTION__))
476 Opcode == Instruction::ExtractValue) &&(static_cast <bool> ((Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? void (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 477, __extension__ __PRETTY_FUNCTION__))
477 "Expected extractelement or extractvalue instruction.")(static_cast <bool> ((Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? void (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 477, __extension__ __PRETTY_FUNCTION__))
;
478 if (Opcode == Instruction::ExtractElement) {
479 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
480 if (!CI)
481 return None;
482 return CI->getZExtValue();
483 }
484 ExtractValueInst *EI = cast<ExtractValueInst>(E);
485 if (EI->getNumIndices() != 1)
486 return None;
487 return *EI->idx_begin();
488}
489
490/// \returns True if in-tree use also needs extract. This refers to
491/// possible scalar operand in vectorized instruction.
492static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
493 TargetLibraryInfo *TLI) {
494 unsigned Opcode = UserInst->getOpcode();
495 switch (Opcode) {
496 case Instruction::Load: {
497 LoadInst *LI = cast<LoadInst>(UserInst);
498 return (LI->getPointerOperand() == Scalar);
499 }
500 case Instruction::Store: {
501 StoreInst *SI = cast<StoreInst>(UserInst);
502 return (SI->getPointerOperand() == Scalar);
503 }
504 case Instruction::Call: {
505 CallInst *CI = cast<CallInst>(UserInst);
506 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
507 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
508 if (hasVectorInstrinsicScalarOpd(ID, i))
509 return (CI->getArgOperand(i) == Scalar);
510 }
511 LLVM_FALLTHROUGH[[gnu::fallthrough]];
512 }
513 default:
514 return false;
515 }
516}
517
518/// \returns the AA location that is being access by the instruction.
519static MemoryLocation getLocation(Instruction *I, AAResults *AA) {
520 if (StoreInst *SI = dyn_cast<StoreInst>(I))
521 return MemoryLocation::get(SI);
522 if (LoadInst *LI = dyn_cast<LoadInst>(I))
523 return MemoryLocation::get(LI);
524 return MemoryLocation();
525}
526
527/// \returns True if the instruction is not a volatile or atomic load/store.
528static bool isSimple(Instruction *I) {
529 if (LoadInst *LI = dyn_cast<LoadInst>(I))
530 return LI->isSimple();
531 if (StoreInst *SI = dyn_cast<StoreInst>(I))
532 return SI->isSimple();
533 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
534 return !MI->isVolatile();
535 return true;
536}
537
538namespace llvm {
539
540static void inversePermutation(ArrayRef<unsigned> Indices,
541 SmallVectorImpl<int> &Mask) {
542 Mask.clear();
543 const unsigned E = Indices.size();
544 Mask.resize(E, E + 1);
545 for (unsigned I = 0; I < E; ++I)
546 Mask[Indices[I]] = I;
547}
548
549/// \returns inserting index of InsertElement or InsertValue instruction,
550/// using Offset as base offset for index.
551static Optional<int> getInsertIndex(Value *InsertInst, unsigned Offset) {
552 int Index = Offset;
553 if (auto *IE = dyn_cast<InsertElementInst>(InsertInst)) {
554 if (auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2))) {
555 auto *VT = cast<FixedVectorType>(IE->getType());
556 if (CI->getValue().uge(VT->getNumElements()))
557 return UndefMaskElem;
558 Index *= VT->getNumElements();
559 Index += CI->getZExtValue();
560 return Index;
561 }
562 if (isa<UndefValue>(IE->getOperand(2)))
563 return UndefMaskElem;
564 return None;
565 }
566
567 auto *IV = cast<InsertValueInst>(InsertInst);
568 Type *CurrentType = IV->getType();
569 for (unsigned I : IV->indices()) {
570 if (auto *ST = dyn_cast<StructType>(CurrentType)) {
571 Index *= ST->getNumElements();
572 CurrentType = ST->getElementType(I);
573 } else if (auto *AT = dyn_cast<ArrayType>(CurrentType)) {
574 Index *= AT->getNumElements();
575 CurrentType = AT->getElementType();
576 } else {
577 return None;
578 }
579 Index += I;
580 }
581 return Index;
582}
583
584namespace slpvectorizer {
585
586/// Bottom Up SLP Vectorizer.
587class BoUpSLP {
588 struct TreeEntry;
589 struct ScheduleData;
590
591public:
592 using ValueList = SmallVector<Value *, 8>;
593 using InstrList = SmallVector<Instruction *, 16>;
594 using ValueSet = SmallPtrSet<Value *, 16>;
595 using StoreList = SmallVector<StoreInst *, 8>;
596 using ExtraValueToDebugLocsMap =
597 MapVector<Value *, SmallVector<Instruction *, 2>>;
598 using OrdersType = SmallVector<unsigned, 4>;
599
600 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
601 TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
602 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
603 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
604 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
605 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
606 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
607 // Use the vector register size specified by the target unless overridden
608 // by a command-line option.
609 // TODO: It would be better to limit the vectorization factor based on
610 // data type rather than just register size. For example, x86 AVX has
611 // 256-bit registers, but it does not support integer operations
612 // at that width (that requires AVX2).
613 if (MaxVectorRegSizeOption.getNumOccurrences())
614 MaxVecRegSize = MaxVectorRegSizeOption;
615 else
616 MaxVecRegSize =
617 TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
618 .getFixedSize();
619
620 if (MinVectorRegSizeOption.getNumOccurrences())
621 MinVecRegSize = MinVectorRegSizeOption;
622 else
623 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
624 }
625
626 /// Vectorize the tree that starts with the elements in \p VL.
627 /// Returns the vectorized root.
628 Value *vectorizeTree();
629
630 /// Vectorize the tree but with the list of externally used values \p
631 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
632 /// generated extractvalue instructions.
633 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
634
635 /// \returns the cost incurred by unwanted spills and fills, caused by
636 /// holding live values over call sites.
637 InstructionCost getSpillCost() const;
638
639 /// \returns the vectorization cost of the subtree that starts at \p VL.
640 /// A negative number means that this is profitable.
641 InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = None);
642
643 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
644 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
645 void buildTree(ArrayRef<Value *> Roots,
646 ArrayRef<Value *> UserIgnoreLst = None);
647
648 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
649 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
650 /// into account (and updating it, if required) list of externally used
651 /// values stored in \p ExternallyUsedValues.
652 void buildTree(ArrayRef<Value *> Roots,
653 ExtraValueToDebugLocsMap &ExternallyUsedValues,
654 ArrayRef<Value *> UserIgnoreLst = None);
655
656 /// Clear the internal data structures that are created by 'buildTree'.
657 void deleteTree() {
658 VectorizableTree.clear();
659 ScalarToTreeEntry.clear();
660 MustGather.clear();
661 ExternalUses.clear();
662 NumOpsWantToKeepOrder.clear();
663 NumOpsWantToKeepOriginalOrder = 0;
664 for (auto &Iter : BlocksSchedules) {
665 BlockScheduling *BS = Iter.second.get();
666 BS->clear();
667 }
668 MinBWs.clear();
669 InstrElementSize.clear();
670 }
671
672 unsigned getTreeSize() const { return VectorizableTree.size(); }
673
674 /// Perform LICM and CSE on the newly generated gather sequences.
675 void optimizeGatherSequence();
676
677 /// \returns The best order of instructions for vectorization.
678 Optional<ArrayRef<unsigned>> bestOrder() const {
679 assert(llvm::all_of((static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
680 NumOpsWantToKeepOrder,(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
681 [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) {(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
682 return D.getFirst().size() ==(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
683 VectorizableTree[0]->Scalars.size();(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
684 }) &&(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
685 "All orders must have the same size as number of instructions in "(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
686 "tree node.")(static_cast <bool> (llvm::all_of( NumOpsWantToKeepOrder
, [this](const decltype(NumOpsWantToKeepOrder)::value_type &
D) { return D.getFirst().size() == VectorizableTree[0]->Scalars
.size(); }) && "All orders must have the same size as number of instructions in "
"tree node.") ? void (0) : __assert_fail ("llvm::all_of( NumOpsWantToKeepOrder, [this](const decltype(NumOpsWantToKeepOrder)::value_type &D) { return D.getFirst().size() == VectorizableTree[0]->Scalars.size(); }) && \"All orders must have the same size as number of instructions in \" \"tree node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 686, __extension__ __PRETTY_FUNCTION__))
;
687 auto I = std::max_element(
688 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
689 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
690 const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
691 return D1.second < D2.second;
692 });
693 if (I == NumOpsWantToKeepOrder.end() ||
694 I->getSecond() <= NumOpsWantToKeepOriginalOrder)
695 return None;
696
697 return makeArrayRef(I->getFirst());
698 }
699
700 /// Builds the correct order for root instructions.
701 /// If some leaves have the same instructions to be vectorized, we may
702 /// incorrectly evaluate the best order for the root node (it is built for the
703 /// vector of instructions without repeated instructions and, thus, has less
704 /// elements than the root node). This function builds the correct order for
705 /// the root node.
706 /// For example, if the root node is \<a+b, a+c, a+d, f+e\>, then the leaves
707 /// are \<a, a, a, f\> and \<b, c, d, e\>. When we try to vectorize the first
708 /// leaf, it will be shrink to \<a, b\>. If instructions in this leaf should
709 /// be reordered, the best order will be \<1, 0\>. We need to extend this
710 /// order for the root node. For the root node this order should look like
711 /// \<3, 0, 1, 2\>. This function extends the order for the reused
712 /// instructions.
713 void findRootOrder(OrdersType &Order) {
714 // If the leaf has the same number of instructions to vectorize as the root
715 // - order must be set already.
716 unsigned RootSize = VectorizableTree[0]->Scalars.size();
717 if (Order.size() == RootSize)
718 return;
719 SmallVector<unsigned, 4> RealOrder(Order.size());
720 std::swap(Order, RealOrder);
721 SmallVector<int, 4> Mask;
722 inversePermutation(RealOrder, Mask);
723 Order.assign(Mask.begin(), Mask.end());
724 // The leaf has less number of instructions - need to find the true order of
725 // the root.
726 // Scan the nodes starting from the leaf back to the root.
727 const TreeEntry *PNode = VectorizableTree.back().get();
728 SmallVector<const TreeEntry *, 4> Nodes(1, PNode);
729 SmallPtrSet<const TreeEntry *, 4> Visited;
730 while (!Nodes.empty() && Order.size() != RootSize) {
731 const TreeEntry *PNode = Nodes.pop_back_val();
732 if (!Visited.insert(PNode).second)
733 continue;
734 const TreeEntry &Node = *PNode;
735 for (const EdgeInfo &EI : Node.UserTreeIndices)
736 if (EI.UserTE)
737 Nodes.push_back(EI.UserTE);
738 if (Node.ReuseShuffleIndices.empty())
739 continue;
740 // Build the order for the parent node.
741 OrdersType NewOrder(Node.ReuseShuffleIndices.size(), RootSize);
742 SmallVector<unsigned, 4> OrderCounter(Order.size(), 0);
743 // The algorithm of the order extension is:
744 // 1. Calculate the number of the same instructions for the order.
745 // 2. Calculate the index of the new order: total number of instructions
746 // with order less than the order of the current instruction + reuse
747 // number of the current instruction.
748 // 3. The new order is just the index of the instruction in the original
749 // vector of the instructions.
750 for (unsigned I : Node.ReuseShuffleIndices)
751 ++OrderCounter[Order[I]];
752 SmallVector<unsigned, 4> CurrentCounter(Order.size(), 0);
753 for (unsigned I = 0, E = Node.ReuseShuffleIndices.size(); I < E; ++I) {
754 unsigned ReusedIdx = Node.ReuseShuffleIndices[I];
755 unsigned OrderIdx = Order[ReusedIdx];
756 unsigned NewIdx = 0;
757 for (unsigned J = 0; J < OrderIdx; ++J)
758 NewIdx += OrderCounter[J];
759 NewIdx += CurrentCounter[OrderIdx];
760 ++CurrentCounter[OrderIdx];
761 assert(NewOrder[NewIdx] == RootSize &&(static_cast <bool> (NewOrder[NewIdx] == RootSize &&
"The order index should not be written already.") ? void (0)
: __assert_fail ("NewOrder[NewIdx] == RootSize && \"The order index should not be written already.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 762, __extension__ __PRETTY_FUNCTION__))
762 "The order index should not be written already.")(static_cast <bool> (NewOrder[NewIdx] == RootSize &&
"The order index should not be written already.") ? void (0)
: __assert_fail ("NewOrder[NewIdx] == RootSize && \"The order index should not be written already.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 762, __extension__ __PRETTY_FUNCTION__))
;
763 NewOrder[NewIdx] = I;
764 }
765 std::swap(Order, NewOrder);
766 }
767 assert(Order.size() == RootSize &&(static_cast <bool> (Order.size() == RootSize &&
"Root node is expected or the size of the order must be the same as "
"the number of elements in the root node.") ? void (0) : __assert_fail
("Order.size() == RootSize && \"Root node is expected or the size of the order must be the same as \" \"the number of elements in the root node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 769, __extension__ __PRETTY_FUNCTION__))
768 "Root node is expected or the size of the order must be the same as "(static_cast <bool> (Order.size() == RootSize &&
"Root node is expected or the size of the order must be the same as "
"the number of elements in the root node.") ? void (0) : __assert_fail
("Order.size() == RootSize && \"Root node is expected or the size of the order must be the same as \" \"the number of elements in the root node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 769, __extension__ __PRETTY_FUNCTION__))
769 "the number of elements in the root node.")(static_cast <bool> (Order.size() == RootSize &&
"Root node is expected or the size of the order must be the same as "
"the number of elements in the root node.") ? void (0) : __assert_fail
("Order.size() == RootSize && \"Root node is expected or the size of the order must be the same as \" \"the number of elements in the root node.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 769, __extension__ __PRETTY_FUNCTION__))
;
770 assert(llvm::all_of(Order,(static_cast <bool> (llvm::all_of(Order, [RootSize](unsigned
Val) { return Val != RootSize; }) && "All indices must be initialized"
) ? void (0) : __assert_fail ("llvm::all_of(Order, [RootSize](unsigned Val) { return Val != RootSize; }) && \"All indices must be initialized\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 772, __extension__ __PRETTY_FUNCTION__))
771 [RootSize](unsigned Val) { return Val != RootSize; }) &&(static_cast <bool> (llvm::all_of(Order, [RootSize](unsigned
Val) { return Val != RootSize; }) && "All indices must be initialized"
) ? void (0) : __assert_fail ("llvm::all_of(Order, [RootSize](unsigned Val) { return Val != RootSize; }) && \"All indices must be initialized\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 772, __extension__ __PRETTY_FUNCTION__))
772 "All indices must be initialized")(static_cast <bool> (llvm::all_of(Order, [RootSize](unsigned
Val) { return Val != RootSize; }) && "All indices must be initialized"
) ? void (0) : __assert_fail ("llvm::all_of(Order, [RootSize](unsigned Val) { return Val != RootSize; }) && \"All indices must be initialized\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 772, __extension__ __PRETTY_FUNCTION__))
;
773 }
774
775 /// \return The vector element size in bits to use when vectorizing the
776 /// expression tree ending at \p V. If V is a store, the size is the width of
777 /// the stored value. Otherwise, the size is the width of the largest loaded
778 /// value reaching V. This method is used by the vectorizer to calculate
779 /// vectorization factors.
780 unsigned getVectorElementSize(Value *V);
781
782 /// Compute the minimum type sizes required to represent the entries in a
783 /// vectorizable tree.
784 void computeMinimumValueSizes();
785
786 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
787 unsigned getMaxVecRegSize() const {
788 return MaxVecRegSize;
789 }
790
791 // \returns minimum vector register size as set by cl::opt.
792 unsigned getMinVecRegSize() const {
793 return MinVecRegSize;
794 }
795
796 unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
797 unsigned MaxVF = MaxVFOption.getNumOccurrences() ?
798 MaxVFOption : TTI->getMaximumVF(ElemWidth, Opcode);
799 return MaxVF ? MaxVF : UINT_MAX(2147483647 *2U +1U);
800 }
801
802 /// Check if homogeneous aggregate is isomorphic to some VectorType.
803 /// Accepts homogeneous multidimensional aggregate of scalars/vectors like
804 /// {[4 x i16], [4 x i16]}, { <2 x float>, <2 x float> },
805 /// {{{i16, i16}, {i16, i16}}, {{i16, i16}, {i16, i16}}} and so on.
806 ///
807 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
808 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
809
810 /// \returns True if the VectorizableTree is both tiny and not fully
811 /// vectorizable. We do not vectorize such trees.
812 bool isTreeTinyAndNotFullyVectorizable() const;
813
814 /// Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values
815 /// can be load combined in the backend. Load combining may not be allowed in
816 /// the IR optimizer, so we do not want to alter the pattern. For example,
817 /// partially transforming a scalar bswap() pattern into vector code is
818 /// effectively impossible for the backend to undo.
819 /// TODO: If load combining is allowed in the IR optimizer, this analysis
820 /// may not be necessary.
821 bool isLoadCombineReductionCandidate(RecurKind RdxKind) const;
822
823 /// Assume that a vector of stores of bitwise-or/shifted/zexted loaded values
824 /// can be load combined in the backend. Load combining may not be allowed in
825 /// the IR optimizer, so we do not want to alter the pattern. For example,
826 /// partially transforming a scalar bswap() pattern into vector code is
827 /// effectively impossible for the backend to undo.
828 /// TODO: If load combining is allowed in the IR optimizer, this analysis
829 /// may not be necessary.
830 bool isLoadCombineCandidate() const;
831
832 OptimizationRemarkEmitter *getORE() { return ORE; }
833
834 /// This structure holds any data we need about the edges being traversed
835 /// during buildTree_rec(). We keep track of:
836 /// (i) the user TreeEntry index, and
837 /// (ii) the index of the edge.
838 struct EdgeInfo {
839 EdgeInfo() = default;
840 EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx)
841 : UserTE(UserTE), EdgeIdx(EdgeIdx) {}
842 /// The user TreeEntry.
843 TreeEntry *UserTE = nullptr;
844 /// The operand index of the use.
845 unsigned EdgeIdx = UINT_MAX(2147483647 *2U +1U);
846#ifndef NDEBUG
847 friend inline raw_ostream &operator<<(raw_ostream &OS,
848 const BoUpSLP::EdgeInfo &EI) {
849 EI.dump(OS);
850 return OS;
851 }
852 /// Debug print.
853 void dump(raw_ostream &OS) const {
854 OS << "{User:" << (UserTE ? std::to_string(UserTE->Idx) : "null")
855 << " EdgeIdx:" << EdgeIdx << "}";
856 }
857 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dump() const { dump(dbgs()); }
858#endif
859 };
860
861 /// A helper data structure to hold the operands of a vector of instructions.
862 /// This supports a fixed vector length for all operand vectors.
863 class VLOperands {
864 /// For each operand we need (i) the value, and (ii) the opcode that it
865 /// would be attached to if the expression was in a left-linearized form.
866 /// This is required to avoid illegal operand reordering.
867 /// For example:
868 /// \verbatim
869 /// 0 Op1
870 /// |/
871 /// Op1 Op2 Linearized + Op2
872 /// \ / ----------> |/
873 /// - -
874 ///
875 /// Op1 - Op2 (0 + Op1) - Op2
876 /// \endverbatim
877 ///
878 /// Value Op1 is attached to a '+' operation, and Op2 to a '-'.
879 ///
880 /// Another way to think of this is to track all the operations across the
881 /// path from the operand all the way to the root of the tree and to
882 /// calculate the operation that corresponds to this path. For example, the
883 /// path from Op2 to the root crosses the RHS of the '-', therefore the
884 /// corresponding operation is a '-' (which matches the one in the
885 /// linearized tree, as shown above).
886 ///
887 /// For lack of a better term, we refer to this operation as Accumulated
888 /// Path Operation (APO).
889 struct OperandData {
890 OperandData() = default;
891 OperandData(Value *V, bool APO, bool IsUsed)
892 : V(V), APO(APO), IsUsed(IsUsed) {}
893 /// The operand value.
894 Value *V = nullptr;
895 /// TreeEntries only allow a single opcode, or an alternate sequence of
896 /// them (e.g, +, -). Therefore, we can safely use a boolean value for the
897 /// APO. It is set to 'true' if 'V' is attached to an inverse operation
898 /// in the left-linearized form (e.g., Sub/Div), and 'false' otherwise
899 /// (e.g., Add/Mul)
900 bool APO = false;
901 /// Helper data for the reordering function.
902 bool IsUsed = false;
903 };
904
905 /// During operand reordering, we are trying to select the operand at lane
906 /// that matches best with the operand at the neighboring lane. Our
907 /// selection is based on the type of value we are looking for. For example,
908 /// if the neighboring lane has a load, we need to look for a load that is
909 /// accessing a consecutive address. These strategies are summarized in the
910 /// 'ReorderingMode' enumerator.
911 enum class ReorderingMode {
912 Load, ///< Matching loads to consecutive memory addresses
913 Opcode, ///< Matching instructions based on opcode (same or alternate)
914 Constant, ///< Matching constants
915 Splat, ///< Matching the same instruction multiple times (broadcast)
916 Failed, ///< We failed to create a vectorizable group
917 };
918
919 using OperandDataVec = SmallVector<OperandData, 2>;
920
921 /// A vector of operand vectors.
922 SmallVector<OperandDataVec, 4> OpsVec;
923
924 const DataLayout &DL;
925 ScalarEvolution &SE;
926 const BoUpSLP &R;
927
928 /// \returns the operand data at \p OpIdx and \p Lane.
929 OperandData &getData(unsigned OpIdx, unsigned Lane) {
930 return OpsVec[OpIdx][Lane];
931 }
932
933 /// \returns the operand data at \p OpIdx and \p Lane. Const version.
934 const OperandData &getData(unsigned OpIdx, unsigned Lane) const {
935 return OpsVec[OpIdx][Lane];
936 }
937
938 /// Clears the used flag for all entries.
939 void clearUsed() {
940 for (unsigned OpIdx = 0, NumOperands = getNumOperands();
941 OpIdx != NumOperands; ++OpIdx)
942 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
943 ++Lane)
944 OpsVec[OpIdx][Lane].IsUsed = false;
945 }
946
947 /// Swap the operand at \p OpIdx1 with that one at \p OpIdx2.
948 void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) {
949 std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
950 }
951
952 // The hard-coded scores listed here are not very important. When computing
953 // the scores of matching one sub-tree with another, we are basically
954 // counting the number of values that are matching. So even if all scores
955 // are set to 1, we would still get a decent matching result.
956 // However, sometimes we have to break ties. For example we may have to
957 // choose between matching loads vs matching opcodes. This is what these
958 // scores are helping us with: they provide the order of preference.
959
960 /// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
961 static const int ScoreConsecutiveLoads = 3;
962 /// ExtractElementInst from same vector and consecutive indexes.
963 static const int ScoreConsecutiveExtracts = 3;
964 /// Constants.
965 static const int ScoreConstants = 2;
966 /// Instructions with the same opcode.
967 static const int ScoreSameOpcode = 2;
968 /// Instructions with alt opcodes (e.g, add + sub).
969 static const int ScoreAltOpcodes = 1;
970 /// Identical instructions (a.k.a. splat or broadcast).
971 static const int ScoreSplat = 1;
972 /// Matching with an undef is preferable to failing.
973 static const int ScoreUndef = 1;
974 /// Score for failing to find a decent match.
975 static const int ScoreFail = 0;
976 /// User exteranl to the vectorized code.
977 static const int ExternalUseCost = 1;
978 /// The user is internal but in a different lane.
979 static const int UserInDiffLaneCost = ExternalUseCost;
980
981 /// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
982 static int getShallowScore(Value *V1, Value *V2, const DataLayout &DL,
983 ScalarEvolution &SE) {
984 auto *LI1 = dyn_cast<LoadInst>(V1);
985 auto *LI2 = dyn_cast<LoadInst>(V2);
986 if (LI1 && LI2) {
987 if (LI1->getParent() != LI2->getParent())
988 return VLOperands::ScoreFail;
989
990 Optional<int> Dist =
991 getPointersDiff(LI1->getPointerOperand(), LI2->getPointerOperand(),
992 DL, SE, /*StrictCheck=*/true);
993 return (Dist && *Dist == 1) ? VLOperands::ScoreConsecutiveLoads
994 : VLOperands::ScoreFail;
995 }
996
997 auto *C1 = dyn_cast<Constant>(V1);
998 auto *C2 = dyn_cast<Constant>(V2);
999 if (C1 && C2)
1000 return VLOperands::ScoreConstants;
1001
1002 // Extracts from consecutive indexes of the same vector better score as
1003 // the extracts could be optimized away.
1004 Value *EV;
1005 ConstantInt *Ex1Idx, *Ex2Idx;
1006 if (match(V1, m_ExtractElt(m_Value(EV), m_ConstantInt(Ex1Idx))) &&
1007 match(V2, m_ExtractElt(m_Deferred(EV), m_ConstantInt(Ex2Idx))) &&
1008 Ex1Idx->getZExtValue() + 1 == Ex2Idx->getZExtValue())
1009 return VLOperands::ScoreConsecutiveExtracts;
1010
1011 auto *I1 = dyn_cast<Instruction>(V1);
1012 auto *I2 = dyn_cast<Instruction>(V2);
1013 if (I1 && I2) {
1014 if (I1 == I2)
1015 return VLOperands::ScoreSplat;
1016 InstructionsState S = getSameOpcode({I1, I2});
1017 // Note: Only consider instructions with <= 2 operands to avoid
1018 // complexity explosion.
1019 if (S.getOpcode() && S.MainOp->getNumOperands() <= 2)
1020 return S.isAltShuffle() ? VLOperands::ScoreAltOpcodes
1021 : VLOperands::ScoreSameOpcode;
1022 }
1023
1024 if (isa<UndefValue>(V2))
1025 return VLOperands::ScoreUndef;
1026
1027 return VLOperands::ScoreFail;
1028 }
1029
1030 /// Holds the values and their lane that are taking part in the look-ahead
1031 /// score calculation. This is used in the external uses cost calculation.
1032 SmallDenseMap<Value *, int> InLookAheadValues;
1033
1034 /// \Returns the additinal cost due to uses of \p LHS and \p RHS that are
1035 /// either external to the vectorized code, or require shuffling.
1036 int getExternalUsesCost(const std::pair<Value *, int> &LHS,
1037 const std::pair<Value *, int> &RHS) {
1038 int Cost = 0;
1039 std::array<std::pair<Value *, int>, 2> Values = {{LHS, RHS}};
1040 for (int Idx = 0, IdxE = Values.size(); Idx != IdxE; ++Idx) {
1041 Value *V = Values[Idx].first;
1042 if (isa<Constant>(V)) {
1043 // Since this is a function pass, it doesn't make semantic sense to
1044 // walk the users of a subclass of Constant. The users could be in
1045 // another function, or even another module that happens to be in
1046 // the same LLVMContext.
1047 continue;
1048 }
1049
1050 // Calculate the absolute lane, using the minimum relative lane of LHS
1051 // and RHS as base and Idx as the offset.
1052 int Ln = std::min(LHS.second, RHS.second) + Idx;
1053 assert(Ln >= 0 && "Bad lane calculation")(static_cast <bool> (Ln >= 0 && "Bad lane calculation"
) ? void (0) : __assert_fail ("Ln >= 0 && \"Bad lane calculation\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1053, __extension__ __PRETTY_FUNCTION__))
;
1054 unsigned UsersBudget = LookAheadUsersBudget;
1055 for (User *U : V->users()) {
1056 if (const TreeEntry *UserTE = R.getTreeEntry(U)) {
1057 // The user is in the VectorizableTree. Check if we need to insert.
1058 auto It = llvm::find(UserTE->Scalars, U);
1059 assert(It != UserTE->Scalars.end() && "U is in UserTE")(static_cast <bool> (It != UserTE->Scalars.end() &&
"U is in UserTE") ? void (0) : __assert_fail ("It != UserTE->Scalars.end() && \"U is in UserTE\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1059, __extension__ __PRETTY_FUNCTION__))
;
1060 int UserLn = std::distance(UserTE->Scalars.begin(), It);
1061 assert(UserLn >= 0 && "Bad lane")(static_cast <bool> (UserLn >= 0 && "Bad lane"
) ? void (0) : __assert_fail ("UserLn >= 0 && \"Bad lane\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1061, __extension__ __PRETTY_FUNCTION__))
;
1062 if (UserLn != Ln)
1063 Cost += UserInDiffLaneCost;
1064 } else {
1065 // Check if the user is in the look-ahead code.
1066 auto It2 = InLookAheadValues.find(U);
1067 if (It2 != InLookAheadValues.end()) {
1068 // The user is in the look-ahead code. Check the lane.
1069 if (It2->second != Ln)
1070 Cost += UserInDiffLaneCost;
1071 } else {
1072 // The user is neither in SLP tree nor in the look-ahead code.
1073 Cost += ExternalUseCost;
1074 }
1075 }
1076 // Limit the number of visited uses to cap compilation time.
1077 if (--UsersBudget == 0)
1078 break;
1079 }
1080 }
1081 return Cost;
1082 }
1083
1084 /// Go through the operands of \p LHS and \p RHS recursively until \p
1085 /// MaxLevel, and return the cummulative score. For example:
1086 /// \verbatim
1087 /// A[0] B[0] A[1] B[1] C[0] D[0] B[1] A[1]
1088 /// \ / \ / \ / \ /
1089 /// + + + +
1090 /// G1 G2 G3 G4
1091 /// \endverbatim
1092 /// The getScoreAtLevelRec(G1, G2) function will try to match the nodes at
1093 /// each level recursively, accumulating the score. It starts from matching
1094 /// the additions at level 0, then moves on to the loads (level 1). The
1095 /// score of G1 and G2 is higher than G1 and G3, because {A[0],A[1]} and
1096 /// {B[0],B[1]} match with VLOperands::ScoreConsecutiveLoads, while
1097 /// {A[0],C[0]} has a score of VLOperands::ScoreFail.
1098 /// Please note that the order of the operands does not matter, as we
1099 /// evaluate the score of all profitable combinations of operands. In
1100 /// other words the score of G1 and G4 is the same as G1 and G2. This
1101 /// heuristic is based on ideas described in:
1102 /// Look-ahead SLP: Auto-vectorization in the presence of commutative
1103 /// operations, CGO 2018 by Vasileios Porpodas, Rodrigo C. O. Rocha,
1104 /// Luís F. W. Góes
1105 int getScoreAtLevelRec(const std::pair<Value *, int> &LHS,
1106 const std::pair<Value *, int> &RHS, int CurrLevel,
1107 int MaxLevel) {
1108
1109 Value *V1 = LHS.first;
1110 Value *V2 = RHS.first;
1111 // Get the shallow score of V1 and V2.
1112 int ShallowScoreAtThisLevel =
1113 std::max((int)ScoreFail, getShallowScore(V1, V2, DL, SE) -
1114 getExternalUsesCost(LHS, RHS));
1115 int Lane1 = LHS.second;
1116 int Lane2 = RHS.second;
1117
1118 // If reached MaxLevel,
1119 // or if V1 and V2 are not instructions,
1120 // or if they are SPLAT,
1121 // or if they are not consecutive, early return the current cost.
1122 auto *I1 = dyn_cast<Instruction>(V1);
1123 auto *I2 = dyn_cast<Instruction>(V2);
1124 if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 ||
1125 ShallowScoreAtThisLevel == VLOperands::ScoreFail ||
1126 (isa<LoadInst>(I1) && isa<LoadInst>(I2) && ShallowScoreAtThisLevel))
1127 return ShallowScoreAtThisLevel;
1128 assert(I1 && I2 && "Should have early exited.")(static_cast <bool> (I1 && I2 && "Should have early exited."
) ? void (0) : __assert_fail ("I1 && I2 && \"Should have early exited.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1128, __extension__ __PRETTY_FUNCTION__))
;
1129
1130 // Keep track of in-tree values for determining the external-use cost.
1131 InLookAheadValues[V1] = Lane1;
1132 InLookAheadValues[V2] = Lane2;
1133
1134 // Contains the I2 operand indexes that got matched with I1 operands.
1135 SmallSet<unsigned, 4> Op2Used;
1136
1137 // Recursion towards the operands of I1 and I2. We are trying all possbile
1138 // operand pairs, and keeping track of the best score.
1139 for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands();
1140 OpIdx1 != NumOperands1; ++OpIdx1) {
1141 // Try to pair op1I with the best operand of I2.
1142 int MaxTmpScore = 0;
1143 unsigned MaxOpIdx2 = 0;
1144 bool FoundBest = false;
1145 // If I2 is commutative try all combinations.
1146 unsigned FromIdx = isCommutative(I2) ? 0 : OpIdx1;
1147 unsigned ToIdx = isCommutative(I2)
1148 ? I2->getNumOperands()
1149 : std::min(I2->getNumOperands(), OpIdx1 + 1);
1150 assert(FromIdx <= ToIdx && "Bad index")(static_cast <bool> (FromIdx <= ToIdx && "Bad index"
) ? void (0) : __assert_fail ("FromIdx <= ToIdx && \"Bad index\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1150, __extension__ __PRETTY_FUNCTION__))
;
1151 for (unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) {
1152 // Skip operands already paired with OpIdx1.
1153 if (Op2Used.count(OpIdx2))
1154 continue;
1155 // Recursively calculate the cost at each level
1156 int TmpScore = getScoreAtLevelRec({I1->getOperand(OpIdx1), Lane1},
1157 {I2->getOperand(OpIdx2), Lane2},
1158 CurrLevel + 1, MaxLevel);
1159 // Look for the best score.
1160 if (TmpScore > VLOperands::ScoreFail && TmpScore > MaxTmpScore) {
1161 MaxTmpScore = TmpScore;
1162 MaxOpIdx2 = OpIdx2;
1163 FoundBest = true;
1164 }
1165 }
1166 if (FoundBest) {
1167 // Pair {OpIdx1, MaxOpIdx2} was found to be best. Never revisit it.
1168 Op2Used.insert(MaxOpIdx2);
1169 ShallowScoreAtThisLevel += MaxTmpScore;
1170 }
1171 }
1172 return ShallowScoreAtThisLevel;
1173 }
1174
1175 /// \Returns the look-ahead score, which tells us how much the sub-trees
1176 /// rooted at \p LHS and \p RHS match, the more they match the higher the
1177 /// score. This helps break ties in an informed way when we cannot decide on
1178 /// the order of the operands by just considering the immediate
1179 /// predecessors.
1180 int getLookAheadScore(const std::pair<Value *, int> &LHS,
1181 const std::pair<Value *, int> &RHS) {
1182 InLookAheadValues.clear();
1183 return getScoreAtLevelRec(LHS, RHS, 1, LookAheadMaxDepth);
1184 }
1185
1186 // Search all operands in Ops[*][Lane] for the one that matches best
1187 // Ops[OpIdx][LastLane] and return its opreand index.
1188 // If no good match can be found, return None.
1189 Optional<unsigned>
1190 getBestOperand(unsigned OpIdx, int Lane, int LastLane,
1191 ArrayRef<ReorderingMode> ReorderingModes) {
1192 unsigned NumOperands = getNumOperands();
1193
1194 // The operand of the previous lane at OpIdx.
1195 Value *OpLastLane = getData(OpIdx, LastLane).V;
1196
1197 // Our strategy mode for OpIdx.
1198 ReorderingMode RMode = ReorderingModes[OpIdx];
1199
1200 // The linearized opcode of the operand at OpIdx, Lane.
1201 bool OpIdxAPO = getData(OpIdx, Lane).APO;
1202
1203 // The best operand index and its score.
1204 // Sometimes we have more than one option (e.g., Opcode and Undefs), so we
1205 // are using the score to differentiate between the two.
1206 struct BestOpData {
1207 Optional<unsigned> Idx = None;
1208 unsigned Score = 0;
1209 } BestOp;
1210
1211 // Iterate through all unused operands and look for the best.
1212 for (unsigned Idx = 0; Idx != NumOperands; ++Idx) {
1213 // Get the operand at Idx and Lane.
1214 OperandData &OpData = getData(Idx, Lane);
1215 Value *Op = OpData.V;
1216 bool OpAPO = OpData.APO;
1217
1218 // Skip already selected operands.
1219 if (OpData.IsUsed)
1220 continue;
1221
1222 // Skip if we are trying to move the operand to a position with a
1223 // different opcode in the linearized tree form. This would break the
1224 // semantics.
1225 if (OpAPO != OpIdxAPO)
1226 continue;
1227
1228 // Look for an operand that matches the current mode.
1229 switch (RMode) {
1230 case ReorderingMode::Load:
1231 case ReorderingMode::Constant:
1232 case ReorderingMode::Opcode: {
1233 bool LeftToRight = Lane > LastLane;
1234 Value *OpLeft = (LeftToRight) ? OpLastLane : Op;
1235 Value *OpRight = (LeftToRight) ? Op : OpLastLane;
1236 unsigned Score =
1237 getLookAheadScore({OpLeft, LastLane}, {OpRight, Lane});
1238 if (Score > BestOp.Score) {
1239 BestOp.Idx = Idx;
1240 BestOp.Score = Score;
1241 }
1242 break;
1243 }
1244 case ReorderingMode::Splat:
1245 if (Op == OpLastLane)
1246 BestOp.Idx = Idx;
1247 break;
1248 case ReorderingMode::Failed:
1249 return None;
1250 }
1251 }
1252
1253 if (BestOp.Idx) {
1254 getData(BestOp.Idx.getValue(), Lane).IsUsed = true;
1255 return BestOp.Idx;
1256 }
1257 // If we could not find a good match return None.
1258 return None;
1259 }
1260
1261 /// Helper for reorderOperandVecs. \Returns the lane that we should start
1262 /// reordering from. This is the one which has the least number of operands
1263 /// that can freely move about.
1264 unsigned getBestLaneToStartReordering() const {
1265 unsigned BestLane = 0;
1266 unsigned Min = UINT_MAX(2147483647 *2U +1U);
1267 for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
1268 ++Lane) {
1269 unsigned NumFreeOps = getMaxNumOperandsThatCanBeReordered(Lane);
1270 if (NumFreeOps < Min) {
1271 Min = NumFreeOps;
1272 BestLane = Lane;
1273 }
1274 }
1275 return BestLane;
1276 }
1277
1278 /// \Returns the maximum number of operands that are allowed to be reordered
1279 /// for \p Lane. This is used as a heuristic for selecting the first lane to
1280 /// start operand reordering.
1281 unsigned getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
1282 unsigned CntTrue = 0;
1283 unsigned NumOperands = getNumOperands();
1284 // Operands with the same APO can be reordered. We therefore need to count
1285 // how many of them we have for each APO, like this: Cnt[APO] = x.
1286 // Since we only have two APOs, namely true and false, we can avoid using
1287 // a map. Instead we can simply count the number of operands that
1288 // correspond to one of them (in this case the 'true' APO), and calculate
1289 // the other by subtracting it from the total number of operands.
1290 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx)
1291 if (getData(OpIdx, Lane).APO)
1292 ++CntTrue;
1293 unsigned CntFalse = NumOperands - CntTrue;
1294 return std::max(CntTrue, CntFalse);
1295 }
1296
1297 /// Go through the instructions in VL and append their operands.
1298 void appendOperandsOfVL(ArrayRef<Value *> VL) {
1299 assert(!VL.empty() && "Bad VL")(static_cast <bool> (!VL.empty() && "Bad VL") ?
void (0) : __assert_fail ("!VL.empty() && \"Bad VL\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1299, __extension__ __PRETTY_FUNCTION__))
;
1300 assert((empty() || VL.size() == getNumLanes()) &&(static_cast <bool> ((empty() || VL.size() == getNumLanes
()) && "Expected same number of lanes") ? void (0) : __assert_fail
("(empty() || VL.size() == getNumLanes()) && \"Expected same number of lanes\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1301, __extension__ __PRETTY_FUNCTION__))
1301 "Expected same number of lanes")(static_cast <bool> ((empty() || VL.size() == getNumLanes
()) && "Expected same number of lanes") ? void (0) : __assert_fail
("(empty() || VL.size() == getNumLanes()) && \"Expected same number of lanes\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1301, __extension__ __PRETTY_FUNCTION__))
;
1302 assert(isa<Instruction>(VL[0]) && "Expected instruction")(static_cast <bool> (isa<Instruction>(VL[0]) &&
"Expected instruction") ? void (0) : __assert_fail ("isa<Instruction>(VL[0]) && \"Expected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1302, __extension__ __PRETTY_FUNCTION__))
;
1303 unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands();
1304 OpsVec.resize(NumOperands);
1305 unsigned NumLanes = VL.size();
1306 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1307 OpsVec[OpIdx].resize(NumLanes);
1308 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
1309 assert(isa<Instruction>(VL[Lane]) && "Expected instruction")(static_cast <bool> (isa<Instruction>(VL[Lane]) &&
"Expected instruction") ? void (0) : __assert_fail ("isa<Instruction>(VL[Lane]) && \"Expected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1309, __extension__ __PRETTY_FUNCTION__))
;
1310 // Our tree has just 3 nodes: the root and two operands.
1311 // It is therefore trivial to get the APO. We only need to check the
1312 // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or
1313 // RHS operand. The LHS operand of both add and sub is never attached
1314 // to an inversese operation in the linearized form, therefore its APO
1315 // is false. The RHS is true only if VL[Lane] is an inverse operation.
1316
1317 // Since operand reordering is performed on groups of commutative
1318 // operations or alternating sequences (e.g., +, -), we can safely
1319 // tell the inverse operations by checking commutativity.
1320 bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane]));
1321 bool APO = (OpIdx == 0) ? false : IsInverseOperation;
1322 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx),
1323 APO, false};
1324 }
1325 }
1326 }
1327
1328 /// \returns the number of operands.
1329 unsigned getNumOperands() const { return OpsVec.size(); }
1330
1331 /// \returns the number of lanes.
1332 unsigned getNumLanes() const { return OpsVec[0].size(); }
1333
1334 /// \returns the operand value at \p OpIdx and \p Lane.
1335 Value *getValue(unsigned OpIdx, unsigned Lane) const {
1336 return getData(OpIdx, Lane).V;
1337 }
1338
1339 /// \returns true if the data structure is empty.
1340 bool empty() const { return OpsVec.empty(); }
1341
1342 /// Clears the data.
1343 void clear() { OpsVec.clear(); }
1344
1345 /// \Returns true if there are enough operands identical to \p Op to fill
1346 /// the whole vector.
1347 /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow.
1348 bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
1349 bool OpAPO = getData(OpIdx, Lane).APO;
1350 for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
1351 if (Ln == Lane)
1352 continue;
1353 // This is set to true if we found a candidate for broadcast at Lane.
1354 bool FoundCandidate = false;
1355 for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) {
1356 OperandData &Data = getData(OpI, Ln);
1357 if (Data.APO != OpAPO || Data.IsUsed)
1358 continue;
1359 if (Data.V == Op) {
1360 FoundCandidate = true;
1361 Data.IsUsed = true;
1362 break;
1363 }
1364 }
1365 if (!FoundCandidate)
1366 return false;
1367 }
1368 return true;
1369 }
1370
1371 public:
1372 /// Initialize with all the operands of the instruction vector \p RootVL.
1373 VLOperands(ArrayRef<Value *> RootVL, const DataLayout &DL,
1374 ScalarEvolution &SE, const BoUpSLP &R)
1375 : DL(DL), SE(SE), R(R) {
1376 // Append all the operands of RootVL.
1377 appendOperandsOfVL(RootVL);
1378 }
1379
1380 /// \Returns a value vector with the operands across all lanes for the
1381 /// opearnd at \p OpIdx.
1382 ValueList getVL(unsigned OpIdx) const {
1383 ValueList OpVL(OpsVec[OpIdx].size());
1384 assert(OpsVec[OpIdx].size() == getNumLanes() &&(static_cast <bool> (OpsVec[OpIdx].size() == getNumLanes
() && "Expected same num of lanes across all operands"
) ? void (0) : __assert_fail ("OpsVec[OpIdx].size() == getNumLanes() && \"Expected same num of lanes across all operands\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1385, __extension__ __PRETTY_FUNCTION__))
1385 "Expected same num of lanes across all operands")(static_cast <bool> (OpsVec[OpIdx].size() == getNumLanes
() && "Expected same num of lanes across all operands"
) ? void (0) : __assert_fail ("OpsVec[OpIdx].size() == getNumLanes() && \"Expected same num of lanes across all operands\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1385, __extension__ __PRETTY_FUNCTION__))
;
1386 for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane)
1387 OpVL[Lane] = OpsVec[OpIdx][Lane].V;
1388 return OpVL;
1389 }
1390
1391 // Performs operand reordering for 2 or more operands.
1392 // The original operands are in OrigOps[OpIdx][Lane].
1393 // The reordered operands are returned in 'SortedOps[OpIdx][Lane]'.
1394 void reorder() {
1395 unsigned NumOperands = getNumOperands();
1396 unsigned NumLanes = getNumLanes();
1397 // Each operand has its own mode. We are using this mode to help us select
1398 // the instructions for each lane, so that they match best with the ones
1399 // we have selected so far.
1400 SmallVector<ReorderingMode, 2> ReorderingModes(NumOperands);
1401
1402 // This is a greedy single-pass algorithm. We are going over each lane
1403 // once and deciding on the best order right away with no back-tracking.
1404 // However, in order to increase its effectiveness, we start with the lane
1405 // that has operands that can move the least. For example, given the
1406 // following lanes:
1407 // Lane 0 : A[0] = B[0] + C[0] // Visited 3rd
1408 // Lane 1 : A[1] = C[1] - B[1] // Visited 1st
1409 // Lane 2 : A[2] = B[2] + C[2] // Visited 2nd
1410 // Lane 3 : A[3] = C[3] - B[3] // Visited 4th
1411 // we will start at Lane 1, since the operands of the subtraction cannot
1412 // be reordered. Then we will visit the rest of the lanes in a circular
1413 // fashion. That is, Lanes 2, then Lane 0, and finally Lane 3.
1414
1415 // Find the first lane that we will start our search from.
1416 unsigned FirstLane = getBestLaneToStartReordering();
1417
1418 // Initialize the modes.
1419 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1420 Value *OpLane0 = getValue(OpIdx, FirstLane);
1421 // Keep track if we have instructions with all the same opcode on one
1422 // side.
1423 if (isa<LoadInst>(OpLane0))
1424 ReorderingModes[OpIdx] = ReorderingMode::Load;
1425 else if (isa<Instruction>(OpLane0)) {
1426 // Check if OpLane0 should be broadcast.
1427 if (shouldBroadcast(OpLane0, OpIdx, FirstLane))
1428 ReorderingModes[OpIdx] = ReorderingMode::Splat;
1429 else
1430 ReorderingModes[OpIdx] = ReorderingMode::Opcode;
1431 }
1432 else if (isa<Constant>(OpLane0))
1433 ReorderingModes[OpIdx] = ReorderingMode::Constant;
1434 else if (isa<Argument>(OpLane0))
1435 // Our best hope is a Splat. It may save some cost in some cases.
1436 ReorderingModes[OpIdx] = ReorderingMode::Splat;
1437 else
1438 // NOTE: This should be unreachable.
1439 ReorderingModes[OpIdx] = ReorderingMode::Failed;
1440 }
1441
1442 // If the initial strategy fails for any of the operand indexes, then we
1443 // perform reordering again in a second pass. This helps avoid assigning
1444 // high priority to the failed strategy, and should improve reordering for
1445 // the non-failed operand indexes.
1446 for (int Pass = 0; Pass != 2; ++Pass) {
1447 // Skip the second pass if the first pass did not fail.
1448 bool StrategyFailed = false;
1449 // Mark all operand data as free to use.
1450 clearUsed();
1451 // We keep the original operand order for the FirstLane, so reorder the
1452 // rest of the lanes. We are visiting the nodes in a circular fashion,
1453 // using FirstLane as the center point and increasing the radius
1454 // distance.
1455 for (unsigned Distance = 1; Distance != NumLanes; ++Distance) {
1456 // Visit the lane on the right and then the lane on the left.
1457 for (int Direction : {+1, -1}) {
1458 int Lane = FirstLane + Direction * Distance;
1459 if (Lane < 0 || Lane >= (int)NumLanes)
1460 continue;
1461 int LastLane = Lane - Direction;
1462 assert(LastLane >= 0 && LastLane < (int)NumLanes &&(static_cast <bool> (LastLane >= 0 && LastLane
< (int)NumLanes && "Out of bounds") ? void (0) : __assert_fail
("LastLane >= 0 && LastLane < (int)NumLanes && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1463, __extension__ __PRETTY_FUNCTION__))
1463 "Out of bounds")(static_cast <bool> (LastLane >= 0 && LastLane
< (int)NumLanes && "Out of bounds") ? void (0) : __assert_fail
("LastLane >= 0 && LastLane < (int)NumLanes && \"Out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1463, __extension__ __PRETTY_FUNCTION__))
;
1464 // Look for a good match for each operand.
1465 for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1466 // Search for the operand that matches SortedOps[OpIdx][Lane-1].
1467 Optional<unsigned> BestIdx =
1468 getBestOperand(OpIdx, Lane, LastLane, ReorderingModes);
1469 // By not selecting a value, we allow the operands that follow to
1470 // select a better matching value. We will get a non-null value in
1471 // the next run of getBestOperand().
1472 if (BestIdx) {
1473 // Swap the current operand with the one returned by
1474 // getBestOperand().
1475 swap(OpIdx, BestIdx.getValue(), Lane);
1476 } else {
1477 // We failed to find a best operand, set mode to 'Failed'.
1478 ReorderingModes[OpIdx] = ReorderingMode::Failed;
1479 // Enable the second pass.
1480 StrategyFailed = true;
1481 }
1482 }
1483 }
1484 }
1485 // Skip second pass if the strategy did not fail.
1486 if (!StrategyFailed)
1487 break;
1488 }
1489 }
1490
1491#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1492 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) static StringRef getModeStr(ReorderingMode RMode) {
1493 switch (RMode) {
1494 case ReorderingMode::Load:
1495 return "Load";
1496 case ReorderingMode::Opcode:
1497 return "Opcode";
1498 case ReorderingMode::Constant:
1499 return "Constant";
1500 case ReorderingMode::Splat:
1501 return "Splat";
1502 case ReorderingMode::Failed:
1503 return "Failed";
1504 }
1505 llvm_unreachable("Unimplemented Reordering Type")::llvm::llvm_unreachable_internal("Unimplemented Reordering Type"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1505)
;
1506 }
1507
1508 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) static raw_ostream &printMode(ReorderingMode RMode,
1509 raw_ostream &OS) {
1510 return OS << getModeStr(RMode);
1511 }
1512
1513 /// Debug print.
1514 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) static void dumpMode(ReorderingMode RMode) {
1515 printMode(RMode, dbgs());
1516 }
1517
1518 friend raw_ostream &operator<<(raw_ostream &OS, ReorderingMode RMode) {
1519 return printMode(RMode, OS);
1520 }
1521
1522 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) raw_ostream &print(raw_ostream &OS) const {
1523 const unsigned Indent = 2;
1524 unsigned Cnt = 0;
1525 for (const OperandDataVec &OpDataVec : OpsVec) {
1526 OS << "Operand " << Cnt++ << "\n";
1527 for (const OperandData &OpData : OpDataVec) {
1528 OS.indent(Indent) << "{";
1529 if (Value *V = OpData.V)
1530 OS << *V;
1531 else
1532 OS << "null";
1533 OS << ", APO:" << OpData.APO << "}\n";
1534 }
1535 OS << "\n";
1536 }
1537 return OS;
1538 }
1539
1540 /// Debug print.
1541 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dump() const { print(dbgs()); }
1542#endif
1543 };
1544
1545 /// Checks if the instruction is marked for deletion.
1546 bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); }
1547
1548 /// Marks values operands for later deletion by replacing them with Undefs.
1549 void eraseInstructions(ArrayRef<Value *> AV);
1550
1551 ~BoUpSLP();
1552
1553private:
1554 /// Checks if all users of \p I are the part of the vectorization tree.
1555 bool areAllUsersVectorized(Instruction *I,
1556 ArrayRef<Value *> VectorizedVals) const;
1557
1558 /// \returns the cost of the vectorizable entry.
1559 InstructionCost getEntryCost(const TreeEntry *E,
1560 ArrayRef<Value *> VectorizedVals);
1561
1562 /// This is the recursive part of buildTree.
1563 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth,
1564 const EdgeInfo &EI);
1565
1566 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
1567 /// be vectorized to use the original vector (or aggregate "bitcast" to a
1568 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
1569 /// returns false, setting \p CurrentOrder to either an empty vector or a
1570 /// non-identity permutation that allows to reuse extract instructions.
1571 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1572 SmallVectorImpl<unsigned> &CurrentOrder) const;
1573
1574 /// Vectorize a single entry in the tree.
1575 Value *vectorizeTree(TreeEntry *E);
1576
1577 /// Vectorize a single entry in the tree, starting in \p VL.
1578 Value *vectorizeTree(ArrayRef<Value *> VL);
1579
1580 /// \returns the scalarization cost for this type. Scalarization in this
1581 /// context means the creation of vectors from a group of scalars.
1582 InstructionCost
1583 getGatherCost(FixedVectorType *Ty,
1584 const DenseSet<unsigned> &ShuffledIndices) const;
1585
1586 /// Checks if the gathered \p VL can be represented as shuffle(s) of previous
1587 /// tree entries.
1588 /// \returns ShuffleKind, if gathered values can be represented as shuffles of
1589 /// previous tree entries. \p Mask is filled with the shuffle mask.
1590 Optional<TargetTransformInfo::ShuffleKind>
1591 isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
1592 SmallVectorImpl<const TreeEntry *> &Entries);
1593
1594 /// \returns the scalarization cost for this list of values. Assuming that
1595 /// this subtree gets vectorized, we may need to extract the values from the
1596 /// roots. This method calculates the cost of extracting the values.
1597 InstructionCost getGatherCost(ArrayRef<Value *> VL) const;
1598
1599 /// Set the Builder insert point to one after the last instruction in
1600 /// the bundle
1601 void setInsertPointAfterBundle(const TreeEntry *E);
1602
1603 /// \returns a vector from a collection of scalars in \p VL.
1604 Value *gather(ArrayRef<Value *> VL);
1605
1606 /// \returns whether the VectorizableTree is fully vectorizable and will
1607 /// be beneficial even the tree height is tiny.
1608 bool isFullyVectorizableTinyTree() const;
1609
1610 /// Reorder commutative or alt operands to get better probability of
1611 /// generating vectorized code.
1612 static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
1613 SmallVectorImpl<Value *> &Left,
1614 SmallVectorImpl<Value *> &Right,
1615 const DataLayout &DL,
1616 ScalarEvolution &SE,
1617 const BoUpSLP &R);
1618 struct TreeEntry {
1619 using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
1620 TreeEntry(VecTreeTy &Container) : Container(Container) {}
1621
1622 /// \returns true if the scalars in VL are equal to this entry.
1623 bool isSame(ArrayRef<Value *> VL) const {
1624 if (VL.size() == Scalars.size())
1625 return std::equal(VL.begin(), VL.end(), Scalars.begin());
1626 return VL.size() == ReuseShuffleIndices.size() &&
1627 std::equal(
1628 VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
1629 [this](Value *V, int Idx) { return V == Scalars[Idx]; });
1630 }
1631
1632 /// A vector of scalars.
1633 ValueList Scalars;
1634
1635 /// The Scalars are vectorized into this value. It is initialized to Null.
1636 Value *VectorizedValue = nullptr;
1637
1638 /// Do we need to gather this sequence or vectorize it
1639 /// (either with vector instruction or with scatter/gather
1640 /// intrinsics for store/load)?
1641 enum EntryState { Vectorize, ScatterVectorize, NeedToGather };
1642 EntryState State;
1643
1644 /// Does this sequence require some shuffling?
1645 SmallVector<int, 4> ReuseShuffleIndices;
1646
1647 /// Does this entry require reordering?
1648 SmallVector<unsigned, 4> ReorderIndices;
1649
1650 /// Points back to the VectorizableTree.
1651 ///
1652 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
1653 /// to be a pointer and needs to be able to initialize the child iterator.
1654 /// Thus we need a reference back to the container to translate the indices
1655 /// to entries.
1656 VecTreeTy &Container;
1657
1658 /// The TreeEntry index containing the user of this entry. We can actually
1659 /// have multiple users so the data structure is not truly a tree.
1660 SmallVector<EdgeInfo, 1> UserTreeIndices;
1661
1662 /// The index of this treeEntry in VectorizableTree.
1663 int Idx = -1;
1664
1665 private:
1666 /// The operands of each instruction in each lane Operands[op_index][lane].
1667 /// Note: This helps avoid the replication of the code that performs the
1668 /// reordering of operands during buildTree_rec() and vectorizeTree().
1669 SmallVector<ValueList, 2> Operands;
1670
1671 /// The main/alternate instruction.
1672 Instruction *MainOp = nullptr;
1673 Instruction *AltOp = nullptr;
1674
1675 public:
1676 /// Set this bundle's \p OpIdx'th operand to \p OpVL.
1677 void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
1678 if (Operands.size() < OpIdx + 1)
1679 Operands.resize(OpIdx + 1);
1680 assert(Operands[OpIdx].empty() && "Already resized?")(static_cast <bool> (Operands[OpIdx].empty() &&
"Already resized?") ? void (0) : __assert_fail ("Operands[OpIdx].empty() && \"Already resized?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1680, __extension__ __PRETTY_FUNCTION__))
;
1681 Operands[OpIdx].resize(Scalars.size());
1682 for (unsigned Lane = 0, E = Scalars.size(); Lane != E; ++Lane)
1683 Operands[OpIdx][Lane] = OpVL[Lane];
1684 }
1685
1686 /// Set the operands of this bundle in their original order.
1687 void setOperandsInOrder() {
1688 assert(Operands.empty() && "Already initialized?")(static_cast <bool> (Operands.empty() && "Already initialized?"
) ? void (0) : __assert_fail ("Operands.empty() && \"Already initialized?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1688, __extension__ __PRETTY_FUNCTION__))
;
1689 auto *I0 = cast<Instruction>(Scalars[0]);
1690 Operands.resize(I0->getNumOperands());
1691 unsigned NumLanes = Scalars.size();
1692 for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands();
1693 OpIdx != NumOperands; ++OpIdx) {
1694 Operands[OpIdx].resize(NumLanes);
1695 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
1696 auto *I = cast<Instruction>(Scalars[Lane]);
1697 assert(I->getNumOperands() == NumOperands &&(static_cast <bool> (I->getNumOperands() == NumOperands
&& "Expected same number of operands") ? void (0) : __assert_fail
("I->getNumOperands() == NumOperands && \"Expected same number of operands\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1698, __extension__ __PRETTY_FUNCTION__))
1698 "Expected same number of operands")(static_cast <bool> (I->getNumOperands() == NumOperands
&& "Expected same number of operands") ? void (0) : __assert_fail
("I->getNumOperands() == NumOperands && \"Expected same number of operands\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1698, __extension__ __PRETTY_FUNCTION__))
;
1699 Operands[OpIdx][Lane] = I->getOperand(OpIdx);
1700 }
1701 }
1702 }
1703
1704 /// \returns the \p OpIdx operand of this TreeEntry.
1705 ValueList &getOperand(unsigned OpIdx) {
1706 assert(OpIdx < Operands.size() && "Off bounds")(static_cast <bool> (OpIdx < Operands.size() &&
"Off bounds") ? void (0) : __assert_fail ("OpIdx < Operands.size() && \"Off bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1706, __extension__ __PRETTY_FUNCTION__))
;
1707 return Operands[OpIdx];
1708 }
1709
1710 /// \returns the number of operands.
1711 unsigned getNumOperands() const { return Operands.size(); }
1712
1713 /// \return the single \p OpIdx operand.
1714 Value *getSingleOperand(unsigned OpIdx) const {
1715 assert(OpIdx < Operands.size() && "Off bounds")(static_cast <bool> (OpIdx < Operands.size() &&
"Off bounds") ? void (0) : __assert_fail ("OpIdx < Operands.size() && \"Off bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1715, __extension__ __PRETTY_FUNCTION__))
;
1716 assert(!Operands[OpIdx].empty() && "No operand available")(static_cast <bool> (!Operands[OpIdx].empty() &&
"No operand available") ? void (0) : __assert_fail ("!Operands[OpIdx].empty() && \"No operand available\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1716, __extension__ __PRETTY_FUNCTION__))
;
1717 return Operands[OpIdx][0];
1718 }
1719
1720 /// Some of the instructions in the list have alternate opcodes.
1721 bool isAltShuffle() const {
1722 return getOpcode() != getAltOpcode();
1723 }
1724
1725 bool isOpcodeOrAlt(Instruction *I) const {
1726 unsigned CheckedOpcode = I->getOpcode();
1727 return (getOpcode() == CheckedOpcode ||
1728 getAltOpcode() == CheckedOpcode);
1729 }
1730
1731 /// Chooses the correct key for scheduling data. If \p Op has the same (or
1732 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is
1733 /// \p OpValue.
1734 Value *isOneOf(Value *Op) const {
1735 auto *I = dyn_cast<Instruction>(Op);
1736 if (I && isOpcodeOrAlt(I))
1737 return Op;
1738 return MainOp;
1739 }
1740
1741 void setOperations(const InstructionsState &S) {
1742 MainOp = S.MainOp;
1743 AltOp = S.AltOp;
1744 }
1745
1746 Instruction *getMainOp() const {
1747 return MainOp;
1748 }
1749
1750 Instruction *getAltOp() const {
1751 return AltOp;
1752 }
1753
1754 /// The main/alternate opcodes for the list of instructions.
1755 unsigned getOpcode() const {
1756 return MainOp ? MainOp->getOpcode() : 0;
1757 }
1758
1759 unsigned getAltOpcode() const {
1760 return AltOp ? AltOp->getOpcode() : 0;
1761 }
1762
1763 /// Update operations state of this entry if reorder occurred.
1764 bool updateStateIfReorder() {
1765 if (ReorderIndices.empty())
1766 return false;
1767 InstructionsState S = getSameOpcode(Scalars, ReorderIndices.front());
1768 setOperations(S);
1769 return true;
1770 }
1771
1772#ifndef NDEBUG
1773 /// Debug printer.
1774 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dump() const {
1775 dbgs() << Idx << ".\n";
1776 for (unsigned OpI = 0, OpE = Operands.size(); OpI != OpE; ++OpI) {
1777 dbgs() << "Operand " << OpI << ":\n";
1778 for (const Value *V : Operands[OpI])
1779 dbgs().indent(2) << *V << "\n";
1780 }
1781 dbgs() << "Scalars: \n";
1782 for (Value *V : Scalars)
1783 dbgs().indent(2) << *V << "\n";
1784 dbgs() << "State: ";
1785 switch (State) {
1786 case Vectorize:
1787 dbgs() << "Vectorize\n";
1788 break;
1789 case ScatterVectorize:
1790 dbgs() << "ScatterVectorize\n";
1791 break;
1792 case NeedToGather:
1793 dbgs() << "NeedToGather\n";
1794 break;
1795 }
1796 dbgs() << "MainOp: ";
1797 if (MainOp)
1798 dbgs() << *MainOp << "\n";
1799 else
1800 dbgs() << "NULL\n";
1801 dbgs() << "AltOp: ";
1802 if (AltOp)
1803 dbgs() << *AltOp << "\n";
1804 else
1805 dbgs() << "NULL\n";
1806 dbgs() << "VectorizedValue: ";
1807 if (VectorizedValue)
1808 dbgs() << *VectorizedValue << "\n";
1809 else
1810 dbgs() << "NULL\n";
1811 dbgs() << "ReuseShuffleIndices: ";
1812 if (ReuseShuffleIndices.empty())
1813 dbgs() << "Empty";
1814 else
1815 for (unsigned ReuseIdx : ReuseShuffleIndices)
1816 dbgs() << ReuseIdx << ", ";
1817 dbgs() << "\n";
1818 dbgs() << "ReorderIndices: ";
1819 for (unsigned ReorderIdx : ReorderIndices)
1820 dbgs() << ReorderIdx << ", ";
1821 dbgs() << "\n";
1822 dbgs() << "UserTreeIndices: ";
1823 for (const auto &EInfo : UserTreeIndices)
1824 dbgs() << EInfo << ", ";
1825 dbgs() << "\n";
1826 }
1827#endif
1828 };
1829
1830#ifndef NDEBUG
1831 void dumpTreeCosts(const TreeEntry *E, InstructionCost ReuseShuffleCost,
1832 InstructionCost VecCost,
1833 InstructionCost ScalarCost) const {
1834 dbgs() << "SLP: Calculated costs for Tree:\n"; E->dump();
1835 dbgs() << "SLP: Costs:\n";
1836 dbgs() << "SLP: ReuseShuffleCost = " << ReuseShuffleCost << "\n";
1837 dbgs() << "SLP: VectorCost = " << VecCost << "\n";
1838 dbgs() << "SLP: ScalarCost = " << ScalarCost << "\n";
1839 dbgs() << "SLP: ReuseShuffleCost + VecCost - ScalarCost = " <<
1840 ReuseShuffleCost + VecCost - ScalarCost << "\n";
1841 }
1842#endif
1843
1844 /// Create a new VectorizableTree entry.
1845 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle,
1846 const InstructionsState &S,
1847 const EdgeInfo &UserTreeIdx,
1848 ArrayRef<unsigned> ReuseShuffleIndices = None,
1849 ArrayRef<unsigned> ReorderIndices = None) {
1850 TreeEntry::EntryState EntryState =
1851 Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
1852 return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
1853 ReuseShuffleIndices, ReorderIndices);
1854 }
1855
1856 TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
1857 TreeEntry::EntryState EntryState,
1858 Optional<ScheduleData *> Bundle,
1859 const InstructionsState &S,
1860 const EdgeInfo &UserTreeIdx,
1861 ArrayRef<unsigned> ReuseShuffleIndices = None,
1862 ArrayRef<unsigned> ReorderIndices = None) {
1863 assert(((!Bundle && EntryState == TreeEntry::NeedToGather) ||(static_cast <bool> (((!Bundle && EntryState ==
TreeEntry::NeedToGather) || (Bundle && EntryState !=
TreeEntry::NeedToGather)) && "Need to vectorize gather entry?"
) ? void (0) : __assert_fail ("((!Bundle && EntryState == TreeEntry::NeedToGather) || (Bundle && EntryState != TreeEntry::NeedToGather)) && \"Need to vectorize gather entry?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1865, __extension__ __PRETTY_FUNCTION__))
1864 (Bundle && EntryState != TreeEntry::NeedToGather)) &&(static_cast <bool> (((!Bundle && EntryState ==
TreeEntry::NeedToGather) || (Bundle && EntryState !=
TreeEntry::NeedToGather)) && "Need to vectorize gather entry?"
) ? void (0) : __assert_fail ("((!Bundle && EntryState == TreeEntry::NeedToGather) || (Bundle && EntryState != TreeEntry::NeedToGather)) && \"Need to vectorize gather entry?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1865, __extension__ __PRETTY_FUNCTION__))
1865 "Need to vectorize gather entry?")(static_cast <bool> (((!Bundle && EntryState ==
TreeEntry::NeedToGather) || (Bundle && EntryState !=
TreeEntry::NeedToGather)) && "Need to vectorize gather entry?"
) ? void (0) : __assert_fail ("((!Bundle && EntryState == TreeEntry::NeedToGather) || (Bundle && EntryState != TreeEntry::NeedToGather)) && \"Need to vectorize gather entry?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1865, __extension__ __PRETTY_FUNCTION__))
;
1866 VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree));
1867 TreeEntry *Last = VectorizableTree.back().get();
1868 Last->Idx = VectorizableTree.size() - 1;
1869 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
1870 Last->State = EntryState;
1871 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
1872 ReuseShuffleIndices.end());
1873 Last->ReorderIndices.append(ReorderIndices.begin(), ReorderIndices.end());
1874 Last->setOperations(S);
1875 if (Last->State != TreeEntry::NeedToGather) {
1876 for (Value *V : VL) {
1877 assert(!getTreeEntry(V) && "Scalar already in tree!")(static_cast <bool> (!getTreeEntry(V) && "Scalar already in tree!"
) ? void (0) : __assert_fail ("!getTreeEntry(V) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1877, __extension__ __PRETTY_FUNCTION__))
;
1878 ScalarToTreeEntry[V] = Last;
1879 }
1880 // Update the scheduler bundle to point to this TreeEntry.
1881 unsigned Lane = 0;
1882 for (ScheduleData *BundleMember = Bundle.getValue(); BundleMember;
1883 BundleMember = BundleMember->NextInBundle) {
1884 BundleMember->TE = Last;
1885 BundleMember->Lane = Lane;
1886 ++Lane;
1887 }
1888 assert((!Bundle.getValue() || Lane == VL.size()) &&(static_cast <bool> ((!Bundle.getValue() || Lane == VL.
size()) && "Bundle and VL out of sync") ? void (0) : __assert_fail
("(!Bundle.getValue() || Lane == VL.size()) && \"Bundle and VL out of sync\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1889, __extension__ __PRETTY_FUNCTION__))
1889 "Bundle and VL out of sync")(static_cast <bool> ((!Bundle.getValue() || Lane == VL.
size()) && "Bundle and VL out of sync") ? void (0) : __assert_fail
("(!Bundle.getValue() || Lane == VL.size()) && \"Bundle and VL out of sync\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1889, __extension__ __PRETTY_FUNCTION__))
;
1890 } else {
1891 MustGather.insert(VL.begin(), VL.end());
1892 }
1893
1894 if (UserTreeIdx.UserTE)
1895 Last->UserTreeIndices.push_back(UserTreeIdx);
1896
1897 return Last;
1898 }
1899
1900 /// -- Vectorization State --
1901 /// Holds all of the tree entries.
1902 TreeEntry::VecTreeTy VectorizableTree;
1903
1904#ifndef NDEBUG
1905 /// Debug printer.
1906 LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dumpVectorizableTree() const {
1907 for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) {
1908 VectorizableTree[Id]->dump();
1909 dbgs() << "\n";
1910 }
1911 }
1912#endif
1913
1914 TreeEntry *getTreeEntry(Value *V) { return ScalarToTreeEntry.lookup(V); }
1915
1916 const TreeEntry *getTreeEntry(Value *V) const {
1917 return ScalarToTreeEntry.lookup(V);
1918 }
1919
1920 /// Maps a specific scalar to its tree entry.
1921 SmallDenseMap<Value*, TreeEntry *> ScalarToTreeEntry;
1922
1923 /// Maps a value to the proposed vectorizable size.
1924 SmallDenseMap<Value *, unsigned> InstrElementSize;
1925
1926 /// A list of scalars that we found that we need to keep as scalars.
1927 ValueSet MustGather;
1928
1929 /// This POD struct describes one external user in the vectorized tree.
1930 struct ExternalUser {
1931 ExternalUser(Value *S, llvm::User *U, int L)
1932 : Scalar(S), User(U), Lane(L) {}
1933
1934 // Which scalar in our function.
1935 Value *Scalar;
1936
1937 // Which user that uses the scalar.
1938 llvm::User *User;
1939
1940 // Which lane does the scalar belong to.
1941 int Lane;
1942 };
1943 using UserList = SmallVector<ExternalUser, 16>;
1944
1945 /// Checks if two instructions may access the same memory.
1946 ///
1947 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
1948 /// is invariant in the calling loop.
1949 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
1950 Instruction *Inst2) {
1951 // First check if the result is already in the cache.
1952 AliasCacheKey key = std::make_pair(Inst1, Inst2);
1953 Optional<bool> &result = AliasCache[key];
1954 if (result.hasValue()) {
1955 return result.getValue();
1956 }
1957 MemoryLocation Loc2 = getLocation(Inst2, AA);
1958 bool aliased = true;
1959 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
1960 // Do the alias check.
1961 aliased = !AA->isNoAlias(Loc1, Loc2);
1962 }
1963 // Store the result in the cache.
1964 result = aliased;
1965 return aliased;
1966 }
1967
1968 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
1969
1970 /// Cache for alias results.
1971 /// TODO: consider moving this to the AliasAnalysis itself.
1972 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
1973
1974 /// Removes an instruction from its block and eventually deletes it.
1975 /// It's like Instruction::eraseFromParent() except that the actual deletion
1976 /// is delayed until BoUpSLP is destructed.
1977 /// This is required to ensure that there are no incorrect collisions in the
1978 /// AliasCache, which can happen if a new instruction is allocated at the
1979 /// same address as a previously deleted instruction.
1980 void eraseInstruction(Instruction *I, bool ReplaceOpsWithUndef = false) {
1981 auto It = DeletedInstructions.try_emplace(I, ReplaceOpsWithUndef).first;
1982 It->getSecond() = It->getSecond() && ReplaceOpsWithUndef;
1983 }
1984
1985 /// Temporary store for deleted instructions. Instructions will be deleted
1986 /// eventually when the BoUpSLP is destructed.
1987 DenseMap<Instruction *, bool> DeletedInstructions;
1988
1989 /// A list of values that need to extracted out of the tree.
1990 /// This list holds pairs of (Internal Scalar : External User). External User
1991 /// can be nullptr, it means that this Internal Scalar will be used later,
1992 /// after vectorization.
1993 UserList ExternalUses;
1994
1995 /// Values used only by @llvm.assume calls.
1996 SmallPtrSet<const Value *, 32> EphValues;
1997
1998 /// Holds all of the instructions that we gathered.
1999 SetVector<Instruction *> GatherSeq;
2000
2001 /// A list of blocks that we are going to CSE.
2002 SetVector<BasicBlock *> CSEBlocks;
2003
2004 /// Contains all scheduling relevant data for an instruction.
2005 /// A ScheduleData either represents a single instruction or a member of an
2006 /// instruction bundle (= a group of instructions which is combined into a
2007 /// vector instruction).
2008 struct ScheduleData {
2009 // The initial value for the dependency counters. It means that the
2010 // dependencies are not calculated yet.
2011 enum { InvalidDeps = -1 };
2012
2013 ScheduleData() = default;
2014
2015 void init(int BlockSchedulingRegionID, Value *OpVal) {
2016 FirstInBundle = this;
2017 NextInBundle = nullptr;
2018 NextLoadStore = nullptr;
2019 IsScheduled = false;
2020 SchedulingRegionID = BlockSchedulingRegionID;
2021 UnscheduledDepsInBundle = UnscheduledDeps;
2022 clearDependencies();
2023 OpValue = OpVal;
2024 TE = nullptr;
2025 Lane = -1;
2026 }
2027
2028 /// Returns true if the dependency information has been calculated.
2029 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
2030
2031 /// Returns true for single instructions and for bundle representatives
2032 /// (= the head of a bundle).
2033 bool isSchedulingEntity() const { return FirstInBundle == this; }
2034
2035 /// Returns true if it represents an instruction bundle and not only a
2036 /// single instruction.
2037 bool isPartOfBundle() const {
2038 return NextInBundle != nullptr || FirstInBundle != this;
2039 }
2040
2041 /// Returns true if it is ready for scheduling, i.e. it has no more
2042 /// unscheduled depending instructions/bundles.
2043 bool isReady() const {
2044 assert(isSchedulingEntity() &&(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2045, __extension__ __PRETTY_FUNCTION__))
2045 "can't consider non-scheduling entity for ready list")(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2045, __extension__ __PRETTY_FUNCTION__))
;
2046 return UnscheduledDepsInBundle == 0 && !IsScheduled;
2047 }
2048
2049 /// Modifies the number of unscheduled dependencies, also updating it for
2050 /// the whole bundle.
2051 int incrementUnscheduledDeps(int Incr) {
2052 UnscheduledDeps += Incr;
2053 return FirstInBundle->UnscheduledDepsInBundle += Incr;
2054 }
2055
2056 /// Sets the number of unscheduled dependencies to the number of
2057 /// dependencies.
2058 void resetUnscheduledDeps() {
2059 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
2060 }
2061
2062 /// Clears all dependency information.
2063 void clearDependencies() {
2064 Dependencies = InvalidDeps;
2065 resetUnscheduledDeps();
2066 MemoryDependencies.clear();
2067 }
2068
2069 void dump(raw_ostream &os) const {
2070 if (!isSchedulingEntity()) {
2071 os << "/ " << *Inst;
2072 } else if (NextInBundle) {
2073 os << '[' << *Inst;
2074 ScheduleData *SD = NextInBundle;
2075 while (SD) {
2076 os << ';' << *SD->Inst;
2077 SD = SD->NextInBundle;
2078 }
2079 os << ']';
2080 } else {
2081 os << *Inst;
2082 }
2083 }
2084
2085 Instruction *Inst = nullptr;
2086
2087 /// Points to the head in an instruction bundle (and always to this for
2088 /// single instructions).
2089 ScheduleData *FirstInBundle = nullptr;
2090
2091 /// Single linked list of all instructions in a bundle. Null if it is a
2092 /// single instruction.
2093 ScheduleData *NextInBundle = nullptr;
2094
2095 /// Single linked list of all memory instructions (e.g. load, store, call)
2096 /// in the block - until the end of the scheduling region.
2097 ScheduleData *NextLoadStore = nullptr;
2098
2099 /// The dependent memory instructions.
2100 /// This list is derived on demand in calculateDependencies().
2101 SmallVector<ScheduleData *, 4> MemoryDependencies;
2102
2103 /// This ScheduleData is in the current scheduling region if this matches
2104 /// the current SchedulingRegionID of BlockScheduling.
2105 int SchedulingRegionID = 0;
2106
2107 /// Used for getting a "good" final ordering of instructions.
2108 int SchedulingPriority = 0;
2109
2110 /// The number of dependencies. Constitutes of the number of users of the
2111 /// instruction plus the number of dependent memory instructions (if any).
2112 /// This value is calculated on demand.
2113 /// If InvalidDeps, the number of dependencies is not calculated yet.
2114 int Dependencies = InvalidDeps;
2115
2116 /// The number of dependencies minus the number of dependencies of scheduled
2117 /// instructions. As soon as this is zero, the instruction/bundle gets ready
2118 /// for scheduling.
2119 /// Note that this is negative as long as Dependencies is not calculated.
2120 int UnscheduledDeps = InvalidDeps;
2121
2122 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
2123 /// single instructions.
2124 int UnscheduledDepsInBundle = InvalidDeps;
2125
2126 /// True if this instruction is scheduled (or considered as scheduled in the
2127 /// dry-run).
2128 bool IsScheduled = false;
2129
2130 /// Opcode of the current instruction in the schedule data.
2131 Value *OpValue = nullptr;
2132
2133 /// The TreeEntry that this instruction corresponds to.
2134 TreeEntry *TE = nullptr;
2135
2136 /// The lane of this node in the TreeEntry.
2137 int Lane = -1;
2138 };
2139
2140#ifndef NDEBUG
2141 friend inline raw_ostream &operator<<(raw_ostream &os,
2142 const BoUpSLP::ScheduleData &SD) {
2143 SD.dump(os);
2144 return os;
2145 }
2146#endif
2147
2148 friend struct GraphTraits<BoUpSLP *>;
2149 friend struct DOTGraphTraits<BoUpSLP *>;
2150
2151 /// Contains all scheduling data for a basic block.
2152 struct BlockScheduling {
2153 BlockScheduling(BasicBlock *BB)
2154 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
2155
2156 void clear() {
2157 ReadyInsts.clear();
2158 ScheduleStart = nullptr;
2159 ScheduleEnd = nullptr;
2160 FirstLoadStoreInRegion = nullptr;
2161 LastLoadStoreInRegion = nullptr;
2162
2163 // Reduce the maximum schedule region size by the size of the
2164 // previous scheduling run.
2165 ScheduleRegionSizeLimit -= ScheduleRegionSize;
2166 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
2167 ScheduleRegionSizeLimit = MinScheduleRegionSize;
2168 ScheduleRegionSize = 0;
2169
2170 // Make a new scheduling region, i.e. all existing ScheduleData is not
2171 // in the new region yet.
2172 ++SchedulingRegionID;
2173 }
2174
2175 ScheduleData *getScheduleData(Value *V) {
2176 ScheduleData *SD = ScheduleDataMap[V];
2177 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
2178 return SD;
2179 return nullptr;
2180 }
2181
2182 ScheduleData *getScheduleData(Value *V, Value *Key) {
2183 if (V == Key)
2184 return getScheduleData(V);
2185 auto I = ExtraScheduleDataMap.find(V);
2186 if (I != ExtraScheduleDataMap.end()) {
2187 ScheduleData *SD = I->second[Key];
2188 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
2189 return SD;
2190 }
2191 return nullptr;
2192 }
2193
2194 bool isInSchedulingRegion(ScheduleData *SD) const {
2195 return SD->SchedulingRegionID == SchedulingRegionID;
2196 }
2197
2198 /// Marks an instruction as scheduled and puts all dependent ready
2199 /// instructions into the ready-list.
2200 template <typename ReadyListType>
2201 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
2202 SD->IsScheduled = true;
2203 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
2204
2205 ScheduleData *BundleMember = SD;
2206 while (BundleMember) {
2207 if (BundleMember->Inst != BundleMember->OpValue) {
2208 BundleMember = BundleMember->NextInBundle;
2209 continue;
2210 }
2211 // Handle the def-use chain dependencies.
2212
2213 // Decrement the unscheduled counter and insert to ready list if ready.
2214 auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
2215 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
2216 if (OpDef && OpDef->hasValidDependencies() &&
2217 OpDef->incrementUnscheduledDeps(-1) == 0) {
2218 // There are no more unscheduled dependencies after
2219 // decrementing, so we can put the dependent instruction
2220 // into the ready list.
2221 ScheduleData *DepBundle = OpDef->FirstInBundle;
2222 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2223, __extension__ __PRETTY_FUNCTION__))
2223 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2223, __extension__ __PRETTY_FUNCTION__))
;
2224 ReadyList.insert(DepBundle);
2225 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
2226 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
2227 }
2228 });
2229 };
2230
2231 // If BundleMember is a vector bundle, its operands may have been
2232 // reordered duiring buildTree(). We therefore need to get its operands
2233 // through the TreeEntry.
2234 if (TreeEntry *TE = BundleMember->TE) {
2235 int Lane = BundleMember->Lane;
2236 assert(Lane >= 0 && "Lane not set")(static_cast <bool> (Lane >= 0 && "Lane not set"
) ? void (0) : __assert_fail ("Lane >= 0 && \"Lane not set\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2236, __extension__ __PRETTY_FUNCTION__))
;
2237
2238 // Since vectorization tree is being built recursively this assertion
2239 // ensures that the tree entry has all operands set before reaching
2240 // this code. Couple of exceptions known at the moment are extracts
2241 // where their second (immediate) operand is not added. Since
2242 // immediates do not affect scheduler behavior this is considered
2243 // okay.
2244 auto *In = TE->getMainOp();
2245 assert(In &&(static_cast <bool> (In && (isa<ExtractValueInst
>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? void (0) : __assert_fail
("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2249, __extension__ __PRETTY_FUNCTION__))
2246 (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) ||(static_cast <bool> (In && (isa<ExtractValueInst
>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? void (0) : __assert_fail
("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2249, __extension__ __PRETTY_FUNCTION__))
2247 isa<InsertElementInst>(In) ||(static_cast <bool> (In && (isa<ExtractValueInst
>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? void (0) : __assert_fail
("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2249, __extension__ __PRETTY_FUNCTION__))
2248 In->getNumOperands() == TE->getNumOperands()) &&(static_cast <bool> (In && (isa<ExtractValueInst
>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? void (0) : __assert_fail
("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2249, __extension__ __PRETTY_FUNCTION__))
2249 "Missed TreeEntry operands?")(static_cast <bool> (In && (isa<ExtractValueInst
>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst
>(In) || In->getNumOperands() == TE->getNumOperands(
)) && "Missed TreeEntry operands?") ? void (0) : __assert_fail
("In && (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) || isa<InsertElementInst>(In) || In->getNumOperands() == TE->getNumOperands()) && \"Missed TreeEntry operands?\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2249, __extension__ __PRETTY_FUNCTION__))
;
2250 (void)In; // fake use to avoid build failure when assertions disabled
2251
2252 for (unsigned OpIdx = 0, NumOperands = TE->getNumOperands();
2253 OpIdx != NumOperands; ++OpIdx)
2254 if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane]))
2255 DecrUnsched(I);
2256 } else {
2257 // If BundleMember is a stand-alone instruction, no operand reordering
2258 // has taken place, so we directly access its operands.
2259 for (Use &U : BundleMember->Inst->operands())
2260 if (auto *I = dyn_cast<Instruction>(U.get()))
2261 DecrUnsched(I);
2262 }
2263 // Handle the memory dependencies.
2264 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
2265 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
2266 // There are no more unscheduled dependencies after decrementing,
2267 // so we can put the dependent instruction into the ready list.
2268 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
2269 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2270, __extension__ __PRETTY_FUNCTION__))
2270 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2270, __extension__ __PRETTY_FUNCTION__))
;
2271 ReadyList.insert(DepBundle);
2272 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
2273 << "SLP: gets ready (mem): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
2274 }
2275 }
2276 BundleMember = BundleMember->NextInBundle;
2277 }
2278 }
2279
2280 void doForAllOpcodes(Value *V,
2281 function_ref<void(ScheduleData *SD)> Action) {
2282 if (ScheduleData *SD = getScheduleData(V))
2283 Action(SD);
2284 auto I = ExtraScheduleDataMap.find(V);
2285 if (I != ExtraScheduleDataMap.end())
2286 for (auto &P : I->second)
2287 if (P.second->SchedulingRegionID == SchedulingRegionID)
2288 Action(P.second);
2289 }
2290
2291 /// Put all instructions into the ReadyList which are ready for scheduling.
2292 template <typename ReadyListType>
2293 void initialFillReadyList(ReadyListType &ReadyList) {
2294 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
2295 doForAllOpcodes(I, [&](ScheduleData *SD) {
2296 if (SD->isSchedulingEntity() && SD->isReady()) {
2297 ReadyList.insert(SD);
2298 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
2299 << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
2300 }
2301 });
2302 }
2303 }
2304
2305 /// Checks if a bundle of instructions can be scheduled, i.e. has no
2306 /// cyclic dependencies. This is only a dry-run, no instructions are
2307 /// actually moved at this stage.
2308 /// \returns the scheduling bundle. The returned Optional value is non-None
2309 /// if \p VL is allowed to be scheduled.
2310 Optional<ScheduleData *>
2311 tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
2312 const InstructionsState &S);
2313
2314 /// Un-bundles a group of instructions.
2315 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
2316
2317 /// Allocates schedule data chunk.
2318 ScheduleData *allocateScheduleDataChunks();
2319
2320 /// Extends the scheduling region so that V is inside the region.
2321 /// \returns true if the region size is within the limit.
2322 bool extendSchedulingRegion(Value *V, const InstructionsState &S);
2323
2324 /// Initialize the ScheduleData structures for new instructions in the
2325 /// scheduling region.
2326 void initScheduleData(Instruction *FromI, Instruction *ToI,
2327 ScheduleData *PrevLoadStore,
2328 ScheduleData *NextLoadStore);
2329
2330 /// Updates the dependency information of a bundle and of all instructions/
2331 /// bundles which depend on the original bundle.
2332 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
2333 BoUpSLP *SLP);
2334
2335 /// Sets all instruction in the scheduling region to un-scheduled.
2336 void resetSchedule();
2337
2338 BasicBlock *BB;
2339
2340 /// Simple memory allocation for ScheduleData.
2341 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
2342
2343 /// The size of a ScheduleData array in ScheduleDataChunks.
2344 int ChunkSize;
2345
2346 /// The allocator position in the current chunk, which is the last entry
2347 /// of ScheduleDataChunks.
2348 int ChunkPos;
2349
2350 /// Attaches ScheduleData to Instruction.
2351 /// Note that the mapping survives during all vectorization iterations, i.e.
2352 /// ScheduleData structures are recycled.
2353 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
2354
2355 /// Attaches ScheduleData to Instruction with the leading key.
2356 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
2357 ExtraScheduleDataMap;
2358
2359 struct ReadyList : SmallVector<ScheduleData *, 8> {
2360 void insert(ScheduleData *SD) { push_back(SD); }
2361 };
2362
2363 /// The ready-list for scheduling (only used for the dry-run).
2364 ReadyList ReadyInsts;
2365
2366 /// The first instruction of the scheduling region.
2367 Instruction *ScheduleStart = nullptr;
2368
2369 /// The first instruction _after_ the scheduling region.
2370 Instruction *ScheduleEnd = nullptr;
2371
2372 /// The first memory accessing instruction in the scheduling region
2373 /// (can be null).
2374 ScheduleData *FirstLoadStoreInRegion = nullptr;
2375
2376 /// The last memory accessing instruction in the scheduling region
2377 /// (can be null).
2378 ScheduleData *LastLoadStoreInRegion = nullptr;
2379
2380 /// The current size of the scheduling region.
2381 int ScheduleRegionSize = 0;
2382
2383 /// The maximum size allowed for the scheduling region.
2384 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
2385
2386 /// The ID of the scheduling region. For a new vectorization iteration this
2387 /// is incremented which "removes" all ScheduleData from the region.
2388 // Make sure that the initial SchedulingRegionID is greater than the
2389 // initial SchedulingRegionID in ScheduleData (which is 0).
2390 int SchedulingRegionID = 1;
2391 };
2392
2393 /// Attaches the BlockScheduling structures to basic blocks.
2394 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
2395
2396 /// Performs the "real" scheduling. Done before vectorization is actually
2397 /// performed in a basic block.
2398 void scheduleBlock(BlockScheduling *BS);
2399
2400 /// List of users to ignore during scheduling and that don't need extracting.
2401 ArrayRef<Value *> UserIgnoreList;
2402
2403 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
2404 /// sorted SmallVectors of unsigned.
2405 struct OrdersTypeDenseMapInfo {
2406 static OrdersType getEmptyKey() {
2407 OrdersType V;
2408 V.push_back(~1U);
2409 return V;
2410 }
2411
2412 static OrdersType getTombstoneKey() {
2413 OrdersType V;
2414 V.push_back(~2U);
2415 return V;
2416 }
2417
2418 static unsigned getHashValue(const OrdersType &V) {
2419 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
2420 }
2421
2422 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
2423 return LHS == RHS;
2424 }
2425 };
2426
2427 /// Contains orders of operations along with the number of bundles that have
2428 /// operations in this order. It stores only those orders that require
2429 /// reordering, if reordering is not required it is counted using \a
2430 /// NumOpsWantToKeepOriginalOrder.
2431 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder;
2432 /// Number of bundles that do not require reordering.
2433 unsigned NumOpsWantToKeepOriginalOrder = 0;
2434
2435 // Analysis and block reference.
2436 Function *F;
2437 ScalarEvolution *SE;
2438 TargetTransformInfo *TTI;
2439 TargetLibraryInfo *TLI;
2440 AAResults *AA;
2441 LoopInfo *LI;
2442 DominatorTree *DT;
2443 AssumptionCache *AC;
2444 DemandedBits *DB;
2445 const DataLayout *DL;
2446 OptimizationRemarkEmitter *ORE;
2447
2448 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
2449 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
2450
2451 /// Instruction builder to construct the vectorized tree.
2452 IRBuilder<> Builder;
2453
2454 /// A map of scalar integer values to the smallest bit width with which they
2455 /// can legally be represented. The values map to (width, signed) pairs,
2456 /// where "width" indicates the minimum bit width and "signed" is True if the
2457 /// value must be signed-extended, rather than zero-extended, back to its
2458 /// original width.
2459 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
2460};
2461
2462} // end namespace slpvectorizer
2463
2464template <> struct GraphTraits<BoUpSLP *> {
2465 using TreeEntry = BoUpSLP::TreeEntry;
2466
2467 /// NodeRef has to be a pointer per the GraphWriter.
2468 using NodeRef = TreeEntry *;
2469
2470 using ContainerTy = BoUpSLP::TreeEntry::VecTreeTy;
2471
2472 /// Add the VectorizableTree to the index iterator to be able to return
2473 /// TreeEntry pointers.
2474 struct ChildIteratorType
2475 : public iterator_adaptor_base<
2476 ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> {
2477 ContainerTy &VectorizableTree;
2478
2479 ChildIteratorType(SmallVector<BoUpSLP::EdgeInfo, 1>::iterator W,
2480 ContainerTy &VT)
2481 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
2482
2483 NodeRef operator*() { return I->UserTE; }
2484 };
2485
2486 static NodeRef getEntryNode(BoUpSLP &R) {
2487 return R.VectorizableTree[0].get();
2488 }
2489
2490 static ChildIteratorType child_begin(NodeRef N) {
2491 return {N->UserTreeIndices.begin(), N->Container};
2492 }
2493
2494 static ChildIteratorType child_end(NodeRef N) {
2495 return {N->UserTreeIndices.end(), N->Container};
2496 }
2497
2498 /// For the node iterator we just need to turn the TreeEntry iterator into a
2499 /// TreeEntry* iterator so that it dereferences to NodeRef.
2500 class nodes_iterator {
2501 using ItTy = ContainerTy::iterator;
2502 ItTy It;
2503
2504 public:
2505 nodes_iterator(const ItTy &It2) : It(It2) {}
2506 NodeRef operator*() { return It->get(); }
2507 nodes_iterator operator++() {
2508 ++It;
2509 return *this;
2510 }
2511 bool operator!=(const nodes_iterator &N2) const { return N2.It != It; }
2512 };
2513
2514 static nodes_iterator nodes_begin(BoUpSLP *R) {
2515 return nodes_iterator(R->VectorizableTree.begin());
2516 }
2517
2518 static nodes_iterator nodes_end(BoUpSLP *R) {
2519 return nodes_iterator(R->VectorizableTree.end());
2520 }
2521
2522 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
2523};
2524
2525template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
2526 using TreeEntry = BoUpSLP::TreeEntry;
2527
2528 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
2529
2530 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
2531 std::string Str;
2532 raw_string_ostream OS(Str);
2533 if (isSplat(Entry->Scalars)) {
2534 OS << "<splat> " << *Entry->Scalars[0];
2535 return Str;
2536 }
2537 for (auto V : Entry->Scalars) {
2538 OS << *V;
2539 if (llvm::any_of(R->ExternalUses, [&](const BoUpSLP::ExternalUser &EU) {
2540 return EU.Scalar == V;
2541 }))
2542 OS << " <extract>";
2543 OS << "\n";
2544 }
2545 return Str;
2546 }
2547
2548 static std::string getNodeAttributes(const TreeEntry *Entry,
2549 const BoUpSLP *) {
2550 if (Entry->State == TreeEntry::NeedToGather)
2551 return "color=red";
2552 return "";
2553 }
2554};
2555
2556} // end namespace llvm
2557
2558BoUpSLP::~BoUpSLP() {
2559 for (const auto &Pair : DeletedInstructions) {
2560 // Replace operands of ignored instructions with Undefs in case if they were
2561 // marked for deletion.
2562 if (Pair.getSecond()) {
2563 Value *Undef = UndefValue::get(Pair.getFirst()->getType());
2564 Pair.getFirst()->replaceAllUsesWith(Undef);
2565 }
2566 Pair.getFirst()->dropAllReferences();
2567 }
2568 for (const auto &Pair : DeletedInstructions) {
2569 assert(Pair.getFirst()->use_empty() &&(static_cast <bool> (Pair.getFirst()->use_empty() &&
"trying to erase instruction with users.") ? void (0) : __assert_fail
("Pair.getFirst()->use_empty() && \"trying to erase instruction with users.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2570, __extension__ __PRETTY_FUNCTION__))
2570 "trying to erase instruction with users.")(static_cast <bool> (Pair.getFirst()->use_empty() &&
"trying to erase instruction with users.") ? void (0) : __assert_fail
("Pair.getFirst()->use_empty() && \"trying to erase instruction with users.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2570, __extension__ __PRETTY_FUNCTION__))
;
2571 Pair.getFirst()->eraseFromParent();
2572 }
2573#ifdef EXPENSIVE_CHECKS
2574 // If we could guarantee that this call is not extremely slow, we could
2575 // remove the ifdef limitation (see PR47712).
2576 assert(!verifyFunction(*F, &dbgs()))(static_cast <bool> (!verifyFunction(*F, &dbgs())) ?
void (0) : __assert_fail ("!verifyFunction(*F, &dbgs())"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2576, __extension__ __PRETTY_FUNCTION__))
;
2577#endif
2578}
2579
2580void BoUpSLP::eraseInstructions(ArrayRef<Value *> AV) {
2581 for (auto *V : AV) {
2582 if (auto *I = dyn_cast<Instruction>(V))
2583 eraseInstruction(I, /*ReplaceOpsWithUndef=*/true);
2584 };
2585}
2586
2587void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
2588 ArrayRef<Value *> UserIgnoreLst) {
2589 ExtraValueToDebugLocsMap ExternallyUsedValues;
2590 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
2591}
2592
2593static int findLaneForValue(ArrayRef<Value *> Scalars,
2594 ArrayRef<int> ReuseShuffleIndices, Value *V) {
2595 unsigned FoundLane = std::distance(Scalars.begin(), find(Scalars, V));
2596 assert(FoundLane < Scalars.size() && "Couldn't find extract lane")(static_cast <bool> (FoundLane < Scalars.size() &&
"Couldn't find extract lane") ? void (0) : __assert_fail ("FoundLane < Scalars.size() && \"Couldn't find extract lane\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2596, __extension__ __PRETTY_FUNCTION__))
;
2597 if (!ReuseShuffleIndices.empty()) {
2598 FoundLane = std::distance(ReuseShuffleIndices.begin(),
2599 find(ReuseShuffleIndices, FoundLane));
2600 }
2601 return FoundLane;
2602}
2603
2604void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
2605 ExtraValueToDebugLocsMap &ExternallyUsedValues,
2606 ArrayRef<Value *> UserIgnoreLst) {
2607 deleteTree();
2608 UserIgnoreList = UserIgnoreLst;
2609 if (!allSameType(Roots))
2610 return;
2611 buildTree_rec(Roots, 0, EdgeInfo());
2612
2613 // Collect the values that we need to extract from the tree.
2614 for (auto &TEPtr : VectorizableTree) {
2615 TreeEntry *Entry = TEPtr.get();
2616
2617 // No need to handle users of gathered values.
2618 if (Entry->State == TreeEntry::NeedToGather)
2619 continue;
2620
2621 // For each lane:
2622 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
2623 Value *Scalar = Entry->Scalars[Lane];
2624 int FoundLane =
2625 findLaneForValue(Entry->Scalars, Entry->ReuseShuffleIndices, Scalar);
2626
2627 // Check if the scalar is externally used as an extra arg.
2628 auto ExtI = ExternallyUsedValues.find(Scalar);
2629 if (ExtI != ExternallyUsedValues.end()) {
2630 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
2631 << Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
2632 ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
2633 }
2634 for (User *U : Scalar->users()) {
2635 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
2636
2637 Instruction *UserInst = dyn_cast<Instruction>(U);
2638 if (!UserInst)
2639 continue;
2640
2641 // Skip in-tree scalars that become vectors
2642 if (TreeEntry *UseEntry = getTreeEntry(U)) {
2643 Value *UseScalar = UseEntry->Scalars[0];
2644 // Some in-tree scalars will remain as scalar in vectorized
2645 // instructions. If that is the case, the one in Lane 0 will
2646 // be used.
2647 if (UseScalar != U ||
2648 UseEntry->State == TreeEntry::ScatterVectorize ||
2649 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
2650 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
2651 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
2652 assert(UseEntry->State != TreeEntry::NeedToGather && "Bad state")(static_cast <bool> (UseEntry->State != TreeEntry::NeedToGather
&& "Bad state") ? void (0) : __assert_fail ("UseEntry->State != TreeEntry::NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2652, __extension__ __PRETTY_FUNCTION__))
;
2653 continue;
2654 }
2655 }
2656
2657 // Ignore users in the user ignore list.
2658 if (is_contained(UserIgnoreList, UserInst))
2659 continue;
2660
2661 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
2662 << Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
2663 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
2664 }
2665 }
2666 }
2667}
2668
2669void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
2670 const EdgeInfo &UserTreeIdx) {
2671 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(static_cast <bool> ((allConstant(VL) || allSameType(VL
)) && "Invalid types!") ? void (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2671, __extension__ __PRETTY_FUNCTION__))
;
1
Assuming the condition is true
2
'?' condition is true
2672
2673 InstructionsState S = getSameOpcode(VL);
2674 if (Depth == RecursionMaxDepth) {
3
Assuming the condition is false
4
Taking false branch
2675 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
2676 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2677 return;
2678 }
2679
2680 // Don't handle vectors.
2681 if (S.OpValue->getType()->isVectorTy() &&
2682 !isa<InsertElementInst>(S.OpValue)) {
2683 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
2684 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2685 return;
2686 }
2687
2688 if (StoreInst *SI
5.1
'SI' is null
5.1
'SI' is null
= dyn_cast<StoreInst>(S.OpValue))
5
Assuming field 'OpValue' is not a 'StoreInst'
6
Taking false branch
2689 if (SI->getValueOperand()->getType()->isVectorTy()) {
2690 LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
2691 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2692 return;
2693 }
2694
2695 // If all of the operands are identical or constant we have a simple solution.
2696 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
7
Assuming the condition is false
8
Assuming the condition is false
9
Taking false branch
2697 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
2698 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2699 return;
2700 }
2701
2702 // We now know that this is a vector of instructions of the same type from
2703 // the same block.
2704
2705 // Don't vectorize ephemeral values.
2706 for (Value *V : VL) {
10
Assuming '__begin1' is equal to '__end1'
2707 if (EphValues.count(V)) {
2708 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *Vdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is ephemeral.\n"; } } while (false)
2709 << ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is ephemeral.\n"; } } while (false)
;
2710 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2711 return;
2712 }
2713 }
2714
2715 // Check if this is a duplicate of another entry.
2716 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
11
Assuming 'E' is null
12
Taking false branch
2717 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*S.OpValue << ".\n"; } } while (false)
;
2718 if (!E->isSame(VL)) {
2719 LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
2720 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2721 return;
2722 }
2723 // Record the reuse of the tree node. FIXME, currently this is only used to
2724 // properly draw the graph rather than for the actual vectorization.
2725 E->UserTreeIndices.push_back(UserTreeIdx);
2726 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValuedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
2727 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
;
2728 return;
2729 }
2730
2731 // Check that none of the instructions in the bundle are already in the tree.
2732 for (Value *V : VL) {
13
Assuming '__begin1' is equal to '__end1'
2733 auto *I = dyn_cast<Instruction>(V);
2734 if (!I)
2735 continue;
2736 if (getTreeEntry(I)) {
2737 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *Vdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is already in tree.\n"; } } while (false)
2738 << ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
V << ") is already in tree.\n"; } } while (false)
;
2739 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2740 return;
2741 }
2742 }
2743
2744 // If any of the scalars is marked as a value that needs to stay scalar, then
2745 // we need to gather the scalars.
2746 // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
2747 for (Value *V : VL) {
14
Assuming '__begin1' is equal to '__end1'
2748 if (MustGather.count(V) || is_contained(UserIgnoreList, V)) {
2749 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
2750 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2751 return;
2752 }
2753 }
2754
2755 // Check that all of the users of the scalars that we want to vectorize are
2756 // schedulable.
2757 auto *VL0 = cast<Instruction>(S.OpValue);
15
Field 'OpValue' is a 'Instruction'
2758 BasicBlock *BB = VL0->getParent();
2759
2760 if (!DT->isReachableFromEntry(BB)) {
16
Assuming the condition is false
17
Taking false branch
2761 // Don't go into unreachable blocks. They may contain instructions with
2762 // dependency cycles which confuse the final scheduling.
2763 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
2764 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2765 return;
2766 }
2767
2768 // Check that every instruction appears once in this bundle.
2769 SmallVector<unsigned, 4> ReuseShuffleIndicies;
2770 SmallVector<Value *, 4> UniqueValues;
2771 DenseMap<Value *, unsigned> UniquePositions;
2772 for (Value *V : VL) {
18
Assuming '__begin1' is equal to '__end1'
2773 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2774 ReuseShuffleIndicies.emplace_back(Res.first->second);
2775 if (Res.second)
2776 UniqueValues.emplace_back(V);
2777 }
2778 size_t NumUniqueScalarValues = UniqueValues.size();
2779 if (NumUniqueScalarValues == VL.size()) {
19
Assuming the condition is true
20
Taking true branch
2780 ReuseShuffleIndicies.clear();
2781 } else {
2782 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Shuffle for reused scalars.\n"
; } } while (false)
;
2783 if (NumUniqueScalarValues <= 1 ||
2784 !llvm::isPowerOf2_32(NumUniqueScalarValues)) {
2785 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
2786 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
2787 return;
2788 }
2789 VL = UniqueValues;
2790 }
2791
2792 auto &BSRef = BlocksSchedules[BB];
2793 if (!BSRef)
21
Taking false branch
2794 BSRef = std::make_unique<BlockScheduling>(BB);
2795
2796 BlockScheduling &BS = *BSRef.get();
2797
2798 Optional<ScheduleData *> Bundle = BS.tryScheduleBundle(VL, this, S);
22
Calling 'BlockScheduling::tryScheduleBundle'
2799 if (!Bundle) {
2800 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
2801 assert((!BS.getScheduleData(VL0) ||(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2803, __extension__ __PRETTY_FUNCTION__))
2802 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2803, __extension__ __PRETTY_FUNCTION__))
2803 "tryScheduleBundle should cancelScheduling on failure")(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2803, __extension__ __PRETTY_FUNCTION__))
;
2804 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2805 ReuseShuffleIndicies);
2806 return;
2807 }
2808 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
2809
2810 unsigned ShuffleOrOp = S.isAltShuffle() ?
2811 (unsigned) Instruction::ShuffleVector : S.getOpcode();
2812 switch (ShuffleOrOp) {
2813 case Instruction::PHI: {
2814 auto *PH = cast<PHINode>(VL0);
2815
2816 // Check for terminator values (e.g. invoke).
2817 for (Value *V : VL)
2818 for (unsigned I = 0, E = PH->getNumIncomingValues(); I < E; ++I) {
2819 Instruction *Term = dyn_cast<Instruction>(
2820 cast<PHINode>(V)->getIncomingValueForBlock(
2821 PH->getIncomingBlock(I)));
2822 if (Term && Term->isTerminator()) {
2823 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (terminator use).\n"
; } } while (false)
2824 << "SLP: Need to swizzle PHINodes (terminator use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (terminator use).\n"
; } } while (false)
;
2825 BS.cancelScheduling(VL, VL0);
2826 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2827 ReuseShuffleIndicies);
2828 return;
2829 }
2830 }
2831
2832 TreeEntry *TE =
2833 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies);
2834 LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
2835
2836 // Keeps the reordered operands to avoid code duplication.
2837 SmallVector<ValueList, 2> OperandsVec;
2838 for (unsigned I = 0, E = PH->getNumIncomingValues(); I < E; ++I) {
2839 if (!DT->isReachableFromEntry(PH->getIncomingBlock(I))) {
2840 ValueList Operands(VL.size(), PoisonValue::get(PH->getType()));
2841 TE->setOperand(I, Operands);
2842 OperandsVec.push_back(Operands);
2843 continue;
2844 }
2845 ValueList Operands;
2846 // Prepare the operand vector.
2847 for (Value *V : VL)
2848 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(
2849 PH->getIncomingBlock(I)));
2850 TE->setOperand(I, Operands);
2851 OperandsVec.push_back(Operands);
2852 }
2853 for (unsigned OpIdx = 0, OpE = OperandsVec.size(); OpIdx != OpE; ++OpIdx)
2854 buildTree_rec(OperandsVec[OpIdx], Depth + 1, {TE, OpIdx});
2855 return;
2856 }
2857 case Instruction::ExtractValue:
2858 case Instruction::ExtractElement: {
2859 OrdersType CurrentOrder;
2860 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
2861 if (Reuse) {
2862 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing or shuffling extract sequence.\n"
; } } while (false)
;
2863 ++NumOpsWantToKeepOriginalOrder;
2864 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2865 ReuseShuffleIndicies);
2866 // This is a special case, as it does not gather, but at the same time
2867 // we are not extending buildTree_rec() towards the operands.
2868 ValueList Op0;
2869 Op0.assign(VL.size(), VL0->getOperand(0));
2870 VectorizableTree.back()->setOperand(0, Op0);
2871 return;
2872 }
2873 if (!CurrentOrder.empty()) {
2874 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2875 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2876 "with order";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2877 for (unsigned Idx : CurrentOrder)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2878 dbgs() << " " << Idx;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2879 dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
2880 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
;
2881 // Insert new order with initial value 0, if it does not exist,
2882 // otherwise return the iterator to the existing one.
2883 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2884 ReuseShuffleIndicies, CurrentOrder);
2885 findRootOrder(CurrentOrder);
2886 ++NumOpsWantToKeepOrder[CurrentOrder];
2887 // This is a special case, as it does not gather, but at the same time
2888 // we are not extending buildTree_rec() towards the operands.
2889 ValueList Op0;
2890 Op0.assign(VL.size(), VL0->getOperand(0));
2891 VectorizableTree.back()->setOperand(0, Op0);
2892 return;
2893 }
2894 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gather extract sequence.\n";
} } while (false)
;
2895 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2896 ReuseShuffleIndicies);
2897 BS.cancelScheduling(VL, VL0);
2898 return;
2899 }
2900 case Instruction::InsertElement: {
2901 assert(ReuseShuffleIndicies.empty() && "All inserts should be unique")(static_cast <bool> (ReuseShuffleIndicies.empty() &&
"All inserts should be unique") ? void (0) : __assert_fail (
"ReuseShuffleIndicies.empty() && \"All inserts should be unique\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2901, __extension__ __PRETTY_FUNCTION__))
;
2902
2903 // Check that we have a buildvector and not a shuffle of 2 or more
2904 // different vectors.
2905 ValueSet SourceVectors;
2906 for (Value *V : VL)
2907 SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
2908
2909 if (count_if(VL, [&SourceVectors](Value *V) {
2910 return !SourceVectors.contains(V);
2911 }) >= 2) {
2912 // Found 2nd source vector - cancel.
2913 LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gather of insertelement vectors with "
"different source vectors.\n"; } } while (false)
2914 "different source vectors.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gather of insertelement vectors with "
"different source vectors.\n"; } } while (false)
;
2915 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2916 ReuseShuffleIndicies);
2917 BS.cancelScheduling(VL, VL0);
2918 return;
2919 }
2920
2921 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx);
2922 LLVM_DEBUG(dbgs() << "SLP: added inserts bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added inserts bundle.\n"; } }
while (false)
;
2923
2924 constexpr int NumOps = 2;
2925 ValueList VectorOperands[NumOps];
2926 for (int I = 0; I < NumOps; ++I) {
2927 for (Value *V : VL)
2928 VectorOperands[I].push_back(cast<Instruction>(V)->getOperand(I));
2929
2930 TE->setOperand(I, VectorOperands[I]);
2931 }
2932 buildTree_rec(VectorOperands[NumOps - 1], Depth + 1, {TE, 0});
2933 return;
2934 }
2935 case Instruction::Load: {
2936 // Check that a vectorized load would load the same memory as a scalar
2937 // load. For example, we don't want to vectorize loads that are smaller
2938 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
2939 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
2940 // from such a struct, we read/write packed bits disagreeing with the
2941 // unvectorized version.
2942 Type *ScalarTy = VL0->getType();
2943
2944 if (DL->getTypeSizeInBits(ScalarTy) !=
2945 DL->getTypeAllocSizeInBits(ScalarTy)) {
2946 BS.cancelScheduling(VL, VL0);
2947 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2948 ReuseShuffleIndicies);
2949 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
2950 return;
2951 }
2952
2953 // Make sure all loads in the bundle are simple - we can't vectorize
2954 // atomic or volatile loads.
2955 SmallVector<Value *, 4> PointerOps(VL.size());
2956 auto POIter = PointerOps.begin();
2957 for (Value *V : VL) {
2958 auto *L = cast<LoadInst>(V);
2959 if (!L->isSimple()) {
2960 BS.cancelScheduling(VL, VL0);
2961 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
2962 ReuseShuffleIndicies);
2963 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
2964 return;
2965 }
2966 *POIter = L->getPointerOperand();
2967 ++POIter;
2968 }
2969
2970 OrdersType CurrentOrder;
2971 // Check the order of pointer operands.
2972 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
2973 Value *Ptr0;
2974 Value *PtrN;
2975 if (CurrentOrder.empty()) {
2976 Ptr0 = PointerOps.front();
2977 PtrN = PointerOps.back();
2978 } else {
2979 Ptr0 = PointerOps[CurrentOrder.front()];
2980 PtrN = PointerOps[CurrentOrder.back()];
2981 }
2982 Optional<int> Diff = getPointersDiff(Ptr0, PtrN, *DL, *SE);
2983 // Check that the sorted loads are consecutive.
2984 if (static_cast<unsigned>(*Diff) == VL.size() - 1) {
2985 if (CurrentOrder.empty()) {
2986 // Original loads are consecutive and does not require reordering.
2987 ++NumOpsWantToKeepOriginalOrder;
2988 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S,
2989 UserTreeIdx, ReuseShuffleIndicies);
2990 TE->setOperandsInOrder();
2991 LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
2992 } else {
2993 // Need to reorder.
2994 TreeEntry *TE =
2995 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
2996 ReuseShuffleIndicies, CurrentOrder);
2997 TE->setOperandsInOrder();
2998 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of jumbled loads.\n"
; } } while (false)
;
2999 findRootOrder(CurrentOrder);
3000 ++NumOpsWantToKeepOrder[CurrentOrder];
3001 }
3002 return;
3003 }
3004 // Vectorizing non-consecutive loads with `llvm.masked.gather`.
3005 TreeEntry *TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
3006 UserTreeIdx, ReuseShuffleIndicies);
3007 TE->setOperandsInOrder();
3008 buildTree_rec(PointerOps, Depth + 1, {TE, 0});
3009 LLVM_DEBUG(dbgs() << "SLP: added a vector of non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of non-consecutive loads.\n"
; } } while (false)
;
3010 return;
3011 }
3012
3013 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
3014 BS.cancelScheduling(VL, VL0);
3015 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3016 ReuseShuffleIndicies);
3017 return;
3018 }
3019 case Instruction::ZExt:
3020 case Instruction::SExt:
3021 case Instruction::FPToUI:
3022 case Instruction::FPToSI:
3023 case Instruction::FPExt:
3024 case Instruction::PtrToInt:
3025 case Instruction::IntToPtr:
3026 case Instruction::SIToFP:
3027 case Instruction::UIToFP:
3028 case Instruction::Trunc:
3029 case Instruction::FPTrunc:
3030 case Instruction::BitCast: {
3031 Type *SrcTy = VL0->getOperand(0)->getType();
3032 for (Value *V : VL) {
3033 Type *Ty = cast<Instruction>(V)->getOperand(0)->getType();
3034 if (Ty != SrcTy || !isValidElementType(Ty)) {
3035 BS.cancelScheduling(VL, VL0);
3036 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3037 ReuseShuffleIndicies);
3038 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
3039 << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
3040 return;
3041 }
3042 }
3043 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3044 ReuseShuffleIndicies);
3045 LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
3046
3047 TE->setOperandsInOrder();
3048 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3049 ValueList Operands;
3050 // Prepare the operand vector.
3051 for (Value *V : VL)
3052 Operands.push_back(cast<Instruction>(V)->getOperand(i));
3053
3054 buildTree_rec(Operands, Depth + 1, {TE, i});
3055 }
3056 return;
3057 }
3058 case Instruction::ICmp:
3059 case Instruction::FCmp: {
3060 // Check that all of the compares have the same predicate.
3061 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3062 CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0);
3063 Type *ComparedTy = VL0->getOperand(0)->getType();
3064 for (Value *V : VL) {
3065 CmpInst *Cmp = cast<CmpInst>(V);
3066 if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) ||
3067 Cmp->getOperand(0)->getType() != ComparedTy) {
3068 BS.cancelScheduling(VL, VL0);
3069 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3070 ReuseShuffleIndicies);
3071 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
3072 << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
3073 return;
3074 }
3075 }
3076
3077 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3078 ReuseShuffleIndicies);
3079 LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
3080
3081 ValueList Left, Right;
3082 if (cast<CmpInst>(VL0)->isCommutative()) {
3083 // Commutative predicate - collect + sort operands of the instructions
3084 // so that each side is more likely to have the same opcode.
3085 assert(P0 == SwapP0 && "Commutative Predicate mismatch")(static_cast <bool> (P0 == SwapP0 && "Commutative Predicate mismatch"
) ? void (0) : __assert_fail ("P0 == SwapP0 && \"Commutative Predicate mismatch\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3085, __extension__ __PRETTY_FUNCTION__))
;
3086 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3087 } else {
3088 // Collect operands - commute if it uses the swapped predicate.
3089 for (Value *V : VL) {
3090 auto *Cmp = cast<CmpInst>(V);
3091 Value *LHS = Cmp->getOperand(0);
3092 Value *RHS = Cmp->getOperand(1);
3093 if (Cmp->getPredicate() != P0)
3094 std::swap(LHS, RHS);
3095 Left.push_back(LHS);
3096 Right.push_back(RHS);
3097 }
3098 }
3099 TE->setOperand(0, Left);
3100 TE->setOperand(1, Right);
3101 buildTree_rec(Left, Depth + 1, {TE, 0});
3102 buildTree_rec(Right, Depth + 1, {TE, 1});
3103 return;
3104 }
3105 case Instruction::Select:
3106 case Instruction::FNeg:
3107 case Instruction::Add:
3108 case Instruction::FAdd:
3109 case Instruction::Sub:
3110 case Instruction::FSub:
3111 case Instruction::Mul:
3112 case Instruction::FMul:
3113 case Instruction::UDiv:
3114 case Instruction::SDiv:
3115 case Instruction::FDiv:
3116 case Instruction::URem:
3117 case Instruction::SRem:
3118 case Instruction::FRem:
3119 case Instruction::Shl:
3120 case Instruction::LShr:
3121 case Instruction::AShr:
3122 case Instruction::And:
3123 case Instruction::Or:
3124 case Instruction::Xor: {
3125 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3126 ReuseShuffleIndicies);
3127 LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of un/bin op.\n"
; } } while (false)
;
3128
3129 // Sort operands of the instructions so that each side is more likely to
3130 // have the same opcode.
3131 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
3132 ValueList Left, Right;
3133 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3134 TE->setOperand(0, Left);
3135 TE->setOperand(1, Right);
3136 buildTree_rec(Left, Depth + 1, {TE, 0});
3137 buildTree_rec(Right, Depth + 1, {TE, 1});
3138 return;
3139 }
3140
3141 TE->setOperandsInOrder();
3142 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3143 ValueList Operands;
3144 // Prepare the operand vector.
3145 for (Value *V : VL)
3146 Operands.push_back(cast<Instruction>(V)->getOperand(i));
3147
3148 buildTree_rec(Operands, Depth + 1, {TE, i});
3149 }
3150 return;
3151 }
3152 case Instruction::GetElementPtr: {
3153 // We don't combine GEPs with complicated (nested) indexing.
3154 for (Value *V : VL) {
3155 if (cast<Instruction>(V)->getNumOperands() != 2) {
3156 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
3157 BS.cancelScheduling(VL, VL0);
3158 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3159 ReuseShuffleIndicies);
3160 return;
3161 }
3162 }
3163
3164 // We can't combine several GEPs into one vector if they operate on
3165 // different types.
3166 Type *Ty0 = VL0->getOperand(0)->getType();
3167 for (Value *V : VL) {
3168 Type *CurTy = cast<Instruction>(V)->getOperand(0)->getType();
3169 if (Ty0 != CurTy) {
3170 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
3171 << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
3172 BS.cancelScheduling(VL, VL0);
3173 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3174 ReuseShuffleIndicies);
3175 return;
3176 }
3177 }
3178
3179 // We don't combine GEPs with non-constant indexes.
3180 Type *Ty1 = VL0->getOperand(1)->getType();
3181 for (Value *V : VL) {
3182 auto Op = cast<Instruction>(V)->getOperand(1);
3183 if (!isa<ConstantInt>(Op) ||
3184 (Op->getType() != Ty1 &&
3185 Op->getType()->getScalarSizeInBits() >
3186 DL->getIndexSizeInBits(
3187 V->getType()->getPointerAddressSpace()))) {
3188 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
3189 << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
3190 BS.cancelScheduling(VL, VL0);
3191 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3192 ReuseShuffleIndicies);
3193 return;
3194 }
3195 }
3196
3197 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3198 ReuseShuffleIndicies);
3199 LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
3200 TE->setOperandsInOrder();
3201 for (unsigned i = 0, e = 2; i < e; ++i) {
3202 ValueList Operands;
3203 // Prepare the operand vector.
3204 for (Value *V : VL)
3205 Operands.push_back(cast<Instruction>(V)->getOperand(i));
3206
3207 buildTree_rec(Operands, Depth + 1, {TE, i});
3208 }
3209 return;
3210 }
3211 case Instruction::Store: {
3212 // Check if the stores are consecutive or if we need to swizzle them.
3213 llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType();
3214 // Avoid types that are padded when being allocated as scalars, while
3215 // being packed together in a vector (such as i1).
3216 if (DL->getTypeSizeInBits(ScalarTy) !=
3217 DL->getTypeAllocSizeInBits(ScalarTy)) {
3218 BS.cancelScheduling(VL, VL0);
3219 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3220 ReuseShuffleIndicies);
3221 LLVM_DEBUG(dbgs() << "SLP: Gathering stores of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering stores of non-packed type.\n"
; } } while (false)
;
3222 return;
3223 }
3224 // Make sure all stores in the bundle are simple - we can't vectorize
3225 // atomic or volatile stores.
3226 SmallVector<Value *, 4> PointerOps(VL.size());
3227 ValueList Operands(VL.size());
3228 auto POIter = PointerOps.begin();
3229 auto OIter = Operands.begin();
3230 for (Value *V : VL) {
3231 auto *SI = cast<StoreInst>(V);
3232 if (!SI->isSimple()) {
3233 BS.cancelScheduling(VL, VL0);
3234 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3235 ReuseShuffleIndicies);
3236 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple stores.\n"
; } } while (false)
;
3237 return;
3238 }
3239 *POIter = SI->getPointerOperand();
3240 *OIter = SI->getValueOperand();
3241 ++POIter;
3242 ++OIter;
3243 }
3244
3245 OrdersType CurrentOrder;
3246 // Check the order of pointer operands.
3247 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
3248 Value *Ptr0;
3249 Value *PtrN;
3250 if (CurrentOrder.empty()) {
3251 Ptr0 = PointerOps.front();
3252 PtrN = PointerOps.back();
3253 } else {
3254 Ptr0 = PointerOps[CurrentOrder.front()];
3255 PtrN = PointerOps[CurrentOrder.back()];
3256 }
3257 Optional<int> Dist = getPointersDiff(Ptr0, PtrN, *DL, *SE);
3258 // Check that the sorted pointer operands are consecutive.
3259 if (static_cast<unsigned>(*Dist) == VL.size() - 1) {
3260 if (CurrentOrder.empty()) {
3261 // Original stores are consecutive and does not require reordering.
3262 ++NumOpsWantToKeepOriginalOrder;
3263 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S,
3264 UserTreeIdx, ReuseShuffleIndicies);
3265 TE->setOperandsInOrder();
3266 buildTree_rec(Operands, Depth + 1, {TE, 0});
3267 LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
3268 } else {
3269 TreeEntry *TE =
3270 newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3271 ReuseShuffleIndicies, CurrentOrder);
3272 TE->setOperandsInOrder();
3273 buildTree_rec(Operands, Depth + 1, {TE, 0});
3274 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of jumbled stores.\n"
; } } while (false)
;
3275 findRootOrder(CurrentOrder);
3276 ++NumOpsWantToKeepOrder[CurrentOrder];
3277 }
3278 return;
3279 }
3280 }
3281
3282 BS.cancelScheduling(VL, VL0);
3283 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3284 ReuseShuffleIndicies);
3285 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
3286 return;
3287 }
3288 case Instruction::Call: {
3289 // Check if the calls are all to the same vectorizable intrinsic or
3290 // library function.
3291 CallInst *CI = cast<CallInst>(VL0);
3292 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3293
3294 VFShape Shape = VFShape::get(
3295 *CI, ElementCount::getFixed(static_cast<unsigned int>(VL.size())),
3296 false /*HasGlobalPred*/);
3297 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3298
3299 if (!VecFunc && !isTriviallyVectorizable(ID)) {
3300 BS.cancelScheduling(VL, VL0);
3301 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3302 ReuseShuffleIndicies);
3303 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
3304 return;
3305 }
3306 Function *F = CI->getCalledFunction();
3307 unsigned NumArgs = CI->getNumArgOperands();
3308 SmallVector<Value*, 4> ScalarArgs(NumArgs, nullptr);
3309 for (unsigned j = 0; j != NumArgs; ++j)
3310 if (hasVectorInstrinsicScalarOpd(ID, j))
3311 ScalarArgs[j] = CI->getArgOperand(j);
3312 for (Value *V : VL) {
3313 CallInst *CI2 = dyn_cast<CallInst>(V);
3314 if (!CI2 || CI2->getCalledFunction() != F ||
3315 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
3316 (VecFunc &&
3317 VecFunc != VFDatabase(*CI2).getVectorizedFunction(Shape)) ||
3318 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
3319 BS.cancelScheduling(VL, VL0);
3320 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3321 ReuseShuffleIndicies);
3322 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *Vdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *V << "\n"; } } while (false)
3323 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *V << "\n"; } } while (false)
;
3324 return;
3325 }
3326 // Some intrinsics have scalar arguments and should be same in order for
3327 // them to be vectorized.
3328 for (unsigned j = 0; j != NumArgs; ++j) {
3329 if (hasVectorInstrinsicScalarOpd(ID, j)) {
3330 Value *A1J = CI2->getArgOperand(j);
3331 if (ScalarArgs[j] != A1J) {
3332 BS.cancelScheduling(VL, VL0);
3333 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3334 ReuseShuffleIndicies);
3335 LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << ScalarArgs[j] <<
"!=" << A1J << "\n"; } } while (false)
3336 << " argument " << ScalarArgs[j] << "!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << ScalarArgs[j] <<
"!=" << A1J << "\n"; } } while (false)
3337 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << ScalarArgs[j] <<
"!=" << A1J << "\n"; } } while (false)
;
3338 return;
3339 }
3340 }
3341 }
3342 // Verify that the bundle operands are identical between the two calls.
3343 if (CI->hasOperandBundles() &&
3344 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
3345 CI->op_begin() + CI->getBundleOperandsEndIndex(),
3346 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
3347 BS.cancelScheduling(VL, VL0);
3348 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3349 ReuseShuffleIndicies);
3350 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *V << '\n'; } } while
(false)
3351 << *CI << "!=" << *V << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *V << '\n'; } } while
(false)
;
3352 return;
3353 }
3354 }
3355
3356 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3357 ReuseShuffleIndicies);
3358 TE->setOperandsInOrder();
3359 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
3360 ValueList Operands;
3361 // Prepare the operand vector.
3362 for (Value *V : VL) {
3363 auto *CI2 = cast<CallInst>(V);
3364 Operands.push_back(CI2->getArgOperand(i));
3365 }
3366 buildTree_rec(Operands, Depth + 1, {TE, i});
3367 }
3368 return;
3369 }
3370 case Instruction::ShuffleVector: {
3371 // If this is not an alternate sequence of opcode like add-sub
3372 // then do not vectorize this instruction.
3373 if (!S.isAltShuffle()) {
3374 BS.cancelScheduling(VL, VL0);
3375 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3376 ReuseShuffleIndicies);
3377 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
3378 return;
3379 }
3380 TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3381 ReuseShuffleIndicies);
3382 LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
3383
3384 // Reorder operands if reordering would enable vectorization.
3385 if (isa<BinaryOperator>(VL0)) {
3386 ValueList Left, Right;
3387 reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3388 TE->setOperand(0, Left);
3389 TE->setOperand(1, Right);
3390 buildTree_rec(Left, Depth + 1, {TE, 0});
3391 buildTree_rec(Right, Depth + 1, {TE, 1});
3392 return;
3393 }
3394
3395 TE->setOperandsInOrder();
3396 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3397 ValueList Operands;
3398 // Prepare the operand vector.
3399 for (Value *V : VL)
3400 Operands.push_back(cast<Instruction>(V)->getOperand(i));
3401
3402 buildTree_rec(Operands, Depth + 1, {TE, i});
3403 }
3404 return;
3405 }
3406 default:
3407 BS.cancelScheduling(VL, VL0);
3408 newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3409 ReuseShuffleIndicies);
3410 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
3411 return;
3412 }
3413}
3414
3415unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
3416 unsigned N = 1;
3417 Type *EltTy = T;
3418
3419 while (isa<StructType>(EltTy) || isa<ArrayType>(EltTy) ||
3420 isa<VectorType>(EltTy)) {
3421 if (auto *ST = dyn_cast<StructType>(EltTy)) {
3422 // Check that struct is homogeneous.
3423 for (const auto *Ty : ST->elements())
3424 if (Ty != *ST->element_begin())
3425 return 0;
3426 N *= ST->getNumElements();
3427 EltTy = *ST->element_begin();
3428 } else if (auto *AT = dyn_cast<ArrayType>(EltTy)) {
3429 N *= AT->getNumElements();
3430 EltTy = AT->getElementType();
3431 } else {
3432 auto *VT = cast<FixedVectorType>(EltTy);
3433 N *= VT->getNumElements();
3434 EltTy = VT->getElementType();
3435 }
3436 }
3437
3438 if (!isValidElementType(EltTy))
3439 return 0;
3440 uint64_t VTSize = DL.getTypeStoreSizeInBits(FixedVectorType::get(EltTy, N));
3441 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
3442 return 0;
3443 return N;
3444}
3445
3446bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
3447 SmallVectorImpl<unsigned> &CurrentOrder) const {
3448 Instruction *E0 = cast<Instruction>(OpValue);
3449 assert(E0->getOpcode() == Instruction::ExtractElement ||(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3450, __extension__ __PRETTY_FUNCTION__))
3450 E0->getOpcode() == Instruction::ExtractValue)(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3450, __extension__ __PRETTY_FUNCTION__))
;
3451 assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode")(static_cast <bool> (E0->getOpcode() == getSameOpcode
(VL).getOpcode() && "Invalid opcode") ? void (0) : __assert_fail
("E0->getOpcode() == getSameOpcode(VL).getOpcode() && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3451, __extension__ __PRETTY_FUNCTION__))
;
3452 // Check if all of the extracts come from the same vector and from the
3453 // correct offset.
3454 Value *Vec = E0->getOperand(0);
3455
3456 CurrentOrder.clear();
3457
3458 // We have to extract from a vector/aggregate with the same number of elements.
3459 unsigned NElts;
3460 if (E0->getOpcode() == Instruction::ExtractValue) {
3461 const DataLayout &DL = E0->getModule()->getDataLayout();
3462 NElts = canMapToVector(Vec->getType(), DL);
3463 if (!NElts)
3464 return false;
3465 // Check if load can be rewritten as load of vector.
3466 LoadInst *LI = dyn_cast<LoadInst>(Vec);
3467 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
3468 return false;
3469 } else {
3470 NElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
3471 }
3472
3473 if (NElts != VL.size())
3474 return false;
3475
3476 // Check that all of the indices extract from the correct offset.
3477 bool ShouldKeepOrder = true;
3478 unsigned E = VL.size();
3479 // Assign to all items the initial value E + 1 so we can check if the extract
3480 // instruction index was used already.
3481 // Also, later we can check that all the indices are used and we have a
3482 // consecutive access in the extract instructions, by checking that no
3483 // element of CurrentOrder still has value E + 1.
3484 CurrentOrder.assign(E, E + 1);
3485 unsigned I = 0;
3486 for (; I < E; ++I) {
3487 auto *Inst = cast<Instruction>(VL[I]);
3488 if (Inst->getOperand(0) != Vec)
3489 break;
3490 Optional<unsigned> Idx = getExtractIndex(Inst);
3491 if (!Idx)
3492 break;
3493 const unsigned ExtIdx = *Idx;
3494 if (ExtIdx != I) {
3495 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
3496 break;
3497 ShouldKeepOrder = false;
3498 CurrentOrder[ExtIdx] = I;
3499 } else {
3500 if (CurrentOrder[I] != E + 1)
3501 break;
3502 CurrentOrder[I] = I;
3503 }
3504 }
3505 if (I < E) {
3506 CurrentOrder.clear();
3507 return false;
3508 }
3509
3510 return ShouldKeepOrder;
3511}
3512
3513bool BoUpSLP::areAllUsersVectorized(Instruction *I,
3514 ArrayRef<Value *> VectorizedVals) const {
3515 return (I->hasOneUse() && is_contained(VectorizedVals, I)) ||
3516 llvm::all_of(I->users(), [this](User *U) {
3517 return ScalarToTreeEntry.count(U) > 0;
3518 });
3519}
3520
3521static std::pair<InstructionCost, InstructionCost>
3522getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
3523 TargetTransformInfo *TTI, TargetLibraryInfo *TLI) {
3524 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3525
3526 // Calculate the cost of the scalar and vector calls.
3527 SmallVector<Type *, 4> VecTys;
3528 for (Use &Arg : CI->args())
3529 VecTys.push_back(
3530 FixedVectorType::get(Arg->getType(), VecTy->getNumElements()));
3531 FastMathFlags FMF;
3532 if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
3533 FMF = FPCI->getFastMathFlags();
3534 SmallVector<const Value *> Arguments(CI->arg_begin(), CI->arg_end());
3535 IntrinsicCostAttributes CostAttrs(ID, VecTy, Arguments, VecTys, FMF,
3536 dyn_cast<IntrinsicInst>(CI));
3537 auto IntrinsicCost =
3538 TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
3539
3540 auto Shape = VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
3541 VecTy->getNumElements())),
3542 false /*HasGlobalPred*/);
3543 Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3544 auto LibCost = IntrinsicCost;
3545 if (!CI->isNoBuiltin() && VecFunc) {
3546 // Calculate the cost of the vector library call.
3547 // If the corresponding vector call is cheaper, return its cost.
3548 LibCost = TTI->getCallInstrCost(nullptr, VecTy, VecTys,
3549 TTI::TCK_RecipThroughput);
3550 }
3551 return {IntrinsicCost, LibCost};
3552}
3553
3554/// Compute the cost of creating a vector of type \p VecTy containing the
3555/// extracted values from \p VL.
3556static InstructionCost
3557computeExtractCost(ArrayRef<Value *> VL, FixedVectorType *VecTy,
3558 TargetTransformInfo::ShuffleKind ShuffleKind,
3559 ArrayRef<int> Mask, TargetTransformInfo &TTI) {
3560 unsigned NumOfParts = TTI.getNumberOfParts(VecTy);
3561
3562 if (ShuffleKind != TargetTransformInfo::SK_PermuteSingleSrc || !NumOfParts ||
3563 VecTy->getNumElements() < NumOfParts)
3564 return TTI.getShuffleCost(ShuffleKind, VecTy, Mask);
3565
3566 bool AllConsecutive = true;
3567 unsigned EltsPerVector = VecTy->getNumElements() / NumOfParts;
3568 unsigned Idx = -1;
3569 InstructionCost Cost = 0;
3570
3571 // Process extracts in blocks of EltsPerVector to check if the source vector
3572 // operand can be re-used directly. If not, add the cost of creating a shuffle
3573 // to extract the values into a vector register.
3574 for (auto *V : VL) {
3575 ++Idx;
3576
3577 // Reached the start of a new vector registers.
3578 if (Idx % EltsPerVector == 0) {
3579 AllConsecutive = true;
3580 continue;
3581 }
3582
3583 // Check all extracts for a vector register on the target directly
3584 // extract values in order.
3585 unsigned CurrentIdx = *getExtractIndex(cast<Instruction>(V));
3586 unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1]));
3587 AllConsecutive &= PrevIdx + 1 == CurrentIdx &&
3588 CurrentIdx % EltsPerVector == Idx % EltsPerVector;
3589
3590 if (AllConsecutive)
3591 continue;
3592
3593 // Skip all indices, except for the last index per vector block.
3594 if ((Idx + 1) % EltsPerVector != 0 && Idx + 1 != VL.size())
3595 continue;
3596
3597 // If we have a series of extracts which are not consecutive and hence
3598 // cannot re-use the source vector register directly, compute the shuffle
3599 // cost to extract the a vector with EltsPerVector elements.
3600 Cost += TTI.getShuffleCost(
3601 TargetTransformInfo::SK_PermuteSingleSrc,
3602 FixedVectorType::get(VecTy->getElementType(), EltsPerVector));
3603 }
3604 return Cost;
3605}
3606
3607InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
3608 ArrayRef<Value *> VectorizedVals) {
3609 ArrayRef<Value*> VL = E->Scalars;
3610
3611 Type *ScalarTy = VL[0]->getType();
3612 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
3613 ScalarTy = SI->getValueOperand()->getType();
3614 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
3615 ScalarTy = CI->getOperand(0)->getType();
3616 else if (auto *IE = dyn_cast<InsertElementInst>(VL[0]))
3617 ScalarTy = IE->getOperand(1)->getType();
3618 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size());
3619 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3620
3621 // If we have computed a smaller type for the expression, update VecTy so
3622 // that the costs will be accurate.
3623 if (MinBWs.count(VL[0]))
3624 VecTy = FixedVectorType::get(
3625 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
3626
3627 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
3628 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3629 InstructionCost ReuseShuffleCost = 0;
3630 if (NeedToShuffleReuses) {
3631 ReuseShuffleCost =
3632 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy,
3633 E->ReuseShuffleIndices);
3634 }
3635 // FIXME: it tries to fix a problem with MSVC buildbots.
3636 TargetTransformInfo &TTIRef = *TTI;
3637 auto &&AdjustExtractsCost = [this, &TTIRef, CostKind, VL, VecTy,
3638 VectorizedVals](InstructionCost &Cost,
3639 bool IsGather) {
3640 DenseMap<Value *, int> ExtractVectorsTys;
3641 for (auto *V : VL) {
3642 // If all users of instruction are going to be vectorized and this
3643 // instruction itself is not going to be vectorized, consider this
3644 // instruction as dead and remove its cost from the final cost of the
3645 // vectorized tree.
3646 if (!areAllUsersVectorized(cast<Instruction>(V), VectorizedVals) ||
3647 (IsGather && ScalarToTreeEntry.count(V)))
3648 continue;
3649 auto *EE = cast<ExtractElementInst>(V);
3650 unsigned Idx = *getExtractIndex(EE);
3651 if (TTIRef.getNumberOfParts(VecTy) !=
3652 TTIRef.getNumberOfParts(EE->getVectorOperandType())) {
3653 auto It =
3654 ExtractVectorsTys.try_emplace(EE->getVectorOperand(), Idx).first;
3655 It->getSecond() = std::min<int>(It->second, Idx);
3656 }
3657 // Take credit for instruction that will become dead.
3658 if (EE->hasOneUse()) {
3659 Instruction *Ext = EE->user_back();
3660 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
3661 all_of(Ext->users(),
3662 [](User *U) { return isa<GetElementPtrInst>(U); })) {
3663 // Use getExtractWithExtendCost() to calculate the cost of
3664 // extractelement/ext pair.
3665 Cost -=
3666 TTIRef.getExtractWithExtendCost(Ext->getOpcode(), Ext->getType(),
3667 EE->getVectorOperandType(), Idx);
3668 // Add back the cost of s|zext which is subtracted separately.
3669 Cost += TTIRef.getCastInstrCost(
3670 Ext->getOpcode(), Ext->getType(), EE->getType(),
3671 TTI::getCastContextHint(Ext), CostKind, Ext);
3672 continue;
3673 }
3674 }
3675 Cost -= TTIRef.getVectorInstrCost(Instruction::ExtractElement,
3676 EE->getVectorOperandType(), Idx);
3677 }
3678 // Add a cost for subvector extracts/inserts if required.
3679 for (const auto &Data : ExtractVectorsTys) {
3680 auto *EEVTy = cast<FixedVectorType>(Data.first->getType());
3681 unsigned NumElts = VecTy->getNumElements();
3682 if (TTIRef.getNumberOfParts(EEVTy) > TTIRef.getNumberOfParts(VecTy)) {
3683 unsigned Idx = (Data.second / NumElts) * NumElts;
3684 unsigned EENumElts = EEVTy->getNumElements();
3685 if (Idx + NumElts <= EENumElts) {
3686 Cost +=
3687 TTIRef.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
3688 EEVTy, None, Idx, VecTy);
3689 } else {
3690 // Need to round up the subvector type vectorization factor to avoid a
3691 // crash in cost model functions. Make SubVT so that Idx + VF of SubVT
3692 // <= EENumElts.
3693 auto *SubVT =
3694 FixedVectorType::get(VecTy->getElementType(), EENumElts - Idx);
3695 Cost +=
3696 TTIRef.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
3697 EEVTy, None, Idx, SubVT);
3698 }
3699 } else {
3700 Cost += TTIRef.getShuffleCost(TargetTransformInfo::SK_InsertSubvector,
3701 VecTy, None, 0, EEVTy);
3702 }
3703 }
3704 };
3705 if (E->State == TreeEntry::NeedToGather) {
3706 if (allConstant(VL))
3707 return 0;
3708 if (isa<InsertElementInst>(VL[0]))
3709 return InstructionCost::getInvalid();
3710 SmallVector<int> Mask;
3711 SmallVector<const TreeEntry *> Entries;
3712 Optional<TargetTransformInfo::ShuffleKind> Shuffle =
3713 isGatherShuffledEntry(E, Mask, Entries);
3714 if (Shuffle.hasValue()) {
3715 InstructionCost GatherCost = 0;
3716 if (ShuffleVectorInst::isIdentityMask(Mask)) {
3717 // Perfect match in the graph, will reuse the previously vectorized
3718 // node. Cost is 0.
3719 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: perfect diamond match for gather bundle that starts with "
<< *VL.front() << ".\n"; } } while (false)
3720 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: perfect diamond match for gather bundle that starts with "
<< *VL.front() << ".\n"; } } while (false)
3721 << "SLP: perfect diamond match for gather bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: perfect diamond match for gather bundle that starts with "
<< *VL.front() << ".\n"; } } while (false)
3722 << *VL.front() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: perfect diamond match for gather bundle that starts with "
<< *VL.front() << ".\n"; } } while (false)
;
3723 } else {
3724 LLVM_DEBUG(dbgs() << "SLP: shuffled " << Entries.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: shuffled " << Entries.
size() << " entries for bundle that starts with " <<
*VL.front() << ".\n"; } } while (false)
3725 << " entries for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: shuffled " << Entries.
size() << " entries for bundle that starts with " <<
*VL.front() << ".\n"; } } while (false)
3726 << *VL.front() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: shuffled " << Entries.
size() << " entries for bundle that starts with " <<
*VL.front() << ".\n"; } } while (false)
;
3727 // Detected that instead of gather we can emit a shuffle of single/two
3728 // previously vectorized nodes. Add the cost of the permutation rather
3729 // than gather.
3730 GatherCost = TTI->getShuffleCost(*Shuffle, VecTy, Mask);
3731 }
3732 return ReuseShuffleCost + GatherCost;
3733 }
3734 if (isSplat(VL)) {
3735 // Found the broadcasting of the single scalar, calculate the cost as the
3736 // broadcast.
3737 return ReuseShuffleCost +
3738 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, None,
3739 0);
3740 }
3741 if (E->getOpcode() == Instruction::ExtractElement && allSameType(VL) &&
3742 allSameBlock(VL)) {
3743 // Check that gather of extractelements can be represented as just a
3744 // shuffle of a single/two vectors the scalars are extracted from.
3745 SmallVector<int> Mask;
3746 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind =
3747 isShuffle(VL, Mask);
3748 if (ShuffleKind.hasValue()) {
3749 // Found the bunch of extractelement instructions that must be gathered
3750 // into a vector and can be represented as a permutation elements in a
3751 // single input vector or of 2 input vectors.
3752 InstructionCost Cost =
3753 computeExtractCost(VL, VecTy, *ShuffleKind, Mask, *TTI);
3754 AdjustExtractsCost(Cost, /*IsGather=*/true);
3755 return ReuseShuffleCost + Cost;
3756 }
3757 }
3758 return ReuseShuffleCost + getGatherCost(VL);
3759 }
3760 assert((E->State == TreeEntry::Vectorize ||(static_cast <bool> ((E->State == TreeEntry::Vectorize
|| E->State == TreeEntry::ScatterVectorize) && "Unhandled state"
) ? void (0) : __assert_fail ("(E->State == TreeEntry::Vectorize || E->State == TreeEntry::ScatterVectorize) && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3762, __extension__ __PRETTY_FUNCTION__))
3761 E->State == TreeEntry::ScatterVectorize) &&(static_cast <bool> ((E->State == TreeEntry::Vectorize
|| E->State == TreeEntry::ScatterVectorize) && "Unhandled state"
) ? void (0) : __assert_fail ("(E->State == TreeEntry::Vectorize || E->State == TreeEntry::ScatterVectorize) && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3762, __extension__ __PRETTY_FUNCTION__))
3762 "Unhandled state")(static_cast <bool> ((E->State == TreeEntry::Vectorize
|| E->State == TreeEntry::ScatterVectorize) && "Unhandled state"
) ? void (0) : __assert_fail ("(E->State == TreeEntry::Vectorize || E->State == TreeEntry::ScatterVectorize) && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3762, __extension__ __PRETTY_FUNCTION__))
;
3763 assert(E->getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL")(static_cast <bool> (E->getOpcode() && allSameType
(VL) && allSameBlock(VL) && "Invalid VL") ? void
(0) : __assert_fail ("E->getOpcode() && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3763, __extension__ __PRETTY_FUNCTION__))
;
3764 Instruction *VL0 = E->getMainOp();
3765 unsigned ShuffleOrOp =
3766 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
3767 switch (ShuffleOrOp) {
3768 case Instruction::PHI:
3769 return 0;
3770
3771 case Instruction::ExtractValue:
3772 case Instruction::ExtractElement: {
3773 // The common cost of removal ExtractElement/ExtractValue instructions +
3774 // the cost of shuffles, if required to resuffle the original vector.
3775 InstructionCost CommonCost = 0;
3776 if (NeedToShuffleReuses) {
3777 unsigned Idx = 0;
3778 for (unsigned I : E->ReuseShuffleIndices) {
3779 if (ShuffleOrOp == Instruction::ExtractElement) {
3780 auto *EE = cast<ExtractElementInst>(VL[I]);
3781 ReuseShuffleCost -= TTI->getVectorInstrCost(
3782 Instruction::ExtractElement, EE->getVectorOperandType(),
3783 *getExtractIndex(EE));
3784 } else {
3785 ReuseShuffleCost -= TTI->getVectorInstrCost(
3786 Instruction::ExtractElement, VecTy, Idx);
3787 ++Idx;
3788 }
3789 }
3790 Idx = ReuseShuffleNumbers;
3791 for (Value *V : VL) {
3792 if (ShuffleOrOp == Instruction::ExtractElement) {
3793 auto *EE = cast<ExtractElementInst>(V);
3794 ReuseShuffleCost += TTI->getVectorInstrCost(
3795 Instruction::ExtractElement, EE->getVectorOperandType(),
3796 *getExtractIndex(EE));
3797 } else {
3798 --Idx;
3799 ReuseShuffleCost += TTI->getVectorInstrCost(
3800 Instruction::ExtractElement, VecTy, Idx);
3801 }
3802 }
3803 CommonCost = ReuseShuffleCost;
3804 } else if (!E->ReorderIndices.empty()) {
3805 SmallVector<int> NewMask;
3806 inversePermutation(E->ReorderIndices, NewMask);
3807 CommonCost = TTI->getShuffleCost(
3808 TargetTransformInfo::SK_PermuteSingleSrc, VecTy, NewMask);
3809 }
3810 if (ShuffleOrOp == Instruction::ExtractValue) {
3811 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
3812 auto *EI = cast<Instruction>(VL[I]);
3813 // Take credit for instruction that will become dead.
3814 if (EI->hasOneUse()) {
3815 Instruction *Ext = EI->user_back();
3816 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
3817 all_of(Ext->users(),
3818 [](User *U) { return isa<GetElementPtrInst>(U); })) {
3819 // Use getExtractWithExtendCost() to calculate the cost of
3820 // extractelement/ext pair.
3821 CommonCost -= TTI->getExtractWithExtendCost(
3822 Ext->getOpcode(), Ext->getType(), VecTy, I);
3823 // Add back the cost of s|zext which is subtracted separately.
3824 CommonCost += TTI->getCastInstrCost(
3825 Ext->getOpcode(), Ext->getType(), EI->getType(),
3826 TTI::getCastContextHint(Ext), CostKind, Ext);
3827 continue;
3828 }
3829 }
3830 CommonCost -=
3831 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, I);
3832 }
3833 } else {
3834 AdjustExtractsCost(CommonCost, /*IsGather=*/false);
3835 }
3836 return CommonCost;
3837 }
3838 case Instruction::InsertElement: {
3839 auto *SrcVecTy = cast<FixedVectorType>(VL0->getType());
3840
3841 unsigned const NumElts = SrcVecTy->getNumElements();
3842 unsigned const NumScalars = VL.size();
3843 APInt DemandedElts = APInt::getNullValue(NumElts);
3844 // TODO: Add support for Instruction::InsertValue.
3845 unsigned Offset = UINT_MAX(2147483647 *2U +1U);
3846 bool IsIdentity = true;
3847 SmallVector<int> ShuffleMask(NumElts, UndefMaskElem);
3848 for (unsigned I = 0; I < NumScalars; ++I) {
3849 Optional<int> InsertIdx = getInsertIndex(VL[I], 0);
3850 if (!InsertIdx || *InsertIdx == UndefMaskElem)
3851 continue;
3852 unsigned Idx = *InsertIdx;
3853 DemandedElts.setBit(Idx);
3854 if (Idx < Offset) {
3855 Offset = Idx;
3856 IsIdentity &= I == 0;
3857 } else {
3858 assert(Idx >= Offset && "Failed to find vector index offset")(static_cast <bool> (Idx >= Offset && "Failed to find vector index offset"
) ? void (0) : __assert_fail ("Idx >= Offset && \"Failed to find vector index offset\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3858, __extension__ __PRETTY_FUNCTION__))
;
3859 IsIdentity &= Idx - Offset == I;
3860 }
3861 ShuffleMask[Idx] = I;
3862 }
3863 assert(Offset < NumElts && "Failed to find vector index offset")(static_cast <bool> (Offset < NumElts && "Failed to find vector index offset"
) ? void (0) : __assert_fail ("Offset < NumElts && \"Failed to find vector index offset\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3863, __extension__ __PRETTY_FUNCTION__))
;
3864
3865 InstructionCost Cost = 0;
3866 Cost -= TTI->getScalarizationOverhead(SrcVecTy, DemandedElts,
3867 /*Insert*/ true, /*Extract*/ false);
3868
3869 if (IsIdentity && NumElts != NumScalars && Offset % NumScalars != 0)
3870 Cost += TTI->getShuffleCost(
3871 TargetTransformInfo::SK_InsertSubvector, SrcVecTy, /*Mask*/ None,
3872 Offset,
3873 FixedVectorType::get(SrcVecTy->getElementType(), NumScalars));
3874 else if (!IsIdentity)
3875 Cost += TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, SrcVecTy,
3876 ShuffleMask);
3877
3878 return Cost;
3879 }
3880 case Instruction::ZExt:
3881 case Instruction::SExt:
3882 case Instruction::FPToUI:
3883 case Instruction::FPToSI:
3884 case Instruction::FPExt:
3885 case Instruction::PtrToInt:
3886 case Instruction::IntToPtr:
3887 case Instruction::SIToFP:
3888 case Instruction::UIToFP:
3889 case Instruction::Trunc:
3890 case Instruction::FPTrunc:
3891 case Instruction::BitCast: {
3892 Type *SrcTy = VL0->getOperand(0)->getType();
3893 InstructionCost ScalarEltCost =
3894 TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy,
3895 TTI::getCastContextHint(VL0), CostKind, VL0);
3896 if (NeedToShuffleReuses) {
3897 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3898 }
3899
3900 // Calculate the cost of this instruction.
3901 InstructionCost ScalarCost = VL.size() * ScalarEltCost;
3902
3903 auto *SrcVecTy = FixedVectorType::get(SrcTy, VL.size());
3904 InstructionCost VecCost = 0;
3905 // Check if the values are candidates to demote.
3906 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
3907 VecCost =
3908 ReuseShuffleCost +
3909 TTI->getCastInstrCost(E->getOpcode(), VecTy, SrcVecTy,
3910 TTI::getCastContextHint(VL0), CostKind, VL0);
3911 }
3912 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost
); } } while (false)
;
3913 return VecCost - ScalarCost;
3914 }
3915 case Instruction::FCmp:
3916 case Instruction::ICmp:
3917 case Instruction::Select: {
3918 // Calculate the cost of this instruction.
3919 InstructionCost ScalarEltCost =
3920 TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, Builder.getInt1Ty(),
3921 CmpInst::BAD_ICMP_PREDICATE, CostKind, VL0);
3922 if (NeedToShuffleReuses) {
3923 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
3924 }
3925 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(), VL.size());
3926 InstructionCost ScalarCost = VecTy->getNumElements() * ScalarEltCost;
3927
3928 // Check if all entries in VL are either compares or selects with compares
3929 // as condition that have the same predicates.
3930 CmpInst::Predicate VecPred = CmpInst::BAD_ICMP_PREDICATE;
3931 bool First = true;
3932 for (auto *V : VL) {
3933 CmpInst::Predicate CurrentPred;
3934 auto MatchCmp = m_Cmp(CurrentPred, m_Value(), m_Value());
3935 if ((!match(V, m_Select(MatchCmp, m_Value(), m_Value())) &&
3936 !match(V, MatchCmp)) ||
3937 (!First && VecPred != CurrentPred)) {
3938 VecPred = CmpInst::BAD_ICMP_PREDICATE;
3939 break;
3940 }
3941 First = false;
3942 VecPred = CurrentPred;
3943 }
3944
3945 InstructionCost VecCost = TTI->getCmpSelInstrCost(
3946 E->getOpcode(), VecTy, MaskTy, VecPred, CostKind, VL0);
3947 // Check if it is possible and profitable to use min/max for selects in
3948 // VL.
3949 //
3950 auto IntrinsicAndUse = canConvertToMinOrMaxIntrinsic(VL);
3951 if (IntrinsicAndUse.first != Intrinsic::not_intrinsic) {
3952 IntrinsicCostAttributes CostAttrs(IntrinsicAndUse.first, VecTy,
3953 {VecTy, VecTy});
3954 InstructionCost IntrinsicCost =
3955 TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
3956 // If the selects are the only uses of the compares, they will be dead
3957 // and we can adjust the cost by removing their cost.
3958 if (IntrinsicAndUse.second)
3959 IntrinsicCost -=
3960 TTI->getCmpSelInstrCost(Instruction::ICmp, VecTy, MaskTy,
3961 CmpInst::BAD_ICMP_PREDICATE, CostKind);
3962 VecCost = std::min(VecCost, IntrinsicCost);
3963 }
3964 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost
); } } while (false)
;
3965 return ReuseShuffleCost + VecCost - ScalarCost;
3966 }
3967 case Instruction::FNeg:
3968 case Instruction::Add:
3969 case Instruction::FAdd:
3970 case Instruction::Sub:
3971 case Instruction::FSub:
3972 case Instruction::Mul:
3973 case Instruction::FMul:
3974 case Instruction::UDiv:
3975 case Instruction::SDiv:
3976 case Instruction::FDiv:
3977 case Instruction::URem:
3978 case Instruction::SRem:
3979 case Instruction::FRem:
3980 case Instruction::Shl:
3981 case Instruction::LShr:
3982 case Instruction::AShr:
3983 case Instruction::And:
3984 case Instruction::Or:
3985 case Instruction::Xor: {
3986 // Certain instructions can be cheaper to vectorize if they have a
3987 // constant second vector operand.
3988 TargetTransformInfo::OperandValueKind Op1VK =
3989 TargetTransformInfo::OK_AnyValue;
3990 TargetTransformInfo::OperandValueKind Op2VK =
3991 TargetTransformInfo::OK_UniformConstantValue;
3992 TargetTransformInfo::OperandValueProperties Op1VP =
3993 TargetTransformInfo::OP_None;
3994 TargetTransformInfo::OperandValueProperties Op2VP =
3995 TargetTransformInfo::OP_PowerOf2;
3996
3997 // If all operands are exactly the same ConstantInt then set the
3998 // operand kind to OK_UniformConstantValue.
3999 // If instead not all operands are constants, then set the operand kind
4000 // to OK_AnyValue. If all operands are constants but not the same,
4001 // then set the operand kind to OK_NonUniformConstantValue.
4002 ConstantInt *CInt0 = nullptr;
4003 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
4004 const Instruction *I = cast<Instruction>(VL[i]);
4005 unsigned OpIdx = isa<BinaryOperator>(I) ? 1 : 0;
4006 ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(OpIdx));
4007 if (!CInt) {
4008 Op2VK = TargetTransformInfo::OK_AnyValue;
4009 Op2VP = TargetTransformInfo::OP_None;
4010 break;
4011 }
4012 if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
4013 !CInt->getValue().isPowerOf2())
4014 Op2VP = TargetTransformInfo::OP_None;
4015 if (i == 0) {
4016 CInt0 = CInt;
4017 continue;
4018 }
4019 if (CInt0 != CInt)
4020 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
4021 }
4022
4023 SmallVector<const Value *, 4> Operands(VL0->operand_values());
4024 InstructionCost ScalarEltCost =
4025 TTI->getArithmeticInstrCost(E->getOpcode(), ScalarTy, CostKind, Op1VK,
4026 Op2VK, Op1VP, Op2VP, Operands, VL0);
4027 if (NeedToShuffleReuses) {
4028 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4029 }
4030 InstructionCost ScalarCost = VecTy->getNumElements() * ScalarEltCost;
4031 InstructionCost VecCost =
4032 TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind, Op1VK,
4033 Op2VK, Op1VP, Op2VP, Operands, VL0);
4034 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost
); } } while (false)
;
4035 return ReuseShuffleCost + VecCost - ScalarCost;
4036 }
4037 case Instruction::GetElementPtr: {
4038 TargetTransformInfo::OperandValueKind Op1VK =
4039 TargetTransformInfo::OK_AnyValue;
4040 TargetTransformInfo::OperandValueKind Op2VK =
4041 TargetTransformInfo::OK_UniformConstantValue;
4042
4043 InstructionCost ScalarEltCost = TTI->getArithmeticInstrCost(
4044 Instruction::Add, ScalarTy, CostKind, Op1VK, Op2VK);
4045 if (NeedToShuffleReuses) {
4046 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4047 }
4048 InstructionCost ScalarCost = VecTy->getNumElements() * ScalarEltCost;
4049 InstructionCost VecCost = TTI->getArithmeticInstrCost(
4050 Instruction::Add, VecTy, CostKind, Op1VK, Op2VK);
4051 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost
); } } while (false)
;
4052 return ReuseShuffleCost + VecCost - ScalarCost;
4053 }
4054 case Instruction::Load: {
4055 // Cost of wide load - cost of scalar loads.
4056 Align alignment = cast<LoadInst>(VL0)->getAlign();
4057 InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
4058 Instruction::Load, ScalarTy, alignment, 0, CostKind, VL0);
4059 if (NeedToShuffleReuses) {
4060 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4061 }
4062 InstructionCost ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
4063 InstructionCost VecLdCost;
4064 if (E->State == TreeEntry::Vectorize) {
4065 VecLdCost = TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0,
4066 CostKind, VL0);
4067 } else {
4068 assert(E->State == TreeEntry::ScatterVectorize && "Unknown EntryState")(static_cast <bool> (E->State == TreeEntry::ScatterVectorize
&& "Unknown EntryState") ? void (0) : __assert_fail (
"E->State == TreeEntry::ScatterVectorize && \"Unknown EntryState\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4068, __extension__ __PRETTY_FUNCTION__))
;
4069 VecLdCost = TTI->getGatherScatterOpCost(
4070 Instruction::Load, VecTy, cast<LoadInst>(VL0)->getPointerOperand(),
4071 /*VariableMask=*/false, alignment, CostKind, VL0);
4072 }
4073 if (!NeedToShuffleReuses && !E->ReorderIndices.empty()) {
4074 SmallVector<int> NewMask;
4075 inversePermutation(E->ReorderIndices, NewMask);
4076 VecLdCost += TTI->getShuffleCost(
4077 TargetTransformInfo::SK_PermuteSingleSrc, VecTy, NewMask);
4078 }
4079 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecLdCost, ScalarLdCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecLdCost, ScalarLdCost
); } } while (false)
;
4080 return ReuseShuffleCost + VecLdCost - ScalarLdCost;
4081 }
4082 case Instruction::Store: {
4083 // We know that we can merge the stores. Calculate the cost.
4084 bool IsReorder = !E->ReorderIndices.empty();
4085 auto *SI =
4086 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
4087 Align Alignment = SI->getAlign();
4088 InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
4089 Instruction::Store, ScalarTy, Alignment, 0, CostKind, VL0);
4090 InstructionCost ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
4091 InstructionCost VecStCost = TTI->getMemoryOpCost(
4092 Instruction::Store, VecTy, Alignment, 0, CostKind, VL0);
4093 if (IsReorder) {
4094 SmallVector<int> NewMask;
4095 inversePermutation(E->ReorderIndices, NewMask);
4096 VecStCost += TTI->getShuffleCost(
4097 TargetTransformInfo::SK_PermuteSingleSrc, VecTy, NewMask);
4098 }
4099 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecStCost, ScalarStCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecStCost, ScalarStCost
); } } while (false)
;
4100 return VecStCost - ScalarStCost;
4101 }
4102 case Instruction::Call: {
4103 CallInst *CI = cast<CallInst>(VL0);
4104 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
4105
4106 // Calculate the cost of the scalar and vector calls.
4107 IntrinsicCostAttributes CostAttrs(ID, *CI, 1);
4108 InstructionCost ScalarEltCost =
4109 TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
4110 if (NeedToShuffleReuses) {
4111 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4112 }
4113 InstructionCost ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
4114
4115 auto VecCallCosts = getVectorCallCosts(CI, VecTy, TTI, TLI);
4116 InstructionCost VecCallCost =
4117 std::min(VecCallCosts.first, VecCallCosts.second);
4118
4119 LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
4120 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
4121 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
4122
4123 return ReuseShuffleCost + VecCallCost - ScalarCallCost;
4124 }
4125 case Instruction::ShuffleVector: {
4126 assert(E->isAltShuffle() &&(static_cast <bool> (E->isAltShuffle() && ((
Instruction::isBinaryOp(E->getOpcode()) && Instruction
::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E
->getOpcode()) && Instruction::isCast(E->getAltOpcode
()))) && "Invalid Shuffle Vector Operand") ? void (0)
: __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
4127 ((Instruction::isBinaryOp(E->getOpcode()) &&(static_cast <bool> (E->isAltShuffle() && ((
Instruction::isBinaryOp(E->getOpcode()) && Instruction
::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E
->getOpcode()) && Instruction::isCast(E->getAltOpcode
()))) && "Invalid Shuffle Vector Operand") ? void (0)
: __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
4128 Instruction::isBinaryOp(E->getAltOpcode())) ||(static_cast <bool> (E->isAltShuffle() && ((
Instruction::isBinaryOp(E->getOpcode()) && Instruction
::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E
->getOpcode()) && Instruction::isCast(E->getAltOpcode
()))) && "Invalid Shuffle Vector Operand") ? void (0)
: __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
4129 (Instruction::isCast(E->getOpcode()) &&(static_cast <bool> (E->isAltShuffle() && ((
Instruction::isBinaryOp(E->getOpcode()) && Instruction
::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E
->getOpcode()) && Instruction::isCast(E->getAltOpcode
()))) && "Invalid Shuffle Vector Operand") ? void (0)
: __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
4130 Instruction::isCast(E->getAltOpcode()))) &&(static_cast <bool> (E->isAltShuffle() && ((
Instruction::isBinaryOp(E->getOpcode()) && Instruction
::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E
->getOpcode()) && Instruction::isCast(E->getAltOpcode
()))) && "Invalid Shuffle Vector Operand") ? void (0)
: __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
4131 "Invalid Shuffle Vector Operand")(static_cast <bool> (E->isAltShuffle() && ((
Instruction::isBinaryOp(E->getOpcode()) && Instruction
::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E
->getOpcode()) && Instruction::isCast(E->getAltOpcode
()))) && "Invalid Shuffle Vector Operand") ? void (0)
: __assert_fail ("E->isAltShuffle() && ((Instruction::isBinaryOp(E->getOpcode()) && Instruction::isBinaryOp(E->getAltOpcode())) || (Instruction::isCast(E->getOpcode()) && Instruction::isCast(E->getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4131, __extension__ __PRETTY_FUNCTION__))
;
4132 InstructionCost ScalarCost = 0;
4133 if (NeedToShuffleReuses) {
4134 for (unsigned Idx : E->ReuseShuffleIndices) {
4135 Instruction *I = cast<Instruction>(VL[Idx]);
4136 ReuseShuffleCost -= TTI->getInstructionCost(I, CostKind);
4137 }
4138 for (Value *V : VL) {
4139 Instruction *I = cast<Instruction>(V);
4140 ReuseShuffleCost += TTI->getInstructionCost(I, CostKind);
4141 }
4142 }
4143 for (Value *V : VL) {
4144 Instruction *I = cast<Instruction>(V);
4145 assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode")(static_cast <bool> (E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"
) ? void (0) : __assert_fail ("E->isOpcodeOrAlt(I) && \"Unexpected main/alternate opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4145, __extension__ __PRETTY_FUNCTION__))
;
4146 ScalarCost += TTI->getInstructionCost(I, CostKind);
4147 }
4148 // VecCost is equal to sum of the cost of creating 2 vectors
4149 // and the cost of creating shuffle.
4150 InstructionCost VecCost = 0;
4151 if (Instruction::isBinaryOp(E->getOpcode())) {
4152 VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind);
4153 VecCost += TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy,
4154 CostKind);
4155 } else {
4156 Type *Src0SclTy = E->getMainOp()->getOperand(0)->getType();
4157 Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType();
4158 auto *Src0Ty = FixedVectorType::get(Src0SclTy, VL.size());
4159 auto *Src1Ty = FixedVectorType::get(Src1SclTy, VL.size());
4160 VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty,
4161 TTI::CastContextHint::None, CostKind);
4162 VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty,
4163 TTI::CastContextHint::None, CostKind);
4164 }
4165
4166 SmallVector<int> Mask(E->Scalars.size());
4167 for (unsigned I = 0, End = E->Scalars.size(); I < End; ++I) {
4168 auto *OpInst = cast<Instruction>(E->Scalars[I]);
4169 assert(E->isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode")(static_cast <bool> (E->isOpcodeOrAlt(OpInst) &&
"Unexpected main/alternate opcode") ? void (0) : __assert_fail
("E->isOpcodeOrAlt(OpInst) && \"Unexpected main/alternate opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4169, __extension__ __PRETTY_FUNCTION__))
;
4170 Mask[I] = I + (OpInst->getOpcode() == E->getAltOpcode() ? End : 0);
4171 }
4172 VecCost +=
4173 TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, Mask, 0);
4174 LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dumpTreeCosts(E, ReuseShuffleCost, VecCost, ScalarCost
); } } while (false)
;
4175 return ReuseShuffleCost + VecCost - ScalarCost;
4176 }
4177 default:
4178 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4178)
;
4179 }
4180}
4181
4182bool BoUpSLP::isFullyVectorizableTinyTree() const {
4183 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
4184 << VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
4185
4186 // We only handle trees of heights 1 and 2.
4187 if (VectorizableTree.size() == 1 &&
4188 VectorizableTree[0]->State == TreeEntry::Vectorize)
4189 return true;
4190
4191 if (VectorizableTree.size() != 2)
4192 return false;
4193
4194 // Handle splat and all-constants stores. Also try to vectorize tiny trees
4195 // with the second gather nodes if they have less scalar operands rather than
4196 // the initial tree element (may be profitable to shuffle the second gather)
4197 // or they are extractelements, which form shuffle.
4198 SmallVector<int> Mask;
4199 if (VectorizableTree[0]->State == TreeEntry::Vectorize &&
4200 (allConstant(VectorizableTree[1]->Scalars) ||
4201 isSplat(VectorizableTree[1]->Scalars) ||
4202 (VectorizableTree[1]->State == TreeEntry::NeedToGather &&
4203 VectorizableTree[1]->Scalars.size() <
4204 VectorizableTree[0]->Scalars.size()) ||
4205 (VectorizableTree[1]->State == TreeEntry::NeedToGather &&
4206 VectorizableTree[1]->getOpcode() == Instruction::ExtractElement &&
4207 isShuffle(VectorizableTree[1]->Scalars, Mask))))
4208 return true;
4209
4210 // Gathering cost would be too much for tiny trees.
4211 if (VectorizableTree[0]->State == TreeEntry::NeedToGather ||
4212 VectorizableTree[1]->State == TreeEntry::NeedToGather)
4213 return false;
4214
4215 return true;
4216}
4217
4218static bool isLoadCombineCandidateImpl(Value *Root, unsigned NumElts,
4219 TargetTransformInfo *TTI,
4220 bool MustMatchOrInst) {
4221 // Look past the root to find a source value. Arbitrarily follow the
4222 // path through operand 0 of any 'or'. Also, peek through optional
4223 // shift-left-by-multiple-of-8-bits.
4224 Value *ZextLoad = Root;
4225 const APInt *ShAmtC;
4226 bool FoundOr = false;
4227 while (!isa<ConstantExpr>(ZextLoad) &&
4228 (match(ZextLoad, m_Or(m_Value(), m_Value())) ||
4229 (match(ZextLoad, m_Shl(m_Value(), m_APInt(ShAmtC))) &&
4230 ShAmtC->urem(8) == 0))) {
4231 auto *BinOp = cast<BinaryOperator>(ZextLoad);
4232 ZextLoad = BinOp->getOperand(0);
4233 if (BinOp->getOpcode() == Instruction::Or)
4234 FoundOr = true;
4235 }
4236 // Check if the input is an extended load of the required or/shift expression.
4237 Value *LoadPtr;
4238 if ((MustMatchOrInst && !FoundOr) || ZextLoad == Root ||
4239 !match(ZextLoad, m_ZExt(m_Load(m_Value(LoadPtr)))))
4240 return false;
4241
4242 // Require that the total load bit width is a legal integer type.
4243 // For example, <8 x i8> --> i64 is a legal integer on a 64-bit target.
4244 // But <16 x i8> --> i128 is not, so the backend probably can't reduce it.
4245 Type *SrcTy = LoadPtr->getType()->getPointerElementType();
4246 unsigned LoadBitWidth = SrcTy->getIntegerBitWidth() * NumElts;
4247 if (!TTI->isTypeLegal(IntegerType::get(Root->getContext(), LoadBitWidth)))
4248 return false;
4249
4250 // Everything matched - assume that we can fold the whole sequence using
4251 // load combining.
4252 LLVM_DEBUG(dbgs() << "SLP: Assume load combining for tree starting at "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Assume load combining for tree starting at "
<< *(cast<Instruction>(Root)) << "\n"; } }
while (false)
4253 << *(cast<Instruction>(Root)) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Assume load combining for tree starting at "
<< *(cast<Instruction>(Root)) << "\n"; } }
while (false)
;
4254
4255 return true;
4256}
4257
4258bool BoUpSLP::isLoadCombineReductionCandidate(RecurKind RdxKind) const {
4259 if (RdxKind != RecurKind::Or)
4260 return false;
4261
4262 unsigned NumElts = VectorizableTree[0]->Scalars.size();
4263 Value *FirstReduced = VectorizableTree[0]->Scalars[0];
4264 return isLoadCombineCandidateImpl(FirstReduced, NumElts, TTI,
4265 /* MatchOr */ false);
4266}
4267
4268bool BoUpSLP::isLoadCombineCandidate() const {
4269 // Peek through a final sequence of stores and check if all operations are
4270 // likely to be load-combined.
4271 unsigned NumElts = VectorizableTree[0]->Scalars.size();
4272 for (Value *Scalar : VectorizableTree[0]->Scalars) {
4273 Value *X;
4274 if (!match(Scalar, m_Store(m_Value(X), m_Value())) ||
4275 !isLoadCombineCandidateImpl(X, NumElts, TTI, /* MatchOr */ true))
4276 return false;
4277 }
4278 return true;
4279}
4280
4281bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() const {
4282 // No need to vectorize inserts of gathered values.
4283 if (VectorizableTree.size() == 2 &&
4284 isa<InsertElementInst>(VectorizableTree[0]->Scalars[0]) &&
4285 VectorizableTree[1]->State == TreeEntry::NeedToGather)
4286 return true;
4287
4288 // We can vectorize the tree if its size is greater than or equal to the
4289 // minimum size specified by the MinTreeSize command line option.
4290 if (VectorizableTree.size() >= MinTreeSize)
4291 return false;
4292
4293 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
4294 // can vectorize it if we can prove it fully vectorizable.
4295 if (isFullyVectorizableTinyTree())
4296 return false;
4297
4298 assert(VectorizableTree.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4300, __extension__ __PRETTY_FUNCTION__))
4299 ? ExternalUses.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4300, __extension__ __PRETTY_FUNCTION__))
4300 : true && "We shouldn't have any external users")(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4300, __extension__ __PRETTY_FUNCTION__))
;
4301
4302 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
4303 // vectorizable.
4304 return true;
4305}
4306
4307InstructionCost BoUpSLP::getSpillCost() const {
4308 // Walk from the bottom of the tree to the top, tracking which values are
4309 // live. When we see a call instruction that is not part of our tree,
4310 // query TTI to see if there is a cost to keeping values live over it
4311 // (for example, if spills and fills are required).
4312 unsigned BundleWidth = VectorizableTree.front()->Scalars.size();
4313 InstructionCost Cost = 0;
4314
4315 SmallPtrSet<Instruction*, 4> LiveValues;
4316 Instruction *PrevInst = nullptr;
4317
4318 // The entries in VectorizableTree are not necessarily ordered by their
4319 // position in basic blocks. Collect them and order them by dominance so later
4320 // instructions are guaranteed to be visited first. For instructions in
4321 // different basic blocks, we only scan to the beginning of the block, so
4322 // their order does not matter, as long as all instructions in a basic block
4323 // are grouped together. Using dominance ensures a deterministic order.
4324 SmallVector<Instruction *, 16> OrderedScalars;
4325 for (const auto &TEPtr : VectorizableTree) {
4326 Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]);
4327 if (!Inst)
4328 continue;
4329 OrderedScalars.push_back(Inst);
4330 }
4331 llvm::sort(OrderedScalars, [&](Instruction *A, Instruction *B) {
4332 auto *NodeA = DT->getNode(A->getParent());
4333 auto *NodeB = DT->getNode(B->getParent());
4334 assert(NodeA && "Should only process reachable instructions")(static_cast <bool> (NodeA && "Should only process reachable instructions"
) ? void (0) : __assert_fail ("NodeA && \"Should only process reachable instructions\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4334, __extension__ __PRETTY_FUNCTION__))
;
4335 assert(NodeB && "Should only process reachable instructions")(static_cast <bool> (NodeB && "Should only process reachable instructions"
) ? void (0) : __assert_fail ("NodeB && \"Should only process reachable instructions\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4335, __extension__ __PRETTY_FUNCTION__))
;
4336 assert((NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&(static_cast <bool> ((NodeA == NodeB) == (NodeA->getDFSNumIn
() == NodeB->getDFSNumIn()) && "Different nodes should have different DFS numbers"
) ? void (0) : __assert_fail ("(NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) && \"Different nodes should have different DFS numbers\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4337, __extension__ __PRETTY_FUNCTION__))
4337 "Different nodes should have different DFS numbers")(static_cast <bool> ((NodeA == NodeB) == (NodeA->getDFSNumIn
() == NodeB->getDFSNumIn()) && "Different nodes should have different DFS numbers"
) ? void (0) : __assert_fail ("(NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) && \"Different nodes should have different DFS numbers\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4337, __extension__ __PRETTY_FUNCTION__))
;
4338 if (NodeA != NodeB)
4339 return NodeA->getDFSNumIn() < NodeB->getDFSNumIn();
4340 return B->comesBefore(A);
4341 });
4342
4343 for (Instruction *Inst : OrderedScalars) {
4344 if (!PrevInst) {
4345 PrevInst = Inst;
4346 continue;
4347 }
4348
4349 // Update LiveValues.
4350 LiveValues.erase(PrevInst);
4351 for (auto &J : PrevInst->operands()) {
4352 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
4353 LiveValues.insert(cast<Instruction>(&*J));
4354 }
4355
4356 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
4357 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
4358 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
4359 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
4360 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
4361 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
4362 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
;
4363
4364 // Now find the sequence of instructions between PrevInst and Inst.
4365 unsigned NumCalls = 0;
4366 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
4367 PrevInstIt =
4368 PrevInst->getIterator().getReverse();
4369 while (InstIt != PrevInstIt) {
4370 if (PrevInstIt == PrevInst->getParent()->rend()) {
4371 PrevInstIt = Inst->getParent()->rbegin();
4372 continue;
4373 }
4374
4375 // Debug information does not impact spill cost.
4376 if ((isa<CallInst>(&*PrevInstIt) &&
4377 !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
4378 &*PrevInstIt != PrevInst)
4379 NumCalls++;
4380
4381 ++PrevInstIt;
4382 }
4383
4384 if (NumCalls) {
4385 SmallVector<Type*, 4> V;
4386 for (auto *II : LiveValues) {
4387 auto *ScalarTy = II->getType();
4388 if (auto *VectorTy = dyn_cast<FixedVectorType>(ScalarTy))
4389 ScalarTy = VectorTy->getElementType();
4390 V.push_back(FixedVectorType::get(ScalarTy, BundleWidth));
4391 }
4392 Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(V);
4393 }
4394
4395 PrevInst = Inst;
4396 }
4397
4398 return Cost;
4399}
4400
4401InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
4402 InstructionCost Cost = 0;
4403 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
4404 << VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
4405
4406 unsigned BundleWidth = VectorizableTree[0]->Scalars.size();
4407
4408 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
4409 TreeEntry &TE = *VectorizableTree[I].get();
4410
4411 InstructionCost C = getEntryCost(&TE, VectorizedVals);
4412 Cost += C;
4413 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4414 << " for bundle that starts with " << *TE.Scalars[0]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4415 << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4416 << "SLP: Current total cost = " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
;
4417 }
4418
4419 SmallPtrSet<Value *, 16> ExtractCostCalculated;
4420 InstructionCost ExtractCost = 0;
4421 SmallBitVector IsIdentity;
4422 SmallVector<unsigned> VF;
4423 SmallVector<SmallVector<int>> ShuffleMask;
4424 SmallVector<Value *> FirstUsers;
4425 SmallVector<APInt> DemandedElts;
4426 for (ExternalUser &EU : ExternalUses) {
4427 // We only add extract cost once for the same scalar.
4428 if (!ExtractCostCalculated.insert(EU.Scalar).second)
4429 continue;
4430
4431 // Uses by ephemeral values are free (because the ephemeral value will be
4432 // removed prior to code generation, and so the extraction will be
4433 // removed as well).
4434 if (EphValues.count(EU.User))
4435 continue;
4436
4437 // No extract cost for vector "scalar"
4438 if (isa<FixedVectorType>(EU.Scalar->getType()))
4439 continue;
4440
4441 // Already counted the cost for external uses when tried to adjust the cost
4442 // for extractelements, no need to add it again.
4443 if (isa<ExtractElementInst>(EU.Scalar))
4444 continue;
4445
4446 // If found user is an insertelement, do not calculate extract cost but try
4447 // to detect it as a final shuffled/identity match.
4448 if (EU.User && isa<InsertElementInst>(EU.User)) {
4449 if (auto *FTy = dyn_cast<FixedVectorType>(EU.User->getType())) {
4450 Optional<int> InsertIdx = getInsertIndex(EU.User, 0);
4451 if (!InsertIdx || *InsertIdx == UndefMaskElem)
4452 continue;
4453 Value *VU = EU.User;
4454 auto *It = find_if(FirstUsers, [VU](Value *V) {
4455 // Checks if 2 insertelements are from the same buildvector.
4456 if (VU->getType() != V->getType())
4457 return false;
4458 auto *IE1 = cast<InsertElementInst>(VU);
4459 auto *IE2 = cast<InsertElementInst>(V);
4460 // Go though of insertelement instructions trying to find either VU as
4461 // the original vector for IE2 or V as the original vector for IE1.
4462 do {
4463 if (IE1 == VU || IE2 == V)
4464 return true;
4465 if (IE1)
4466 IE1 = dyn_cast<InsertElementInst>(IE1->getOperand(0));
4467 if (IE2)
4468 IE2 = dyn_cast<InsertElementInst>(IE2->getOperand(0));
4469 } while (IE1 || IE2);
4470 return false;
4471 });
4472 int VecId = -1;
4473 if (It == FirstUsers.end()) {
4474 VF.push_back(FTy->getNumElements());
4475 ShuffleMask.emplace_back(VF.back(), UndefMaskElem);
4476 FirstUsers.push_back(EU.User);
4477 DemandedElts.push_back(APInt::getNullValue(VF.back()));
4478 IsIdentity.push_back(true);
4479 VecId = FirstUsers.size() - 1;
4480 } else {
4481 VecId = std::distance(FirstUsers.begin(), It);
4482 }
4483 int Idx = *InsertIdx;
4484 ShuffleMask[VecId][Idx] = EU.Lane;
4485 IsIdentity.set(IsIdentity.test(VecId) &
4486 (EU.Lane == Idx || EU.Lane == UndefMaskElem));
4487 DemandedElts[VecId].setBit(Idx);
4488 }
4489 }
4490
4491 // If we plan to rewrite the tree in a smaller type, we will need to sign
4492 // extend the extracted value back to the original type. Here, we account
4493 // for the extract and the added cost of the sign extend if needed.
4494 auto *VecTy = FixedVectorType::get(EU.Scalar->getType(), BundleWidth);
4495 auto *ScalarRoot = VectorizableTree[0]->Scalars[0];
4496 if (MinBWs.count(ScalarRoot)) {
4497 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
4498 auto Extend =
4499 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
4500 VecTy = FixedVectorType::get(MinTy, BundleWidth);
4501 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
4502 VecTy, EU.Lane);
4503 } else {
4504 ExtractCost +=
4505 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
4506 }
4507 }
4508
4509 InstructionCost SpillCost = getSpillCost();
4510 Cost += SpillCost + ExtractCost;
4511 for (int I = 0, E = FirstUsers.size(); I < E; ++I) {
4512 if (!IsIdentity.test(I)) {
4513 InstructionCost C = TTI->getShuffleCost(
4514 TTI::SK_PermuteSingleSrc,
4515 cast<FixedVectorType>(FirstUsers[I]->getType()), ShuffleMask[I]);
4516 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of insertelement external users " <<
*VectorizableTree.front()->Scalars.front() << ".\n"
<< "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4517 << " for final shuffle of insertelement external users "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of insertelement external users " <<
*VectorizableTree.front()->Scalars.front() << ".\n"
<< "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4518 << *VectorizableTree.front()->Scalars.front() << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of insertelement external users " <<
*VectorizableTree.front()->Scalars.front() << ".\n"
<< "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4519 << "SLP: Current total cost = " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of insertelement external users " <<
*VectorizableTree.front()->Scalars.front() << ".\n"
<< "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
;
4520 Cost += C;
4521 }
4522 unsigned VF = ShuffleMask[I].size();
4523 for (int &Mask : ShuffleMask[I])
4524 Mask = (Mask == UndefMaskElem ? 0 : VF) + Mask;
4525 InstructionCost C = TTI->getShuffleCost(
4526 TTI::SK_PermuteTwoSrc, cast<FixedVectorType>(FirstUsers[I]->getType()),
4527 ShuffleMask[I]);
4528 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of vector node and external insertelement users "
<< *VectorizableTree.front()->Scalars.front() <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4529 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of vector node and external insertelement users "
<< *VectorizableTree.front()->Scalars.front() <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4530 << "SLP: Adding cost " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of vector node and external insertelement users "
<< *VectorizableTree.front()->Scalars.front() <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4531 << " for final shuffle of vector node and external insertelement users "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of vector node and external insertelement users "
<< *VectorizableTree.front()->Scalars.front() <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4532 << *VectorizableTree.front()->Scalars.front() << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of vector node and external insertelement users "
<< *VectorizableTree.front()->Scalars.front() <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
4533 << "SLP: Current total cost = " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for final shuffle of vector node and external insertelement users "
<< *VectorizableTree.front()->Scalars.front() <<
".\n" << "SLP: Current total cost = " << Cost <<
"\n"; } } while (false)
;
4534 Cost += C;
4535 InstructionCost InsertCost = TTI->getScalarizationOverhead(
4536 cast<FixedVectorType>(FirstUsers[I]->getType()), DemandedElts[I],
4537 /*Insert*/ true,
4538 /*Extract*/ false);
4539 Cost -= InsertCost;
4540 LLVM_DEBUG(dbgs() << "SLP: subtracting the cost " << InsertCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: subtracting the cost " <<
InsertCost << " for insertelements gather.\n" <<
"SLP: Current total cost = " << Cost << "\n"; } }
while (false)
4541 << " for insertelements gather.\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: subtracting the cost " <<
InsertCost << " for insertelements gather.\n" <<
"SLP: Current total cost = " << Cost << "\n"; } }
while (false)
4542 << "SLP: Current total cost = " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: subtracting the cost " <<
InsertCost << " for insertelements gather.\n" <<
"SLP: Current total cost = " << Cost << "\n"; } }
while (false)
;
4543 }
4544
4545#ifndef NDEBUG
4546 SmallString<256> Str;
4547 {
4548 raw_svector_ostream OS(Str);
4549 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
4550 << "SLP: Extract Cost = " << ExtractCost << ".\n"
4551 << "SLP: Total Cost = " << Cost << ".\n";
4552 }
4553 LLVM_DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
4554 if (ViewSLPTree)
4555 ViewGraph(this, "SLP" + F->getName(), false, Str);
4556#endif
4557
4558 return Cost;
4559}
4560
4561Optional<TargetTransformInfo::ShuffleKind>
4562BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
4563 SmallVectorImpl<const TreeEntry *> &Entries) {
4564 // TODO: currently checking only for Scalars in the tree entry, need to count
4565 // reused elements too for better cost estimation.
4566 Mask.assign(TE->Scalars.size(), UndefMaskElem);
4567 Entries.clear();
4568 // Build a lists of values to tree entries.
4569 DenseMap<Value *, SmallPtrSet<const TreeEntry *, 4>> ValueToTEs;
4570 for (const std::unique_ptr<TreeEntry> &EntryPtr : VectorizableTree) {
4571 if (EntryPtr.get() == TE)
4572 break;
4573 if (EntryPtr->State != TreeEntry::NeedToGather)
4574 continue;
4575 for (Value *V : EntryPtr->Scalars)
4576 ValueToTEs.try_emplace(V).first->getSecond().insert(EntryPtr.get());
4577 }
4578 // Find all tree entries used by the gathered values. If no common entries
4579 // found - not a shuffle.
4580 // Here we build a set of tree nodes for each gathered value and trying to
4581 // find the intersection between these sets. If we have at least one common
4582 // tree node for each gathered value - we have just a permutation of the
4583 // single vector. If we have 2 different sets, we're in situation where we
4584 // have a permutation of 2 input vectors.
4585 SmallVector<SmallPtrSet<const TreeEntry *, 4>> UsedTEs;
4586 DenseMap<Value *, int> UsedValuesEntry;
4587 for (Value *V : TE->Scalars) {
4588 if (isa<UndefValue>(V))
4589 continue;
4590 // Build a list of tree entries where V is used.
4591 SmallPtrSet<const TreeEntry *, 4> VToTEs;
4592 auto It = ValueToTEs.find(V);
4593 if (It != ValueToTEs.end())
4594 VToTEs = It->second;
4595 if (const TreeEntry *VTE = getTreeEntry(V))
4596 VToTEs.insert(VTE);
4597 if (VToTEs.empty())
4598 return None;
4599 if (UsedTEs.empty()) {
4600 // The first iteration, just insert the list of nodes to vector.
4601 UsedTEs.push_back(VToTEs);
4602 } else {
4603 // Need to check if there are any previously used tree nodes which use V.
4604 // If there are no such nodes, consider that we have another one input
4605 // vector.
4606 SmallPtrSet<const TreeEntry *, 4> SavedVToTEs(VToTEs);
4607 unsigned Idx = 0;
4608 for (SmallPtrSet<const TreeEntry *, 4> &Set : UsedTEs) {
4609 // Do we have a non-empty intersection of previously listed tree entries
4610 // and tree entries using current V?
4611 set_intersect(VToTEs, Set);
4612 if (!VToTEs.empty()) {
4613 // Yes, write the new subset and continue analysis for the next
4614 // scalar.
4615 Set.swap(VToTEs);
4616 break;
4617 }
4618 VToTEs = SavedVToTEs;
4619 ++Idx;
4620 }
4621 // No non-empty intersection found - need to add a second set of possible
4622 // source vectors.
4623 if (Idx == UsedTEs.size()) {
4624 // If the number of input vectors is greater than 2 - not a permutation,
4625 // fallback to the regular gather.
4626 if (UsedTEs.size() == 2)
4627 return None;
4628 UsedTEs.push_back(SavedVToTEs);
4629 Idx = UsedTEs.size() - 1;
4630 }
4631 UsedValuesEntry.try_emplace(V, Idx);
4632 }
4633 }
4634
4635 unsigned VF = 0;
4636 if (UsedTEs.size() == 1) {
4637 // Try to find the perfect match in another gather node at first.
4638 auto It = find_if(UsedTEs.front(), [TE](const TreeEntry *EntryPtr) {
4639 return EntryPtr->isSame(TE->Scalars);
4640 });
4641 if (It != UsedTEs.front().end()) {
4642 Entries.push_back(*It);
4643 std::iota(Mask.begin(), Mask.end(), 0);
4644 return TargetTransformInfo::SK_PermuteSingleSrc;
4645 }
4646 // No perfect match, just shuffle, so choose the first tree node.
4647 Entries.push_back(*UsedTEs.front().begin());
4648 } else {
4649 // Try to find nodes with the same vector factor.
4650 assert(UsedTEs.size() == 2 && "Expected at max 2 permuted entries.")(static_cast <bool> (UsedTEs.size() == 2 && "Expected at max 2 permuted entries."
) ? void (0) : __assert_fail ("UsedTEs.size() == 2 && \"Expected at max 2 permuted entries.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4650, __extension__ __PRETTY_FUNCTION__))
;
4651 // FIXME: Shall be replaced by GetVF function once non-power-2 patch is
4652 // landed.
4653 auto &&GetVF = [](const TreeEntry *TE) {
4654 if (!TE->ReuseShuffleIndices.empty())
4655 return TE->ReuseShuffleIndices.size();
4656 return TE->Scalars.size();
4657 };
4658 DenseMap<int, const TreeEntry *> VFToTE;
4659 for (const TreeEntry *TE : UsedTEs.front())
4660 VFToTE.try_emplace(GetVF(TE), TE);
4661 for (const TreeEntry *TE : UsedTEs.back()) {
4662 auto It = VFToTE.find(GetVF(TE));
4663 if (It != VFToTE.end()) {
4664 VF = It->first;
4665 Entries.push_back(It->second);
4666 Entries.push_back(TE);
4667 break;
4668 }
4669 }
4670 // No 2 source vectors with the same vector factor - give up and do regular
4671 // gather.
4672 if (Entries.empty())
4673 return None;
4674 }
4675
4676 // Build a shuffle mask for better cost estimation and vector emission.
4677 for (int I = 0, E = TE->Scalars.size(); I < E; ++I) {
4678 Value *V = TE->Scalars[I];
4679 if (isa<UndefValue>(V))
4680 continue;
4681 unsigned Idx = UsedValuesEntry.lookup(V);
4682 const TreeEntry *VTE = Entries[Idx];
4683 int FoundLane = findLaneForValue(VTE->Scalars, VTE->ReuseShuffleIndices, V);
4684 Mask[I] = Idx * VF + FoundLane;
4685 // Extra check required by isSingleSourceMaskImpl function (called by
4686 // ShuffleVectorInst::isSingleSourceMask).
4687 if (Mask[I] >= 2 * E)
4688 return None;
4689 }
4690 switch (Entries.size()) {
4691 case 1:
4692 return TargetTransformInfo::SK_PermuteSingleSrc;
4693 case 2:
4694 return TargetTransformInfo::SK_PermuteTwoSrc;
4695 default:
4696 break;
4697 }
4698 return None;
4699}
4700
4701InstructionCost
4702BoUpSLP::getGatherCost(FixedVectorType *Ty,
4703 const DenseSet<unsigned> &ShuffledIndices) const {
4704 unsigned NumElts = Ty->getNumElements();
4705 APInt DemandedElts = APInt::getNullValue(NumElts);
4706 for (unsigned I = 0; I < NumElts; ++I)
4707 if (!ShuffledIndices.count(I))
4708 DemandedElts.setBit(I);
4709 InstructionCost Cost =
4710 TTI->getScalarizationOverhead(Ty, DemandedElts, /*Insert*/ true,
4711 /*Extract*/ false);
4712 if (!ShuffledIndices.empty())
4713 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
4714 return Cost;
4715}
4716
4717InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL) const {
4718 // Find the type of the operands in VL.
4719 Type *ScalarTy = VL[0]->getType();
4720 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
4721 ScalarTy = SI->getValueOperand()->getType();
4722 auto *VecTy = FixedVectorType::get(ScalarTy, VL.size());
4723 // Find the cost of inserting/extracting values from the vector.
4724 // Check if the same elements are inserted several times and count them as
4725 // shuffle candidates.
4726 DenseSet<unsigned> ShuffledElements;
4727 DenseSet<Value *> UniqueElements;
4728 // Iterate in reverse order to consider insert elements with the high cost.
4729 for (unsigned I = VL.size(); I > 0; --I) {
4730 unsigned Idx = I - 1;
4731 if (isConstant(VL[Idx]))
4732 continue;
4733 if (!UniqueElements.insert(VL[Idx]).second)
4734 ShuffledElements.insert(Idx);
4735 }
4736 return getGatherCost(VecTy, ShuffledElements);
4737}
4738
4739// Perform operand reordering on the instructions in VL and return the reordered
4740// operands in Left and Right.
4741void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
4742 SmallVectorImpl<Value *> &Left,
4743 SmallVectorImpl<Value *> &Right,
4744 const DataLayout &DL,
4745 ScalarEvolution &SE,
4746 const BoUpSLP &R) {
4747 if (VL.empty())
4748 return;
4749 VLOperands Ops(VL, DL, SE, R);
4750 // Reorder the operands in place.
4751 Ops.reorder();
4752 Left = Ops.getVL(0);
4753 Right = Ops.getVL(1);
4754}
4755
4756void BoUpSLP::setInsertPointAfterBundle(const TreeEntry *E) {
4757 // Get the basic block this bundle is in. All instructions in the bundle
4758 // should be in this block.
4759 auto *Front = E->getMainOp();
4760 auto *BB = Front->getParent();
4761 assert(llvm::all_of(E->Scalars, [=](Value *V) -> bool {(static_cast <bool> (llvm::all_of(E->Scalars, [=](Value
*V) -> bool { auto *I = cast<Instruction>(V); return
!E->isOpcodeOrAlt(I) || I->getParent() == BB; })) ? void
(0) : __assert_fail ("llvm::all_of(E->Scalars, [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4764, __extension__ __PRETTY_FUNCTION__))
4762 auto *I = cast<Instruction>(V);(static_cast <bool> (llvm::all_of(E->Scalars, [=](Value
*V) -> bool { auto *I = cast<Instruction>(V); return
!E->isOpcodeOrAlt(I) || I->getParent() == BB; })) ? void
(0) : __assert_fail ("llvm::all_of(E->Scalars, [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4764, __extension__ __PRETTY_FUNCTION__))
4763 return !E->isOpcodeOrAlt(I) || I->getParent() == BB;(static_cast <bool> (llvm::all_of(E->Scalars, [=](Value
*V) -> bool { auto *I = cast<Instruction>(V); return
!E->isOpcodeOrAlt(I) || I->getParent() == BB; })) ? void
(0) : __assert_fail ("llvm::all_of(E->Scalars, [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4764, __extension__ __PRETTY_FUNCTION__))
4764 }))(static_cast <bool> (llvm::all_of(E->Scalars, [=](Value
*V) -> bool { auto *I = cast<Instruction>(V); return
!E->isOpcodeOrAlt(I) || I->getParent() == BB; })) ? void
(0) : __assert_fail ("llvm::all_of(E->Scalars, [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !E->isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4764, __extension__ __PRETTY_FUNCTION__))
;
4765
4766 // The last instruction in the bundle in program order.
4767 Instruction *LastInst = nullptr;
4768
4769 // Find the last instruction. The common case should be that BB has been
4770 // scheduled, and the last instruction is VL.back(). So we start with
4771 // VL.back() and iterate over schedule data until we reach the end of the
4772 // bundle. The end of the bundle is marked by null ScheduleData.
4773 if (BlocksSchedules.count(BB)) {
4774 auto *Bundle =
4775 BlocksSchedules[BB]->getScheduleData(E->isOneOf(E->Scalars.back()));
4776 if (Bundle && Bundle->isPartOfBundle())
4777 for (; Bundle; Bundle = Bundle->NextInBundle)
4778 if (Bundle->OpValue == Bundle->Inst)
4779 LastInst = Bundle->Inst;
4780 }
4781
4782 // LastInst can still be null at this point if there's either not an entry
4783 // for BB in BlocksSchedules or there's no ScheduleData available for
4784 // VL.back(). This can be the case if buildTree_rec aborts for various
4785 // reasons (e.g., the maximum recursion depth is reached, the maximum region
4786 // size is reached, etc.). ScheduleData is initialized in the scheduling
4787 // "dry-run".
4788 //
4789 // If this happens, we can still find the last instruction by brute force. We
4790 // iterate forwards from Front (inclusive) until we either see all
4791 // instructions in the bundle or reach the end of the block. If Front is the
4792 // last instruction in program order, LastInst will be set to Front, and we
4793 // will visit all the remaining instructions in the block.
4794 //
4795 // One of the reasons we exit early from buildTree_rec is to place an upper
4796 // bound on compile-time. Thus, taking an additional compile-time hit here is
4797 // not ideal. However, this should be exceedingly rare since it requires that
4798 // we both exit early from buildTree_rec and that the bundle be out-of-order
4799 // (causing us to iterate all the way to the end of the block).
4800 if (!LastInst) {
4801 SmallPtrSet<Value *, 16> Bundle(E->Scalars.begin(), E->Scalars.end());
4802 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
4803 if (Bundle.erase(&I) && E->isOpcodeOrAlt(&I))
4804 LastInst = &I;
4805 if (Bundle.empty())
4806 break;
4807 }
4808 }
4809 assert(LastInst && "Failed to find last instruction in bundle")(static_cast <bool> (LastInst && "Failed to find last instruction in bundle"
) ? void (0) : __assert_fail ("LastInst && \"Failed to find last instruction in bundle\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4809, __extension__ __PRETTY_FUNCTION__))
;
4810
4811 // Set the insertion point after the last instruction in the bundle. Set the
4812 // debug location to Front.
4813 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
4814 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
4815}
4816
4817Value *BoUpSLP::gather(ArrayRef<Value *> VL) {
4818 // List of instructions/lanes from current block and/or the blocks which are
4819 // part of the current loop. These instructions will be inserted at the end to
4820 // make it possible to optimize loops and hoist invariant instructions out of
4821 // the loops body with better chances for success.
4822 SmallVector<std::pair<Value *, unsigned>, 4> PostponedInsts;
4823 SmallSet<int, 4> PostponedIndices;
4824 Loop *L = LI->getLoopFor(Builder.GetInsertBlock());
4825 auto &&CheckPredecessor = [](BasicBlock *InstBB, BasicBlock *InsertBB) {
4826 SmallPtrSet<BasicBlock *, 4> Visited;
4827 while (InsertBB && InsertBB != InstBB && Visited.insert(InsertBB).second)
4828 InsertBB = InsertBB->getSinglePredecessor();
4829 return InsertBB && InsertBB == InstBB;
4830 };
4831 for (int I = 0, E = VL.size(); I < E; ++I) {
4832 if (auto *Inst = dyn_cast<Instruction>(VL[I]))
4833 if ((CheckPredecessor(Inst->getParent(), Builder.GetInsertBlock()) ||
4834 getTreeEntry(Inst) || (L && (L->contains(Inst)))) &&
4835 PostponedIndices.insert(I).second)
4836 PostponedInsts.emplace_back(Inst, I);
4837 }
4838
4839 auto &&CreateInsertElement = [this](Value *Vec, Value *V, unsigned Pos) {
4840 // No need to insert undefs elements - exit.
4841 if (isa<UndefValue>(V))
4842 return Vec;
4843 Vec = Builder.CreateInsertElement(Vec, V, Builder.getInt32(Pos));
4844 auto *InsElt = dyn_cast<InsertElementInst>(Vec);
4845 if (!InsElt)
4846 return Vec;
4847 GatherSeq.insert(InsElt);
4848 CSEBlocks.insert(InsElt->getParent());
4849 // Add to our 'need-to-extract' list.
4850 if (TreeEntry *Entry = getTreeEntry(V)) {
4851 // Find which lane we need to extract.
4852 unsigned FoundLane =
4853 std::distance(Entry->Scalars.begin(), find(Entry->Scalars, V));
4854 assert(FoundLane < Entry->Scalars.size() && "Couldn't find extract lane")(static_cast <bool> (FoundLane < Entry->Scalars.size
() && "Couldn't find extract lane") ? void (0) : __assert_fail
("FoundLane < Entry->Scalars.size() && \"Couldn't find extract lane\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4854, __extension__ __PRETTY_FUNCTION__))
;
4855 if (!Entry->ReuseShuffleIndices.empty()) {
4856 FoundLane = std::distance(Entry->ReuseShuffleIndices.begin(),
4857 find(Entry->ReuseShuffleIndices, FoundLane));
4858 }
4859 ExternalUses.emplace_back(V, InsElt, FoundLane);
4860 }
4861 return Vec;
4862 };
4863 Value *Val0 =
4864 isa<StoreInst>(VL[0]) ? cast<StoreInst>(VL[0])->getValueOperand() : VL[0];
4865 FixedVectorType *VecTy = FixedVectorType::get(Val0->getType(), VL.size());
4866 Value *Vec = PoisonValue::get(VecTy);
4867 for (int I = 0, E = VL.size(); I < E; ++I) {
4868 if (PostponedIndices.contains(I))
4869 continue;
4870 Vec = CreateInsertElement(Vec, VL[I], I);
4871 }
4872 // Append instructions, which are/may be part of the loop, in the end to make
4873 // it possible to hoist non-loop-based instructions.
4874 for (const std::pair<Value *, unsigned> &Pair : PostponedInsts)
4875 Vec = CreateInsertElement(Vec, Pair.first, Pair.second);
4876
4877 return Vec;
4878}
4879
4880namespace {
4881/// Merges shuffle masks and emits final shuffle instruction, if required.
4882class ShuffleInstructionBuilder {
4883 IRBuilderBase &Builder;
4884 const unsigned VF = 0;
4885 bool IsFinalized = false;
4886 SmallVector<int, 4> Mask;
4887
4888public:
4889 ShuffleInstructionBuilder(IRBuilderBase &Builder, unsigned VF)
4890 : Builder(Builder), VF(VF) {}
4891
4892 /// Adds a mask, inverting it before applying.
4893 void addInversedMask(ArrayRef<unsigned> SubMask) {
4894 if (SubMask.empty())
4895 return;
4896 SmallVector<int, 4> NewMask;
4897 inversePermutation(SubMask, NewMask);
4898 addMask(NewMask);
4899 }
4900
4901 /// Functions adds masks, merging them into single one.
4902 void addMask(ArrayRef<unsigned> SubMask) {
4903 SmallVector<int, 4> NewMask(SubMask.begin(), SubMask.end());
4904 addMask(NewMask);
4905 }
4906
4907 void addMask(ArrayRef<int> SubMask) {
4908 if (SubMask.empty())
4909 return;
4910 if (Mask.empty()) {
4911 Mask.append(SubMask.begin(), SubMask.end());
4912 return;
4913 }
4914 SmallVector<int, 4> NewMask(SubMask.size(), SubMask.size());
4915 int TermValue = std::min(Mask.size(), SubMask.size());
4916 for (int I = 0, E = SubMask.size(); I < E; ++I) {
4917 if (SubMask[I] >= TermValue || SubMask[I] == UndefMaskElem ||
4918 Mask[SubMask[I]] >= TermValue) {
4919 NewMask[I] = UndefMaskElem;
4920 continue;
4921 }
4922 NewMask[I] = Mask[SubMask[I]];
4923 }
4924 Mask.swap(NewMask);
4925 }
4926
4927 Value *finalize(Value *V) {
4928 IsFinalized = true;
4929 unsigned ValueVF = cast<FixedVectorType>(V->getType())->getNumElements();
4930 if (VF == ValueVF && Mask.empty())
4931 return V;
4932 SmallVector<int, 4> NormalizedMask(VF, UndefMaskElem);
4933 std::iota(NormalizedMask.begin(), NormalizedMask.end(), 0);
4934 addMask(NormalizedMask);
4935
4936 if (VF == ValueVF && ShuffleVectorInst::isIdentityMask(Mask))
4937 return V;
4938 return Builder.CreateShuffleVector(V, Mask, "shuffle");
4939 }
4940
4941 ~ShuffleInstructionBuilder() {
4942 assert((IsFinalized || Mask.empty()) &&(static_cast <bool> ((IsFinalized || Mask.empty()) &&
"Shuffle construction must be finalized.") ? void (0) : __assert_fail
("(IsFinalized || Mask.empty()) && \"Shuffle construction must be finalized.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4943, __extension__ __PRETTY_FUNCTION__))
4943 "Shuffle construction must be finalized.")(static_cast <bool> ((IsFinalized || Mask.empty()) &&
"Shuffle construction must be finalized.") ? void (0) : __assert_fail
("(IsFinalized || Mask.empty()) && \"Shuffle construction must be finalized.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4943, __extension__ __PRETTY_FUNCTION__))
;
4944 }
4945};
4946} // namespace
4947
4948Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
4949 unsigned VF = VL.size();
4950 InstructionsState S = getSameOpcode(VL);
4951 if (S.getOpcode()) {
4952 if (TreeEntry *E = getTreeEntry(S.OpValue))
4953 if (E->isSame(VL)) {
4954 Value *V = vectorizeTree(E);
4955 if (VF != cast<FixedVectorType>(V->getType())->getNumElements()) {
4956 if (!E->ReuseShuffleIndices.empty()) {
4957 // Reshuffle to get only unique values.
4958 // If some of the scalars are duplicated in the vectorization tree
4959 // entry, we do not vectorize them but instead generate a mask for
4960 // the reuses. But if there are several users of the same entry,
4961 // they may have different vectorization factors. This is especially
4962 // important for PHI nodes. In this case, we need to adapt the
4963 // resulting instruction for the user vectorization factor and have
4964 // to reshuffle it again to take only unique elements of the vector.
4965 // Without this code the function incorrectly returns reduced vector
4966 // instruction with the same elements, not with the unique ones.
4967
4968 // block:
4969 // %phi = phi <2 x > { .., %entry} {%shuffle, %block}
4970 // %2 = shuffle <2 x > %phi, %poison, <4 x > <0, 0, 1, 1>
4971 // ... (use %2)
4972 // %shuffle = shuffle <2 x> %2, poison, <2 x> {0, 2}
4973 // br %block
4974 SmallVector<int> UniqueIdxs;
4975 SmallSet<int, 4> UsedIdxs;
4976 int Pos = 0;
4977 int Sz = VL.size();
4978 for (int Idx : E->ReuseShuffleIndices) {
4979 if (Idx != Sz && UsedIdxs.insert(Idx).second)
4980 UniqueIdxs.emplace_back(Pos);
4981 ++Pos;
4982 }
4983 assert(VF >= UsedIdxs.size() && "Expected vectorization factor "(static_cast <bool> (VF >= UsedIdxs.size() &&
"Expected vectorization factor " "less than original vector size."
) ? void (0) : __assert_fail ("VF >= UsedIdxs.size() && \"Expected vectorization factor \" \"less than original vector size.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4984, __extension__ __PRETTY_FUNCTION__))
4984 "less than original vector size.")(static_cast <bool> (VF >= UsedIdxs.size() &&
"Expected vectorization factor " "less than original vector size."
) ? void (0) : __assert_fail ("VF >= UsedIdxs.size() && \"Expected vectorization factor \" \"less than original vector size.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4984, __extension__ __PRETTY_FUNCTION__))
;
4985 UniqueIdxs.append(VF - UsedIdxs.size(), UndefMaskElem);
4986 V = Builder.CreateShuffleVector(V, UniqueIdxs, "shrink.shuffle");
4987 } else {
4988 assert(VF < cast<FixedVectorType>(V->getType())->getNumElements() &&(static_cast <bool> (VF < cast<FixedVectorType>
(V->getType())->getNumElements() && "Expected vectorization factor less "
"than original vector size.") ? void (0) : __assert_fail ("VF < cast<FixedVectorType>(V->getType())->getNumElements() && \"Expected vectorization factor less \" \"than original vector size.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4990, __extension__ __PRETTY_FUNCTION__))
4989 "Expected vectorization factor less "(static_cast <bool> (VF < cast<FixedVectorType>
(V->getType())->getNumElements() && "Expected vectorization factor less "
"than original vector size.") ? void (0) : __assert_fail ("VF < cast<FixedVectorType>(V->getType())->getNumElements() && \"Expected vectorization factor less \" \"than original vector size.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4990, __extension__ __PRETTY_FUNCTION__))
4990 "than original vector size.")(static_cast <bool> (VF < cast<FixedVectorType>
(V->getType())->getNumElements() && "Expected vectorization factor less "
"than original vector size.") ? void (0) : __assert_fail ("VF < cast<FixedVectorType>(V->getType())->getNumElements() && \"Expected vectorization factor less \" \"than original vector size.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4990, __extension__ __PRETTY_FUNCTION__))
;
4991 SmallVector<int> UniformMask(VF, 0);
4992 std::iota(UniformMask.begin(), UniformMask.end(), 0);
4993 V = Builder.CreateShuffleVector(V, UniformMask, "shrink.shuffle");
4994 }
4995 }
4996 return V;
4997 }
4998 }
4999
5000 // Check that every instruction appears once in this bundle.
5001 SmallVector<int> ReuseShuffleIndicies;
5002 SmallVector<Value *> UniqueValues;
5003 if (VL.size() > 2) {
5004 DenseMap<Value *, unsigned> UniquePositions;
5005 unsigned NumValues =
5006 std::distance(VL.begin(), find_if(reverse(VL), [](Value *V) {
5007 return !isa<UndefValue>(V);
5008 }).base());
5009 VF = std::max<unsigned>(VF, PowerOf2Ceil(NumValues));
5010 int UniqueVals = 0;
5011 bool HasUndefs = false;
5012 for (Value *V : VL.drop_back(VL.size() - VF)) {
5013 if (isa<UndefValue>(V)) {
5014 ReuseShuffleIndicies.emplace_back(UndefMaskElem);
5015 HasUndefs = true;
5016 continue;
5017 }
5018 if (isConstant(V)) {
5019 ReuseShuffleIndicies.emplace_back(UniqueValues.size());
5020 UniqueValues.emplace_back(V);
5021 continue;
5022 }
5023 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
5024 ReuseShuffleIndicies.emplace_back(Res.first->second);
5025 if (Res.second) {
5026 UniqueValues.emplace_back(V);
5027 ++UniqueVals;
5028 }
5029 }
5030 if (HasUndefs && UniqueVals == 1 && UniqueValues.size() == 1) {
5031 // Emit pure splat vector.
5032 // FIXME: why it is not identified as an identity.
5033 unsigned NumUndefs = count(ReuseShuffleIndicies, UndefMaskElem);
5034 if (NumUndefs == ReuseShuffleIndicies.size() - 1)
5035 ReuseShuffleIndicies.append(VF - ReuseShuffleIndicies.size(),
5036 UndefMaskElem);
5037 else
5038 ReuseShuffleIndicies.assign(VF, 0);
5039 } else if (UniqueValues.size() >= VF - 1 || UniqueValues.size() <= 1) {
5040 ReuseShuffleIndicies.clear();
5041 UniqueValues.clear();
5042 UniqueValues.append(VL.begin(), std::next(VL.begin(), NumValues));
5043 }
5044 UniqueValues.append(VF - UniqueValues.size(),
5045 UndefValue::get(VL[0]->getType()));
5046 VL = UniqueValues;
5047 }
5048
5049 ShuffleInstructionBuilder ShuffleBuilder(Builder, VF);
5050 Value *Vec = gather(VL);
5051 if (!ReuseShuffleIndicies.empty()) {
5052 ShuffleBuilder.addMask(ReuseShuffleIndicies);
5053 Vec = ShuffleBuilder.finalize(Vec);
5054 if (auto *I = dyn_cast<Instruction>(Vec)) {
5055 GatherSeq.insert(I);
5056 CSEBlocks.insert(I->getParent());
5057 }
5058 }
5059 return Vec;
5060}
5061
5062Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
5063 IRBuilder<>::InsertPointGuard Guard(Builder);
5064
5065 if (E->VectorizedValue) {
5066 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
5067 return E->VectorizedValue;
5068 }
5069
5070 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
5071 unsigned VF = E->Scalars.size();
5072 if (NeedToShuffleReuses)
5073 VF = E->ReuseShuffleIndices.size();
5074 ShuffleInstructionBuilder ShuffleBuilder(Builder, VF);
5075 if (E->State == TreeEntry::NeedToGather) {
5076 setInsertPointAfterBundle(E);
5077 Value *Vec;
5078 SmallVector<int> Mask;
5079 SmallVector<const TreeEntry *> Entries;
5080 Optional<TargetTransformInfo::ShuffleKind> Shuffle =
5081 isGatherShuffledEntry(E, Mask, Entries);
5082 if (Shuffle.hasValue()) {
5083 assert((Entries.size() == 1 || Entries.size() == 2) &&(static_cast <bool> ((Entries.size() == 1 || Entries.size
() == 2) && "Expected shuffle of 1 or 2 entries.") ? void
(0) : __assert_fail ("(Entries.size() == 1 || Entries.size() == 2) && \"Expected shuffle of 1 or 2 entries.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5084, __extension__ __PRETTY_FUNCTION__))
5084 "Expected shuffle of 1 or 2 entries.")(static_cast <bool> ((Entries.size() == 1 || Entries.size
() == 2) && "Expected shuffle of 1 or 2 entries.") ? void
(0) : __assert_fail ("(Entries.size() == 1 || Entries.size() == 2) && \"Expected shuffle of 1 or 2 entries.\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5084, __extension__ __PRETTY_FUNCTION__))
;
5085 Vec = Builder.CreateShuffleVector(Entries.front()->VectorizedValue,
5086 Entries.back()->VectorizedValue, Mask);
5087 } else {
5088 Vec = gather(E->Scalars);
5089 }
5090 if (NeedToShuffleReuses) {
5091 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5092 Vec = ShuffleBuilder.finalize(Vec);
5093 if (auto *I = dyn_cast<Instruction>(Vec)) {
5094 GatherSeq.insert(I);
5095 CSEBlocks.insert(I->getParent());
5096 }
5097 }
5098 E->VectorizedValue = Vec;
5099 return Vec;
5100 }
5101
5102 assert((E->State == TreeEntry::Vectorize ||(static_cast <bool> ((E->State == TreeEntry::Vectorize
|| E->State == TreeEntry::ScatterVectorize) && "Unhandled state"
) ? void (0) : __assert_fail ("(E->State == TreeEntry::Vectorize || E->State == TreeEntry::ScatterVectorize) && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5104, __extension__ __PRETTY_FUNCTION__))
5103 E->State == TreeEntry::ScatterVectorize) &&(static_cast <bool> ((E->State == TreeEntry::Vectorize
|| E->State == TreeEntry::ScatterVectorize) && "Unhandled state"
) ? void (0) : __assert_fail ("(E->State == TreeEntry::Vectorize || E->State == TreeEntry::ScatterVectorize) && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5104, __extension__ __PRETTY_FUNCTION__))
5104 "Unhandled state")(static_cast <bool> ((E->State == TreeEntry::Vectorize
|| E->State == TreeEntry::ScatterVectorize) && "Unhandled state"
) ? void (0) : __assert_fail ("(E->State == TreeEntry::Vectorize || E->State == TreeEntry::ScatterVectorize) && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5104, __extension__ __PRETTY_FUNCTION__))
;
5105 unsigned ShuffleOrOp =
5106 E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
5107 Instruction *VL0 = E->getMainOp();
5108 Type *ScalarTy = VL0->getType();
5109 if (auto *Store = dyn_cast<StoreInst>(VL0))
5110 ScalarTy = Store->getValueOperand()->getType();
5111 else if (auto *IE = dyn_cast<InsertElementInst>(VL0))
5112 ScalarTy = IE->getOperand(1)->getType();
5113 auto *VecTy = FixedVectorType::get(ScalarTy, E->Scalars.size());
5114 switch (ShuffleOrOp) {
5115 case Instruction::PHI: {
5116 auto *PH = cast<PHINode>(VL0);
5117 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
5118 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
5119 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
5120 Value *V = NewPhi;
5121 if (NeedToShuffleReuses)
5122 V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
5123
5124 E->VectorizedValue = V;
5125
5126 // PHINodes may have multiple entries from the same block. We want to
5127 // visit every block once.
5128 SmallPtrSet<BasicBlock*, 4> VisitedBBs;
5129
5130 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
5131 ValueList Operands;
5132 BasicBlock *IBB = PH->getIncomingBlock(i);
5133
5134 if (!VisitedBBs.insert(IBB).second) {
5135 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
5136 continue;
5137 }
5138
5139 Builder.SetInsertPoint(IBB->getTerminator());
5140 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
5141 Value *Vec = vectorizeTree(E->getOperand(i));
5142 NewPhi->addIncoming(Vec, IBB);
5143 }
5144
5145 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5146, __extension__ __PRETTY_FUNCTION__))
5146 "Invalid number of incoming values")(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5146, __extension__ __PRETTY_FUNCTION__))
;
5147 return V;
5148 }
5149
5150 case Instruction::ExtractElement: {
5151 Value *V = E->getSingleOperand(0);
5152 Builder.SetInsertPoint(VL0);
5153 ShuffleBuilder.addInversedMask(E->ReorderIndices);
5154 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5155 V = ShuffleBuilder.finalize(V);
5156 E->VectorizedValue = V;
5157 return V;
5158 }
5159 case Instruction::ExtractValue: {
5160 auto *LI = cast<LoadInst>(E->getSingleOperand(0));
5161 Builder.SetInsertPoint(LI);
5162 auto *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
5163 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
5164 LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlign());
5165 Value *NewV = propagateMetadata(V, E->Scalars);
5166 ShuffleBuilder.addInversedMask(E->ReorderIndices);
5167 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5168 NewV = ShuffleBuilder.finalize(NewV);
5169 E->VectorizedValue = NewV;
5170 return NewV;
5171 }
5172 case Instruction::InsertElement: {
5173 Builder.SetInsertPoint(VL0);
5174 Value *V = vectorizeTree(E->getOperand(1));
5175
5176 const unsigned NumElts =
5177 cast<FixedVectorType>(VL0->getType())->getNumElements();
5178 const unsigned NumScalars = E->Scalars.size();
5179
5180 // Create InsertVector shuffle if necessary
5181 Instruction *FirstInsert = nullptr;
5182 bool IsIdentity = true;
5183 unsigned Offset = UINT_MAX(2147483647 *2U +1U);
5184 for (unsigned I = 0; I < NumScalars; ++I) {
5185 Value *Scalar = E->Scalars[I];
5186 if (!FirstInsert &&
5187 !is_contained(E->Scalars, cast<Instruction>(Scalar)->getOperand(0)))
5188 FirstInsert = cast<Instruction>(Scalar);
5189 Optional<int> InsertIdx = getInsertIndex(Scalar, 0);
5190 if (!InsertIdx || *InsertIdx == UndefMaskElem)
5191 continue;
5192 unsigned Idx = *InsertIdx;
5193 if (Idx < Offset) {
5194 Offset = Idx;
5195 IsIdentity &= I == 0;
5196 } else {
5197 assert(Idx >= Offset && "Failed to find vector index offset")(static_cast <bool> (Idx >= Offset && "Failed to find vector index offset"
) ? void (0) : __assert_fail ("Idx >= Offset && \"Failed to find vector index offset\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5197, __extension__ __PRETTY_FUNCTION__))
;
5198 IsIdentity &= Idx - Offset == I;
5199 }
5200 }
5201 assert(Offset < NumElts && "Failed to find vector index offset")(static_cast <bool> (Offset < NumElts && "Failed to find vector index offset"
) ? void (0) : __assert_fail ("Offset < NumElts && \"Failed to find vector index offset\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5201, __extension__ __PRETTY_FUNCTION__))
;
5202
5203 // Create shuffle to resize vector
5204 SmallVector<int> Mask(NumElts, UndefMaskElem);
5205 if (!IsIdentity) {
5206 for (unsigned I = 0; I < NumScalars; ++I) {
5207 Value *Scalar = E->Scalars[I];
5208 Optional<int> InsertIdx = getInsertIndex(Scalar, 0);
5209 if (!InsertIdx || *InsertIdx == UndefMaskElem)
5210 continue;
5211 Mask[*InsertIdx - Offset] = I;
5212 }
5213 } else {
5214 std::iota(Mask.begin(), std::next(Mask.begin(), NumScalars), 0);
5215 }
5216 if (!IsIdentity || NumElts != NumScalars)
5217 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()), Mask);
5218
5219 if (NumElts != NumScalars) {
5220 SmallVector<int> InsertMask(NumElts);
5221 std::iota(InsertMask.begin(), InsertMask.end(), 0);
5222 for (unsigned I = 0; I < NumElts; I++) {
5223 if (Mask[I] != UndefMaskElem)
5224 InsertMask[Offset + I] = NumElts + I;
5225 }
5226
5227 V = Builder.CreateShuffleVector(
5228 FirstInsert->getOperand(0), V, InsertMask,
5229 cast<Instruction>(E->Scalars.back())->getName());
5230 }
5231
5232 ++NumVectorInstructions;
5233 E->VectorizedValue = V;
5234 return V;
5235 }
5236 case Instruction::ZExt:
5237 case Instruction::SExt:
5238 case Instruction::FPToUI:
5239 case Instruction::FPToSI:
5240 case Instruction::FPExt:
5241 case Instruction::PtrToInt:
5242 case Instruction::IntToPtr:
5243 case Instruction::SIToFP:
5244 case Instruction::UIToFP:
5245 case Instruction::Trunc:
5246 case Instruction::FPTrunc:
5247 case Instruction::BitCast: {
5248 setInsertPointAfterBundle(E);
5249
5250 Value *InVec = vectorizeTree(E->getOperand(0));
5251
5252 if (E->VectorizedValue) {
5253 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
5254 return E->VectorizedValue;
5255 }
5256
5257 auto *CI = cast<CastInst>(VL0);
5258 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
5259 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5260 V = ShuffleBuilder.finalize(V);
5261
5262 E->VectorizedValue = V;
5263 ++NumVectorInstructions;
5264 return V;
5265 }
5266 case Instruction::FCmp:
5267 case Instruction::ICmp: {
5268 setInsertPointAfterBundle(E);
5269
5270 Value *L = vectorizeTree(E->getOperand(0));
5271 Value *R = vectorizeTree(E->getOperand(1));
5272
5273 if (E->VectorizedValue) {
5274 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
5275 return E->VectorizedValue;
5276 }
5277
5278 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
5279 Value *V = Builder.CreateCmp(P0, L, R);
5280 propagateIRFlags(V, E->Scalars, VL0);
5281 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5282 V = ShuffleBuilder.finalize(V);
5283
5284 E->VectorizedValue = V;
5285 ++NumVectorInstructions;
5286 return V;
5287 }
5288 case Instruction::Select: {
5289 setInsertPointAfterBundle(E);
5290
5291 Value *Cond = vectorizeTree(E->getOperand(0));
5292 Value *True = vectorizeTree(E->getOperand(1));
5293 Value *False = vectorizeTree(E->getOperand(2));
5294
5295 if (E->VectorizedValue) {
5296 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
5297 return E->VectorizedValue;
5298 }
5299
5300 Value *V = Builder.CreateSelect(Cond, True, False);
5301 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5302 V = ShuffleBuilder.finalize(V);
5303
5304 E->VectorizedValue = V;
5305 ++NumVectorInstructions;
5306 return V;
5307 }
5308 case Instruction::FNeg: {
5309 setInsertPointAfterBundle(E);
5310
5311 Value *Op = vectorizeTree(E->getOperand(0));
5312
5313 if (E->VectorizedValue) {
5314 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
5315 return E->VectorizedValue;
5316 }
5317
5318 Value *V = Builder.CreateUnOp(
5319 static_cast<Instruction::UnaryOps>(E->getOpcode()), Op);
5320 propagateIRFlags(V, E->Scalars, VL0);
5321 if (auto *I = dyn_cast<Instruction>(V))
5322 V = propagateMetadata(I, E->Scalars);
5323
5324 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5325 V = ShuffleBuilder.finalize(V);
5326
5327 E->VectorizedValue = V;
5328 ++NumVectorInstructions;
5329
5330 return V;
5331 }
5332 case Instruction::Add:
5333 case Instruction::FAdd:
5334 case Instruction::Sub:
5335 case Instruction::FSub:
5336 case Instruction::Mul:
5337 case Instruction::FMul:
5338 case Instruction::UDiv:
5339 case Instruction::SDiv:
5340 case Instruction::FDiv:
5341 case Instruction::URem:
5342 case Instruction::SRem:
5343 case Instruction::FRem:
5344 case Instruction::Shl:
5345 case Instruction::LShr:
5346 case Instruction::AShr:
5347 case Instruction::And:
5348 case Instruction::Or:
5349 case Instruction::Xor: {
5350 setInsertPointAfterBundle(E);
5351
5352 Value *LHS = vectorizeTree(E->getOperand(0));
5353 Value *RHS = vectorizeTree(E->getOperand(1));
5354
5355 if (E->VectorizedValue) {
5356 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
5357 return E->VectorizedValue;
5358 }
5359
5360 Value *V = Builder.CreateBinOp(
5361 static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS,
5362 RHS);
5363 propagateIRFlags(V, E->Scalars, VL0);
5364 if (auto *I = dyn_cast<Instruction>(V))
5365 V = propagateMetadata(I, E->Scalars);
5366
5367 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5368 V = ShuffleBuilder.finalize(V);
5369
5370 E->VectorizedValue = V;
5371 ++NumVectorInstructions;
5372
5373 return V;
5374 }
5375 case Instruction::Load: {
5376 // Loads are inserted at the head of the tree because we don't want to
5377 // sink them all the way down past store instructions.
5378 bool IsReorder = E->updateStateIfReorder();
5379 if (IsReorder)
5380 VL0 = E->getMainOp();
5381 setInsertPointAfterBundle(E);
5382
5383 LoadInst *LI = cast<LoadInst>(VL0);
5384 Instruction *NewLI;
5385 unsigned AS = LI->getPointerAddressSpace();
5386 Value *PO = LI->getPointerOperand();
5387 if (E->State == TreeEntry::Vectorize) {
5388
5389 Value *VecPtr = Builder.CreateBitCast(PO, VecTy->getPointerTo(AS));
5390
5391 // The pointer operand uses an in-tree scalar so we add the new BitCast
5392 // to ExternalUses list to make sure that an extract will be generated
5393 // in the future.
5394 if (getTreeEntry(PO))
5395 ExternalUses.emplace_back(PO, cast<User>(VecPtr), 0);
5396
5397 NewLI = Builder.CreateAlignedLoad(VecTy, VecPtr, LI->getAlign());
5398 } else {
5399 assert(E->State == TreeEntry::ScatterVectorize && "Unhandled state")(static_cast <bool> (E->State == TreeEntry::ScatterVectorize
&& "Unhandled state") ? void (0) : __assert_fail ("E->State == TreeEntry::ScatterVectorize && \"Unhandled state\""
, "/build/llvm-toolchain-snapshot-13~++20210610111127+c5ffc6f8bd6a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5399, __extension__ __PRETTY_FUNCTION__))
;
5400 Value *VecPtr = vectorizeTree(E->getOperand(0));
5401 // Use the minimum alignment of the gathered loads.
5402 Align CommonAlignment = LI->getAlign();
5403 for (Value *V : E->Scalars)
5404 CommonAlignment =
5405 commonAlignment(CommonAlignment, cast<LoadInst>(V)->getAlign());
5406 NewLI = Builder.CreateMaskedGather(VecPtr, CommonAlignment);
5407 }
5408 Value *V = propagateMetadata(NewLI, E->Scalars);
5409
5410 ShuffleBuilder.addInversedMask(E->ReorderIndices);
5411 ShuffleBuilder.addMask(E->ReuseShuffleIndices);
5412 V = ShuffleBuilder.finalize(V);
5413 E->VectorizedValue = V;
5414 ++NumVectorInstructions;
5415 return V;
5416 }
5417 case Instruction::Store: {
5418 bool IsReorder = !E->ReorderIndices.empty();
5419 auto *SI = cast<StoreInst>(
5420 IsReorder ? E->Scalars[E->ReorderIndices.front()] : VL0);
5421 unsigned AS = SI->getPointerAddressSpace();
5422
5423 setInsertPointAfterBundle(E);
5424
5425 Value *VecValue = vectorizeTree(E->getOperand(0));
5426 ShuffleBuilder.addMask(E->ReorderIndices);
5427 VecValue = ShuffleBuilder.finalize(VecValue);
5428
5429 Value *ScalarPtr = SI->getPointerOperand();
5430 Value *VecPtr = Builder.CreateBitCast(
5431 ScalarPtr, VecValue->getType()->getPointerTo(AS));
5432 StoreInst *ST = Builder.CreateAlignedStore(VecValue, VecPtr,
5433 SI->getAlign());
5434
5435 // The pointer operand uses an in-tree scalar, so add the new BitCast to
5436 // ExternalUses to make sure that an extract will be generated in the
5437