Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 3312, column 27
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SLPVectorizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn325118/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-7~svn325118/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn325118/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn325118/build-llvm/lib/Transforms/Vectorize -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-02-14-150435-17243-1 -x c++ /build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp

/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp

1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11// stores that can be put together into vector-stores. Next, it attempts to
12// construct vectorizable tree using the use-def chains. If a profitable tree
13// was found, the SLP vectorizer performs vectorization on the tree.
14//
15// The pass is inspired by the work described in the paper:
16// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/DenseSet.h"
24#include "llvm/ADT/MapVector.h"
25#include "llvm/ADT/None.h"
26#include "llvm/ADT/Optional.h"
27#include "llvm/ADT/PostOrderIterator.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SetVector.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/iterator.h"
35#include "llvm/ADT/iterator_range.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/Analysis/CodeMetrics.h"
38#include "llvm/Analysis/DemandedBits.h"
39#include "llvm/Analysis/GlobalsModRef.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PassManager.h"
69#include "llvm/IR/PatternMatch.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/IR/Verifier.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/DOTGraphTraits.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/ErrorHandling.h"
83#include "llvm/Support/GraphWriter.h"
84#include "llvm/Support/KnownBits.h"
85#include "llvm/Support/MathExtras.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Transforms/Utils/LoopUtils.h"
88#include "llvm/Transforms/Vectorize.h"
89#include <algorithm>
90#include <cassert>
91#include <cstdint>
92#include <iterator>
93#include <memory>
94#include <set>
95#include <string>
96#include <tuple>
97#include <utility>
98#include <vector>
99
100using namespace llvm;
101using namespace llvm::PatternMatch;
102using namespace slpvectorizer;
103
104#define SV_NAME"slp-vectorizer" "slp-vectorizer"
105#define DEBUG_TYPE"SLP" "SLP"
106
107STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, {false}}
;
108
109static cl::opt<int>
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
112 "number "));
113
114static cl::opt<bool>
115ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
117
118static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120 cl::desc(
121 "Attempt to vectorize horizontal reductions feeding into a store"));
122
123static cl::opt<int>
124MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
126
127/// Limits the size of scheduling regions in a block.
128/// It avoid long compile times for _very_ large blocks where vector
129/// instructions are spread over a wide range.
130/// This limit is way higher than needed by real-world functions.
131static cl::opt<int>
132ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
134
135static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
138
139static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
142
143static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
146
147static cl::opt<bool>
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
150
151// Limit the number of alias checks. The limit is chosen so that
152// it has no negative effect on the llvm benchmarks.
153static const unsigned AliasedCheckLimit = 10;
154
155// Another limit for the alias checks: The maximum distance between load/store
156// instructions where alias checks are done.
157// This limit is useful for very large basic blocks.
158static const unsigned MaxMemDepDistance = 160;
159
160/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161/// regions to be handled.
162static const int MinScheduleRegionSize = 16;
163
164/// \brief Predicate for the element types that the SLP vectorizer supports.
165///
166/// The most important thing to filter here are types which are invalid in LLVM
167/// vectors. We also filter target specific types which have absolutely no
168/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169/// avoids spending time checking the cost model and realizing that they will
170/// be inevitably scalarized.
171static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
174}
175
176/// \returns true if all of the instructions in \p VL are in the same block or
177/// false otherwise.
178static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180 if (!I0)
181 return false;
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
185 if (!I)
186 return false;
187
188 if (BB != I->getParent())
189 return false;
190 }
191 return true;
192}
193
194/// \returns True if all of the values in \p VL are constants.
195static bool allConstant(ArrayRef<Value *> VL) {
196 for (Value *i : VL)
197 if (!isa<Constant>(i))
198 return false;
199 return true;
200}
201
202/// \returns True if all of the values in \p VL are identical.
203static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
205 if (VL[i] != VL[0])
206 return false;
207 return true;
208}
209
210/// Checks if the vector of instructions can be represented as a shuffle, like:
211/// %x0 = extractelement <4 x i8> %x, i32 0
212/// %x3 = extractelement <4 x i8> %x, i32 3
213/// %y1 = extractelement <4 x i8> %y, i32 1
214/// %y2 = extractelement <4 x i8> %y, i32 2
215/// %x0x0 = mul i8 %x0, %x0
216/// %x3x3 = mul i8 %x3, %x3
217/// %y1y1 = mul i8 %y1, %y1
218/// %y2y2 = mul i8 %y2, %y2
219/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223/// ret <4 x i8> %ins4
224/// can be transformed into:
225/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226/// i32 6>
227/// %2 = mul <4 x i8> %1, %1
228/// ret <4 x i8> %2
229/// We convert this initially to something like:
230/// %x0 = extractelement <4 x i8> %x, i32 0
231/// %x3 = extractelement <4 x i8> %x, i32 3
232/// %y1 = extractelement <4 x i8> %y, i32 1
233/// %y2 = extractelement <4 x i8> %y, i32 2
234/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238/// %5 = mul <4 x i8> %4, %4
239/// %6 = extractelement <4 x i8> %5, i32 0
240/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241/// %7 = extractelement <4 x i8> %5, i32 1
242/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243/// %8 = extractelement <4 x i8> %5, i32 2
244/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245/// %9 = extractelement <4 x i8> %5, i32 3
246/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247/// ret <4 x i8> %ins4
248/// InstCombiner transforms this into a shuffle and vector mul
249static Optional<TargetTransformInfo::ShuffleKind>
250isShuffle(ArrayRef<Value *> VL) {
251 auto *EI0 = cast<ExtractElementInst>(VL[0]);
252 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253 Value *Vec1 = nullptr;
254 Value *Vec2 = nullptr;
255 enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256 ShuffleMode CommonShuffleMode = Unknown;
257 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258 auto *EI = cast<ExtractElementInst>(VL[I]);
259 auto *Vec = EI->getVectorOperand();
260 // All vector operands must have the same number of vector elements.
261 if (Vec->getType()->getVectorNumElements() != Size)
262 return None;
263 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264 if (!Idx)
265 return None;
266 // Undefined behavior if Idx is negative or >= Size.
267 if (Idx->getValue().uge(Size))
268 continue;
269 unsigned IntIdx = Idx->getValue().getZExtValue();
270 // We can extractelement from undef vector.
271 if (isa<UndefValue>(Vec))
272 continue;
273 // For correct shuffling we have to have at most 2 different vector operands
274 // in all extractelement instructions.
275 if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276 return None;
277 if (CommonShuffleMode == Permute)
278 continue;
279 // If the extract index is not the same as the operation number, it is a
280 // permutation.
281 if (IntIdx != I) {
282 CommonShuffleMode = Permute;
283 continue;
284 }
285 // Check the shuffle mode for the current operation.
286 if (!Vec1)
287 Vec1 = Vec;
288 else if (Vec != Vec1)
289 Vec2 = Vec;
290 // Example: shufflevector A, B, <0,5,2,7>
291 // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292 // I is even and IntIdx for B == I - FirstAlternate shuffle.
293 // Example: shufflevector A, B, <4,1,6,3>
294 // I is even and IntIdx for A == I - SecondAlternate shuffle.
295 // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296 const bool IIsEven = I & 1;
297 const bool CurrVecIsA = Vec == Vec1;
298 const bool IIsOdd = !IIsEven;
299 const bool CurrVecIsB = !CurrVecIsA;
300 ShuffleMode CurrentShuffleMode =
301 ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302 : SecondAlternate;
303 // Common mode is not set or the same as the shuffle mode of the current
304 // operation - alternate.
305 if (CommonShuffleMode == Unknown)
306 CommonShuffleMode = CurrentShuffleMode;
307 // Common shuffle mode is not the same as the shuffle mode of the current
308 // operation - permutation.
309 if (CommonShuffleMode != CurrentShuffleMode)
310 CommonShuffleMode = Permute;
311 }
312 // If we're not crossing lanes in different vectors, consider it as blending.
313 if ((CommonShuffleMode == FirstAlternate ||
314 CommonShuffleMode == SecondAlternate) &&
315 Vec2)
316 return TargetTransformInfo::SK_Alternate;
317 // If Vec2 was never used, we have a permutation of a single vector, otherwise
318 // we have permutation of 2 vectors.
319 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
320 : TargetTransformInfo::SK_PermuteSingleSrc;
321}
322
323///\returns Opcode that can be clubbed with \p Op to create an alternate
324/// sequence which can later be merged as a ShuffleVector instruction.
325static unsigned getAltOpcode(unsigned Op) {
326 switch (Op) {
327 case Instruction::FAdd:
328 return Instruction::FSub;
329 case Instruction::FSub:
330 return Instruction::FAdd;
331 case Instruction::Add:
332 return Instruction::Sub;
333 case Instruction::Sub:
334 return Instruction::Add;
335 default:
336 return 0;
337 }
338}
339
340static bool isOdd(unsigned Value) {
341 return Value & 1;
342}
343
344static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345 unsigned CheckedOpcode) {
346 return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347}
348
349/// Chooses the correct key for scheduling data. If \p Op has the same (or
350/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
351/// OpValue.
352static Value *isOneOf(Value *OpValue, Value *Op) {
353 auto *I = dyn_cast<Instruction>(Op);
354 if (!I)
355 return OpValue;
356 auto *OpInst = cast<Instruction>(OpValue);
357 unsigned OpInstOpcode = OpInst->getOpcode();
358 unsigned IOpcode = I->getOpcode();
359 if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
360 return Op;
361 return OpValue;
362}
363
364namespace {
365
366/// Contains data for the instructions going to be vectorized.
367struct RawInstructionsData {
368 /// Main Opcode of the instructions going to be vectorized.
369 unsigned Opcode = 0;
370
371 /// The list of instructions have some instructions with alternate opcodes.
372 bool HasAltOpcodes = false;
373};
374
375} // end anonymous namespace
376
377/// Checks the list of the vectorized instructions \p VL and returns info about
378/// this list.
379static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380 auto *I0 = dyn_cast<Instruction>(VL[0]);
381 if (!I0)
382 return {};
383 RawInstructionsData Res;
384 unsigned Opcode = I0->getOpcode();
385 // Walk through the list of the vectorized instructions
386 // in order to check its structure described by RawInstructionsData.
387 for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388 auto *I = dyn_cast<Instruction>(VL[Cnt]);
389 if (!I)
390 return {};
391 if (Opcode != I->getOpcode())
392 Res.HasAltOpcodes = true;
393 }
394 Res.Opcode = Opcode;
395 return Res;
396}
397
398namespace {
399
400/// Main data required for vectorization of instructions.
401struct InstructionsState {
402 /// The very first instruction in the list with the main opcode.
403 Value *OpValue = nullptr;
404
405 /// The main opcode for the list of instructions.
406 unsigned Opcode = 0;
407
408 /// Some of the instructions in the list have alternate opcodes.
409 bool IsAltShuffle = false;
410
411 InstructionsState() = default;
412 InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413 : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
414};
415
416} // end anonymous namespace
417
418/// \returns analysis of the Instructions in \p VL described in
419/// InstructionsState, the Opcode that we suppose the whole list
420/// could be vectorized even if its structure is diverse.
421static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422 auto Res = getMainOpcode(VL);
423 unsigned Opcode = Res.Opcode;
424 if (!Res.HasAltOpcodes)
425 return InstructionsState(VL[0], Opcode, false);
426 auto *OpInst = cast<Instruction>(VL[0]);
427 unsigned AltOpcode = getAltOpcode(Opcode);
428 // Examine each element in the list instructions VL to determine
429 // if some operations there could be considered as an alternative
430 // (for example as subtraction relates to addition operation).
431 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432 auto *I = cast<Instruction>(VL[Cnt]);
433 unsigned InstOpcode = I->getOpcode();
434 if ((Res.HasAltOpcodes &&
435 InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436 (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437 return InstructionsState(OpInst, 0, false);
438 }
439 }
440 return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
441}
442
443/// \returns true if all of the values in \p VL have the same type or false
444/// otherwise.
445static bool allSameType(ArrayRef<Value *> VL) {
446 Type *Ty = VL[0]->getType();
447 for (int i = 1, e = VL.size(); i < e; i++)
448 if (VL[i]->getType() != Ty)
449 return false;
450
451 return true;
452}
453
454/// \returns True if Extract{Value,Element} instruction extracts element Idx.
455static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
456 assert(Opcode == Instruction::ExtractElement ||(static_cast <bool> (Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) ? void (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 457, __extension__ __PRETTY_FUNCTION__))
457 Opcode == Instruction::ExtractValue)(static_cast <bool> (Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) ? void (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 457, __extension__ __PRETTY_FUNCTION__))
;
458 if (Opcode == Instruction::ExtractElement) {
459 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
460 return CI && CI->getZExtValue() == Idx;
461 } else {
462 ExtractValueInst *EI = cast<ExtractValueInst>(E);
463 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
464 }
465}
466
467/// \returns True if in-tree use also needs extract. This refers to
468/// possible scalar operand in vectorized instruction.
469static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
470 TargetLibraryInfo *TLI) {
471 unsigned Opcode = UserInst->getOpcode();
472 switch (Opcode) {
473 case Instruction::Load: {
474 LoadInst *LI = cast<LoadInst>(UserInst);
475 return (LI->getPointerOperand() == Scalar);
476 }
477 case Instruction::Store: {
478 StoreInst *SI = cast<StoreInst>(UserInst);
479 return (SI->getPointerOperand() == Scalar);
480 }
481 case Instruction::Call: {
482 CallInst *CI = cast<CallInst>(UserInst);
483 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
484 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
485 return (CI->getArgOperand(1) == Scalar);
486 }
487 LLVM_FALLTHROUGH[[clang::fallthrough]];
488 }
489 default:
490 return false;
491 }
492}
493
494/// \returns the AA location that is being access by the instruction.
495static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
496 if (StoreInst *SI = dyn_cast<StoreInst>(I))
497 return MemoryLocation::get(SI);
498 if (LoadInst *LI = dyn_cast<LoadInst>(I))
499 return MemoryLocation::get(LI);
500 return MemoryLocation();
501}
502
503/// \returns True if the instruction is not a volatile or atomic load/store.
504static bool isSimple(Instruction *I) {
505 if (LoadInst *LI = dyn_cast<LoadInst>(I))
506 return LI->isSimple();
507 if (StoreInst *SI = dyn_cast<StoreInst>(I))
508 return SI->isSimple();
509 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
510 return !MI->isVolatile();
511 return true;
512}
513
514namespace llvm {
515
516namespace slpvectorizer {
517
518/// Bottom Up SLP Vectorizer.
519class BoUpSLP {
520public:
521 using ValueList = SmallVector<Value *, 8>;
522 using InstrList = SmallVector<Instruction *, 16>;
523 using ValueSet = SmallPtrSet<Value *, 16>;
524 using StoreList = SmallVector<StoreInst *, 8>;
525 using ExtraValueToDebugLocsMap =
526 MapVector<Value *, SmallVector<Instruction *, 2>>;
527
528 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
529 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
530 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
531 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
532 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
533 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
534 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
535 // Use the vector register size specified by the target unless overridden
536 // by a command-line option.
537 // TODO: It would be better to limit the vectorization factor based on
538 // data type rather than just register size. For example, x86 AVX has
539 // 256-bit registers, but it does not support integer operations
540 // at that width (that requires AVX2).
541 if (MaxVectorRegSizeOption.getNumOccurrences())
542 MaxVecRegSize = MaxVectorRegSizeOption;
543 else
544 MaxVecRegSize = TTI->getRegisterBitWidth(true);
545
546 if (MinVectorRegSizeOption.getNumOccurrences())
547 MinVecRegSize = MinVectorRegSizeOption;
548 else
549 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
550 }
551
552 /// \brief Vectorize the tree that starts with the elements in \p VL.
553 /// Returns the vectorized root.
554 Value *vectorizeTree();
555
556 /// Vectorize the tree but with the list of externally used values \p
557 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
558 /// generated extractvalue instructions.
559 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
560
561 /// \returns the cost incurred by unwanted spills and fills, caused by
562 /// holding live values over call sites.
563 int getSpillCost();
564
565 /// \returns the vectorization cost of the subtree that starts at \p VL.
566 /// A negative number means that this is profitable.
567 int getTreeCost();
568
569 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
570 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
571 void buildTree(ArrayRef<Value *> Roots,
572 ArrayRef<Value *> UserIgnoreLst = None);
573
574 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
576 /// into account (anf updating it, if required) list of externally used
577 /// values stored in \p ExternallyUsedValues.
578 void buildTree(ArrayRef<Value *> Roots,
579 ExtraValueToDebugLocsMap &ExternallyUsedValues,
580 ArrayRef<Value *> UserIgnoreLst = None);
581
582 /// Clear the internal data structures that are created by 'buildTree'.
583 void deleteTree() {
584 VectorizableTree.clear();
585 ScalarToTreeEntry.clear();
586 MustGather.clear();
587 ExternalUses.clear();
588 NumOpsWantToKeepOrder.clear();
589 for (auto &Iter : BlocksSchedules) {
590 BlockScheduling *BS = Iter.second.get();
591 BS->clear();
592 }
593 MinBWs.clear();
594 }
595
596 unsigned getTreeSize() const { return VectorizableTree.size(); }
597
598 /// \brief Perform LICM and CSE on the newly generated gather sequences.
599 void optimizeGatherSequence();
600
601 /// \returns true if it is beneficial to reverse the vector order.
602 bool shouldReorder() const {
603 return std::accumulate(
604 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(), 0,
605 [](int Val1,
606 const decltype(NumOpsWantToKeepOrder)::value_type &Val2) {
607 return Val1 + (Val2.second < 0 ? 1 : -1);
608 }) > 0;
609 }
610
611 /// \return The vector element size in bits to use when vectorizing the
612 /// expression tree ending at \p V. If V is a store, the size is the width of
613 /// the stored value. Otherwise, the size is the width of the largest loaded
614 /// value reaching V. This method is used by the vectorizer to calculate
615 /// vectorization factors.
616 unsigned getVectorElementSize(Value *V);
617
618 /// Compute the minimum type sizes required to represent the entries in a
619 /// vectorizable tree.
620 void computeMinimumValueSizes();
621
622 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
623 unsigned getMaxVecRegSize() const {
624 return MaxVecRegSize;
625 }
626
627 // \returns minimum vector register size as set by cl::opt.
628 unsigned getMinVecRegSize() const {
629 return MinVecRegSize;
630 }
631
632 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
633 ///
634 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
635 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
636
637 /// \returns True if the VectorizableTree is both tiny and not fully
638 /// vectorizable. We do not vectorize such trees.
639 bool isTreeTinyAndNotFullyVectorizable();
640
641 OptimizationRemarkEmitter *getORE() { return ORE; }
642
643private:
644 struct TreeEntry;
645
646 /// Checks if all users of \p I are the part of the vectorization tree.
647 bool areAllUsersVectorized(Instruction *I) const;
648
649 /// \returns the cost of the vectorizable entry.
650 int getEntryCost(TreeEntry *E);
651
652 /// This is the recursive part of buildTree.
653 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
654
655 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
656 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
657 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const;
658
659 /// Vectorize a single entry in the tree.
660 Value *vectorizeTree(TreeEntry *E);
661
662 /// Vectorize a single entry in the tree, starting in \p VL.
663 Value *vectorizeTree(ArrayRef<Value *> VL);
664
665 /// \returns the scalarization cost for this type. Scalarization in this
666 /// context means the creation of vectors from a group of scalars.
667 int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
668
669 /// \returns the scalarization cost for this list of values. Assuming that
670 /// this subtree gets vectorized, we may need to extract the values from the
671 /// roots. This method calculates the cost of extracting the values.
672 int getGatherCost(ArrayRef<Value *> VL);
673
674 /// \brief Set the Builder insert point to one after the last instruction in
675 /// the bundle
676 void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
677
678 /// \returns a vector from a collection of scalars in \p VL.
679 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
680
681 /// \returns whether the VectorizableTree is fully vectorizable and will
682 /// be beneficial even the tree height is tiny.
683 bool isFullyVectorizableTinyTree();
684
685 /// \reorder commutative operands in alt shuffle if they result in
686 /// vectorized code.
687 void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
688 SmallVectorImpl<Value *> &Left,
689 SmallVectorImpl<Value *> &Right);
690
691 /// \reorder commutative operands to get better probability of
692 /// generating vectorized code.
693 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
694 SmallVectorImpl<Value *> &Left,
695 SmallVectorImpl<Value *> &Right);
696 struct TreeEntry {
697 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
698
699 /// \returns true if the scalars in VL are equal to this entry.
700 bool isSame(ArrayRef<Value *> VL) const {
701 if (VL.size() == Scalars.size())
702 return std::equal(VL.begin(), VL.end(), Scalars.begin());
703 return VL.size() == ReuseShuffleIndices.size() &&
704 std::equal(
705 VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
706 [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
707 }
708
709 /// A vector of scalars.
710 ValueList Scalars;
711
712 /// The Scalars are vectorized into this value. It is initialized to Null.
713 Value *VectorizedValue = nullptr;
714
715 /// Do we need to gather this sequence ?
716 bool NeedToGather = false;
717
718 /// Does this sequence require some shuffling?
719 SmallVector<unsigned, 4> ReuseShuffleIndices;
720
721 /// Points back to the VectorizableTree.
722 ///
723 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
724 /// to be a pointer and needs to be able to initialize the child iterator.
725 /// Thus we need a reference back to the container to translate the indices
726 /// to entries.
727 std::vector<TreeEntry> &Container;
728
729 /// The TreeEntry index containing the user of this entry. We can actually
730 /// have multiple users so the data structure is not truly a tree.
731 SmallVector<int, 1> UserTreeIndices;
732 };
733
734 /// Create a new VectorizableTree entry.
735 void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
736 ArrayRef<unsigned> ReuseShuffleIndices = None) {
737 VectorizableTree.emplace_back(VectorizableTree);
738 int idx = VectorizableTree.size() - 1;
739 TreeEntry *Last = &VectorizableTree[idx];
740 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
741 Last->NeedToGather = !Vectorized;
742 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
743 ReuseShuffleIndices.end());
744 if (Vectorized) {
745 for (int i = 0, e = VL.size(); i != e; ++i) {
746 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!")(static_cast <bool> (!getTreeEntry(VL[i]) && "Scalar already in tree!"
) ? void (0) : __assert_fail ("!getTreeEntry(VL[i]) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 746, __extension__ __PRETTY_FUNCTION__))
;
747 ScalarToTreeEntry[VL[i]] = idx;
748 }
749 } else {
750 MustGather.insert(VL.begin(), VL.end());
751 }
752
753 if (UserTreeIdx >= 0)
754 Last->UserTreeIndices.push_back(UserTreeIdx);
755 UserTreeIdx = idx;
756 }
757
758 /// -- Vectorization State --
759 /// Holds all of the tree entries.
760 std::vector<TreeEntry> VectorizableTree;
761
762 TreeEntry *getTreeEntry(Value *V) {
763 auto I = ScalarToTreeEntry.find(V);
764 if (I != ScalarToTreeEntry.end())
765 return &VectorizableTree[I->second];
766 return nullptr;
767 }
768
769 /// Maps a specific scalar to its tree entry.
770 SmallDenseMap<Value*, int> ScalarToTreeEntry;
771
772 /// A list of scalars that we found that we need to keep as scalars.
773 ValueSet MustGather;
774
775 /// This POD struct describes one external user in the vectorized tree.
776 struct ExternalUser {
777 ExternalUser(Value *S, llvm::User *U, int L)
778 : Scalar(S), User(U), Lane(L) {}
779
780 // Which scalar in our function.
781 Value *Scalar;
782
783 // Which user that uses the scalar.
784 llvm::User *User;
785
786 // Which lane does the scalar belong to.
787 int Lane;
788 };
789 using UserList = SmallVector<ExternalUser, 16>;
790
791 /// Checks if two instructions may access the same memory.
792 ///
793 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
794 /// is invariant in the calling loop.
795 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
796 Instruction *Inst2) {
797 // First check if the result is already in the cache.
798 AliasCacheKey key = std::make_pair(Inst1, Inst2);
799 Optional<bool> &result = AliasCache[key];
800 if (result.hasValue()) {
801 return result.getValue();
802 }
803 MemoryLocation Loc2 = getLocation(Inst2, AA);
804 bool aliased = true;
805 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
806 // Do the alias check.
807 aliased = AA->alias(Loc1, Loc2);
808 }
809 // Store the result in the cache.
810 result = aliased;
811 return aliased;
812 }
813
814 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
815
816 /// Cache for alias results.
817 /// TODO: consider moving this to the AliasAnalysis itself.
818 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
819
820 /// Removes an instruction from its block and eventually deletes it.
821 /// It's like Instruction::eraseFromParent() except that the actual deletion
822 /// is delayed until BoUpSLP is destructed.
823 /// This is required to ensure that there are no incorrect collisions in the
824 /// AliasCache, which can happen if a new instruction is allocated at the
825 /// same address as a previously deleted instruction.
826 void eraseInstruction(Instruction *I) {
827 I->removeFromParent();
828 I->dropAllReferences();
829 DeletedInstructions.emplace_back(I);
830 }
831
832 /// Temporary store for deleted instructions. Instructions will be deleted
833 /// eventually when the BoUpSLP is destructed.
834 SmallVector<unique_value, 8> DeletedInstructions;
835
836 /// A list of values that need to extracted out of the tree.
837 /// This list holds pairs of (Internal Scalar : External User). External User
838 /// can be nullptr, it means that this Internal Scalar will be used later,
839 /// after vectorization.
840 UserList ExternalUses;
841
842 /// Values used only by @llvm.assume calls.
843 SmallPtrSet<const Value *, 32> EphValues;
844
845 /// Holds all of the instructions that we gathered.
846 SetVector<Instruction *> GatherSeq;
847
848 /// A list of blocks that we are going to CSE.
849 SetVector<BasicBlock *> CSEBlocks;
850
851 /// Contains all scheduling relevant data for an instruction.
852 /// A ScheduleData either represents a single instruction or a member of an
853 /// instruction bundle (= a group of instructions which is combined into a
854 /// vector instruction).
855 struct ScheduleData {
856 // The initial value for the dependency counters. It means that the
857 // dependencies are not calculated yet.
858 enum { InvalidDeps = -1 };
859
860 ScheduleData() = default;
861
862 void init(int BlockSchedulingRegionID, Value *OpVal) {
863 FirstInBundle = this;
864 NextInBundle = nullptr;
865 NextLoadStore = nullptr;
866 IsScheduled = false;
867 SchedulingRegionID = BlockSchedulingRegionID;
868 UnscheduledDepsInBundle = UnscheduledDeps;
869 clearDependencies();
870 OpValue = OpVal;
871 }
872
873 /// Returns true if the dependency information has been calculated.
874 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
875
876 /// Returns true for single instructions and for bundle representatives
877 /// (= the head of a bundle).
878 bool isSchedulingEntity() const { return FirstInBundle == this; }
879
880 /// Returns true if it represents an instruction bundle and not only a
881 /// single instruction.
882 bool isPartOfBundle() const {
883 return NextInBundle != nullptr || FirstInBundle != this;
884 }
885
886 /// Returns true if it is ready for scheduling, i.e. it has no more
887 /// unscheduled depending instructions/bundles.
888 bool isReady() const {
889 assert(isSchedulingEntity() &&(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 890, __extension__ __PRETTY_FUNCTION__))
890 "can't consider non-scheduling entity for ready list")(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 890, __extension__ __PRETTY_FUNCTION__))
;
891 return UnscheduledDepsInBundle == 0 && !IsScheduled;
892 }
893
894 /// Modifies the number of unscheduled dependencies, also updating it for
895 /// the whole bundle.
896 int incrementUnscheduledDeps(int Incr) {
897 UnscheduledDeps += Incr;
898 return FirstInBundle->UnscheduledDepsInBundle += Incr;
899 }
900
901 /// Sets the number of unscheduled dependencies to the number of
902 /// dependencies.
903 void resetUnscheduledDeps() {
904 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
905 }
906
907 /// Clears all dependency information.
908 void clearDependencies() {
909 Dependencies = InvalidDeps;
910 resetUnscheduledDeps();
911 MemoryDependencies.clear();
912 }
913
914 void dump(raw_ostream &os) const {
915 if (!isSchedulingEntity()) {
916 os << "/ " << *Inst;
917 } else if (NextInBundle) {
918 os << '[' << *Inst;
919 ScheduleData *SD = NextInBundle;
920 while (SD) {
921 os << ';' << *SD->Inst;
922 SD = SD->NextInBundle;
923 }
924 os << ']';
925 } else {
926 os << *Inst;
927 }
928 }
929
930 Instruction *Inst = nullptr;
931
932 /// Points to the head in an instruction bundle (and always to this for
933 /// single instructions).
934 ScheduleData *FirstInBundle = nullptr;
935
936 /// Single linked list of all instructions in a bundle. Null if it is a
937 /// single instruction.
938 ScheduleData *NextInBundle = nullptr;
939
940 /// Single linked list of all memory instructions (e.g. load, store, call)
941 /// in the block - until the end of the scheduling region.
942 ScheduleData *NextLoadStore = nullptr;
943
944 /// The dependent memory instructions.
945 /// This list is derived on demand in calculateDependencies().
946 SmallVector<ScheduleData *, 4> MemoryDependencies;
947
948 /// This ScheduleData is in the current scheduling region if this matches
949 /// the current SchedulingRegionID of BlockScheduling.
950 int SchedulingRegionID = 0;
951
952 /// Used for getting a "good" final ordering of instructions.
953 int SchedulingPriority = 0;
954
955 /// The number of dependencies. Constitutes of the number of users of the
956 /// instruction plus the number of dependent memory instructions (if any).
957 /// This value is calculated on demand.
958 /// If InvalidDeps, the number of dependencies is not calculated yet.
959 int Dependencies = InvalidDeps;
960
961 /// The number of dependencies minus the number of dependencies of scheduled
962 /// instructions. As soon as this is zero, the instruction/bundle gets ready
963 /// for scheduling.
964 /// Note that this is negative as long as Dependencies is not calculated.
965 int UnscheduledDeps = InvalidDeps;
966
967 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
968 /// single instructions.
969 int UnscheduledDepsInBundle = InvalidDeps;
970
971 /// True if this instruction is scheduled (or considered as scheduled in the
972 /// dry-run).
973 bool IsScheduled = false;
974
975 /// Opcode of the current instruction in the schedule data.
976 Value *OpValue = nullptr;
977 };
978
979#ifndef NDEBUG
980 friend inline raw_ostream &operator<<(raw_ostream &os,
981 const BoUpSLP::ScheduleData &SD) {
982 SD.dump(os);
983 return os;
984 }
985#endif
986
987 friend struct GraphTraits<BoUpSLP *>;
988 friend struct DOTGraphTraits<BoUpSLP *>;
989
990 /// Contains all scheduling data for a basic block.
991 struct BlockScheduling {
992 BlockScheduling(BasicBlock *BB)
993 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
994
995 void clear() {
996 ReadyInsts.clear();
997 ScheduleStart = nullptr;
998 ScheduleEnd = nullptr;
999 FirstLoadStoreInRegion = nullptr;
1000 LastLoadStoreInRegion = nullptr;
1001
1002 // Reduce the maximum schedule region size by the size of the
1003 // previous scheduling run.
1004 ScheduleRegionSizeLimit -= ScheduleRegionSize;
1005 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1006 ScheduleRegionSizeLimit = MinScheduleRegionSize;
1007 ScheduleRegionSize = 0;
1008
1009 // Make a new scheduling region, i.e. all existing ScheduleData is not
1010 // in the new region yet.
1011 ++SchedulingRegionID;
1012 }
1013
1014 ScheduleData *getScheduleData(Value *V) {
1015 ScheduleData *SD = ScheduleDataMap[V];
1016 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1017 return SD;
1018 return nullptr;
1019 }
1020
1021 ScheduleData *getScheduleData(Value *V, Value *Key) {
1022 if (V == Key)
1023 return getScheduleData(V);
1024 auto I = ExtraScheduleDataMap.find(V);
1025 if (I != ExtraScheduleDataMap.end()) {
1026 ScheduleData *SD = I->second[Key];
1027 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1028 return SD;
1029 }
1030 return nullptr;
1031 }
1032
1033 bool isInSchedulingRegion(ScheduleData *SD) {
1034 return SD->SchedulingRegionID == SchedulingRegionID;
1035 }
1036
1037 /// Marks an instruction as scheduled and puts all dependent ready
1038 /// instructions into the ready-list.
1039 template <typename ReadyListType>
1040 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1041 SD->IsScheduled = true;
1042 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
1043
1044 ScheduleData *BundleMember = SD;
1045 while (BundleMember) {
1046 if (BundleMember->Inst != BundleMember->OpValue) {
1047 BundleMember = BundleMember->NextInBundle;
1048 continue;
1049 }
1050 // Handle the def-use chain dependencies.
1051 for (Use &U : BundleMember->Inst->operands()) {
1052 auto *I = dyn_cast<Instruction>(U.get());
1053 if (!I)
1054 continue;
1055 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1056 if (OpDef && OpDef->hasValidDependencies() &&
1057 OpDef->incrementUnscheduledDeps(-1) == 0) {
1058 // There are no more unscheduled dependencies after
1059 // decrementing, so we can put the dependent instruction
1060 // into the ready list.
1061 ScheduleData *DepBundle = OpDef->FirstInBundle;
1062 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1063, __extension__ __PRETTY_FUNCTION__))
1063 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1063, __extension__ __PRETTY_FUNCTION__))
;
1064 ReadyList.insert(DepBundle);
1065 DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
1066 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
1067 }
1068 });
1069 }
1070 // Handle the memory dependencies.
1071 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1072 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1073 // There are no more unscheduled dependencies after decrementing,
1074 // so we can put the dependent instruction into the ready list.
1075 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1076 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1077, __extension__ __PRETTY_FUNCTION__))
1077 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1077, __extension__ __PRETTY_FUNCTION__))
;
1078 ReadyList.insert(DepBundle);
1079 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundledo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
1080 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
1081 }
1082 }
1083 BundleMember = BundleMember->NextInBundle;
1084 }
1085 }
1086
1087 void doForAllOpcodes(Value *V,
1088 function_ref<void(ScheduleData *SD)> Action) {
1089 if (ScheduleData *SD = getScheduleData(V))
1090 Action(SD);
1091 auto I = ExtraScheduleDataMap.find(V);
1092 if (I != ExtraScheduleDataMap.end())
1093 for (auto &P : I->second)
1094 if (P.second->SchedulingRegionID == SchedulingRegionID)
1095 Action(P.second);
1096 }
1097
1098 /// Put all instructions into the ReadyList which are ready for scheduling.
1099 template <typename ReadyListType>
1100 void initialFillReadyList(ReadyListType &ReadyList) {
1101 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1102 doForAllOpcodes(I, [&](ScheduleData *SD) {
1103 if (SD->isSchedulingEntity() && SD->isReady()) {
1104 ReadyList.insert(SD);
1105 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
1106 }
1107 });
1108 }
1109 }
1110
1111 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1112 /// cyclic dependencies. This is only a dry-run, no instructions are
1113 /// actually moved at this stage.
1114 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1115
1116 /// Un-bundles a group of instructions.
1117 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1118
1119 /// Allocates schedule data chunk.
1120 ScheduleData *allocateScheduleDataChunks();
1121
1122 /// Extends the scheduling region so that V is inside the region.
1123 /// \returns true if the region size is within the limit.
1124 bool extendSchedulingRegion(Value *V, Value *OpValue);
1125
1126 /// Initialize the ScheduleData structures for new instructions in the
1127 /// scheduling region.
1128 void initScheduleData(Instruction *FromI, Instruction *ToI,
1129 ScheduleData *PrevLoadStore,
1130 ScheduleData *NextLoadStore);
1131
1132 /// Updates the dependency information of a bundle and of all instructions/
1133 /// bundles which depend on the original bundle.
1134 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1135 BoUpSLP *SLP);
1136
1137 /// Sets all instruction in the scheduling region to un-scheduled.
1138 void resetSchedule();
1139
1140 BasicBlock *BB;
1141
1142 /// Simple memory allocation for ScheduleData.
1143 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1144
1145 /// The size of a ScheduleData array in ScheduleDataChunks.
1146 int ChunkSize;
1147
1148 /// The allocator position in the current chunk, which is the last entry
1149 /// of ScheduleDataChunks.
1150 int ChunkPos;
1151
1152 /// Attaches ScheduleData to Instruction.
1153 /// Note that the mapping survives during all vectorization iterations, i.e.
1154 /// ScheduleData structures are recycled.
1155 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1156
1157 /// Attaches ScheduleData to Instruction with the leading key.
1158 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1159 ExtraScheduleDataMap;
1160
1161 struct ReadyList : SmallVector<ScheduleData *, 8> {
1162 void insert(ScheduleData *SD) { push_back(SD); }
1163 };
1164
1165 /// The ready-list for scheduling (only used for the dry-run).
1166 ReadyList ReadyInsts;
1167
1168 /// The first instruction of the scheduling region.
1169 Instruction *ScheduleStart = nullptr;
1170
1171 /// The first instruction _after_ the scheduling region.
1172 Instruction *ScheduleEnd = nullptr;
1173
1174 /// The first memory accessing instruction in the scheduling region
1175 /// (can be null).
1176 ScheduleData *FirstLoadStoreInRegion = nullptr;
1177
1178 /// The last memory accessing instruction in the scheduling region
1179 /// (can be null).
1180 ScheduleData *LastLoadStoreInRegion = nullptr;
1181
1182 /// The current size of the scheduling region.
1183 int ScheduleRegionSize = 0;
1184
1185 /// The maximum size allowed for the scheduling region.
1186 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1187
1188 /// The ID of the scheduling region. For a new vectorization iteration this
1189 /// is incremented which "removes" all ScheduleData from the region.
1190 // Make sure that the initial SchedulingRegionID is greater than the
1191 // initial SchedulingRegionID in ScheduleData (which is 0).
1192 int SchedulingRegionID = 1;
1193 };
1194
1195 /// Attaches the BlockScheduling structures to basic blocks.
1196 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1197
1198 /// Performs the "real" scheduling. Done before vectorization is actually
1199 /// performed in a basic block.
1200 void scheduleBlock(BlockScheduling *BS);
1201
1202 /// List of users to ignore during scheduling and that don't need extracting.
1203 ArrayRef<Value *> UserIgnoreList;
1204
1205 /// Number of operation bundles that contain consecutive operations - number
1206 /// of operation bundles that contain consecutive operations in reversed
1207 /// order.
1208 DenseMap<unsigned, int> NumOpsWantToKeepOrder;
1209
1210 // Analysis and block reference.
1211 Function *F;
1212 ScalarEvolution *SE;
1213 TargetTransformInfo *TTI;
1214 TargetLibraryInfo *TLI;
1215 AliasAnalysis *AA;
1216 LoopInfo *LI;
1217 DominatorTree *DT;
1218 AssumptionCache *AC;
1219 DemandedBits *DB;
1220 const DataLayout *DL;
1221 OptimizationRemarkEmitter *ORE;
1222
1223 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1224 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1225
1226 /// Instruction builder to construct the vectorized tree.
1227 IRBuilder<> Builder;
1228
1229 /// A map of scalar integer values to the smallest bit width with which they
1230 /// can legally be represented. The values map to (width, signed) pairs,
1231 /// where "width" indicates the minimum bit width and "signed" is True if the
1232 /// value must be signed-extended, rather than zero-extended, back to its
1233 /// original width.
1234 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1235};
1236
1237} // end namespace slpvectorizer
1238
1239template <> struct GraphTraits<BoUpSLP *> {
1240 using TreeEntry = BoUpSLP::TreeEntry;
1241
1242 /// NodeRef has to be a pointer per the GraphWriter.
1243 using NodeRef = TreeEntry *;
1244
1245 /// \brief Add the VectorizableTree to the index iterator to be able to return
1246 /// TreeEntry pointers.
1247 struct ChildIteratorType
1248 : public iterator_adaptor_base<ChildIteratorType,
1249 SmallVector<int, 1>::iterator> {
1250 std::vector<TreeEntry> &VectorizableTree;
1251
1252 ChildIteratorType(SmallVector<int, 1>::iterator W,
1253 std::vector<TreeEntry> &VT)
1254 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1255
1256 NodeRef operator*() { return &VectorizableTree[*I]; }
1257 };
1258
1259 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1260
1261 static ChildIteratorType child_begin(NodeRef N) {
1262 return {N->UserTreeIndices.begin(), N->Container};
1263 }
1264
1265 static ChildIteratorType child_end(NodeRef N) {
1266 return {N->UserTreeIndices.end(), N->Container};
1267 }
1268
1269 /// For the node iterator we just need to turn the TreeEntry iterator into a
1270 /// TreeEntry* iterator so that it dereferences to NodeRef.
1271 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1272
1273 static nodes_iterator nodes_begin(BoUpSLP *R) {
1274 return nodes_iterator(R->VectorizableTree.begin());
1275 }
1276
1277 static nodes_iterator nodes_end(BoUpSLP *R) {
1278 return nodes_iterator(R->VectorizableTree.end());
1279 }
1280
1281 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1282};
1283
1284template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1285 using TreeEntry = BoUpSLP::TreeEntry;
1286
1287 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1288
1289 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1290 std::string Str;
1291 raw_string_ostream OS(Str);
1292 if (isSplat(Entry->Scalars)) {
1293 OS << "<splat> " << *Entry->Scalars[0];
1294 return Str;
1295 }
1296 for (auto V : Entry->Scalars) {
1297 OS << *V;
1298 if (std::any_of(
1299 R->ExternalUses.begin(), R->ExternalUses.end(),
1300 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1301 OS << " <extract>";
1302 OS << "\n";
1303 }
1304 return Str;
1305 }
1306
1307 static std::string getNodeAttributes(const TreeEntry *Entry,
1308 const BoUpSLP *) {
1309 if (Entry->NeedToGather)
1310 return "color=red";
1311 return "";
1312 }
1313};
1314
1315} // end namespace llvm
1316
1317void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1318 ArrayRef<Value *> UserIgnoreLst) {
1319 ExtraValueToDebugLocsMap ExternallyUsedValues;
1320 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1321}
1322
1323void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1324 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1325 ArrayRef<Value *> UserIgnoreLst) {
1326 deleteTree();
1327 UserIgnoreList = UserIgnoreLst;
1328 if (!allSameType(Roots))
1329 return;
1330 buildTree_rec(Roots, 0, -1);
1331
1332 // Collect the values that we need to extract from the tree.
1333 for (TreeEntry &EIdx : VectorizableTree) {
1334 TreeEntry *Entry = &EIdx;
1335
1336 // No need to handle users of gathered values.
1337 if (Entry->NeedToGather)
1338 continue;
1339
1340 // For each lane:
1341 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1342 Value *Scalar = Entry->Scalars[Lane];
1343 int FoundLane = Lane;
1344 if (!Entry->ReuseShuffleIndices.empty()) {
1345 FoundLane =
1346 std::distance(Entry->ReuseShuffleIndices.begin(),
1347 llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1348 }
1349
1350 // Check if the scalar is externally used as an extra arg.
1351 auto ExtI = ExternallyUsedValues.find(Scalar);
1352 if (ExtI != ExternallyUsedValues.end()) {
1353 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
1354 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
1355 ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1356 }
1357 for (User *U : Scalar->users()) {
1358 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
1359
1360 Instruction *UserInst = dyn_cast<Instruction>(U);
1361 if (!UserInst)
1362 continue;
1363
1364 // Skip in-tree scalars that become vectors
1365 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1366 Value *UseScalar = UseEntry->Scalars[0];
1367 // Some in-tree scalars will remain as scalar in vectorized
1368 // instructions. If that is the case, the one in Lane 0 will
1369 // be used.
1370 if (UseScalar != U ||
1371 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1372 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1373 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
1374 assert(!UseEntry->NeedToGather && "Bad state")(static_cast <bool> (!UseEntry->NeedToGather &&
"Bad state") ? void (0) : __assert_fail ("!UseEntry->NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1374, __extension__ __PRETTY_FUNCTION__))
;
1375 continue;
1376 }
1377 }
1378
1379 // Ignore users in the user ignore list.
1380 if (is_contained(UserIgnoreList, UserInst))
1381 continue;
1382
1383 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1384 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
1385 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1386 }
1387 }
1388 }
1389}
1390
1391void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1392 int UserTreeIdx) {
1393 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(static_cast <bool> ((allConstant(VL) || allSameType(VL
)) && "Invalid types!") ? void (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1393, __extension__ __PRETTY_FUNCTION__))
;
1394
1395 InstructionsState S = getSameOpcode(VL);
1396 if (Depth == RecursionMaxDepth) {
1397 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
1398 newTreeEntry(VL, false, UserTreeIdx);
1399 return;
1400 }
1401
1402 // Don't handle vectors.
1403 if (S.OpValue->getType()->isVectorTy()) {
1404 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
1405 newTreeEntry(VL, false, UserTreeIdx);
1406 return;
1407 }
1408
1409 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1410 if (SI->getValueOperand()->getType()->isVectorTy()) {
1411 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1412 newTreeEntry(VL, false, UserTreeIdx);
1413 return;
1414 }
1415
1416 // If all of the operands are identical or constant we have a simple solution.
1417 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1418 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1419 newTreeEntry(VL, false, UserTreeIdx);
1420 return;
1421 }
1422
1423 // We now know that this is a vector of instructions of the same type from
1424 // the same block.
1425
1426 // Don't vectorize ephemeral values.
1427 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1428 if (EphValues.count(VL[i])) {
1429 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1430 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1431 newTreeEntry(VL, false, UserTreeIdx);
1432 return;
1433 }
1434 }
1435
1436 // Check if this is a duplicate of another entry.
1437 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1438 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*S.OpValue << ".\n"; } } while (false)
;
1439 if (!E->isSame(VL)) {
1440 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1441 newTreeEntry(VL, false, UserTreeIdx);
1442 return;
1443 }
1444 // Record the reuse of the tree node. FIXME, currently this is only used to
1445 // properly draw the graph rather than for the actual vectorization.
1446 E->UserTreeIndices.push_back(UserTreeIdx);
1447 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
;
1448 return;
1449 }
1450
1451 // Check that none of the instructions in the bundle are already in the tree.
1452 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1453 auto *I = dyn_cast<Instruction>(VL[i]);
1454 if (!I)
1455 continue;
1456 if (getTreeEntry(I)) {
1457 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1458 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1459 newTreeEntry(VL, false, UserTreeIdx);
1460 return;
1461 }
1462 }
1463
1464 // If any of the scalars is marked as a value that needs to stay scalar, then
1465 // we need to gather the scalars.
1466 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1467 if (MustGather.count(VL[i])) {
1468 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1469 newTreeEntry(VL, false, UserTreeIdx);
1470 return;
1471 }
1472 }
1473
1474 // Check that all of the users of the scalars that we want to vectorize are
1475 // schedulable.
1476 auto *VL0 = cast<Instruction>(S.OpValue);
1477 BasicBlock *BB = VL0->getParent();
1478
1479 if (!DT->isReachableFromEntry(BB)) {
1480 // Don't go into unreachable blocks. They may contain instructions with
1481 // dependency cycles which confuse the final scheduling.
1482 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1483 newTreeEntry(VL, false, UserTreeIdx);
1484 return;
1485 }
1486
1487 // Check that every instruction appears once in this bundle.
1488 SmallVector<unsigned, 4> ReuseShuffleIndicies;
1489 SmallVector<Value *, 4> UniqueValues;
1490 DenseMap<Value *, unsigned> UniquePositions;
1491 for (Value *V : VL) {
1492 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1493 ReuseShuffleIndicies.emplace_back(Res.first->second);
1494 if (Res.second)
1495 UniqueValues.emplace_back(V);
1496 }
1497 if (UniqueValues.size() == VL.size()) {
1498 ReuseShuffleIndicies.clear();
1499 } else {
1500 DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Shuffle for reused scalars.\n"
; } } while (false)
;
1501 if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1502 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1503 newTreeEntry(VL, false, UserTreeIdx);
1504 return;
1505 }
1506 VL = UniqueValues;
1507 }
1508
1509 auto &BSRef = BlocksSchedules[BB];
1510 if (!BSRef)
1511 BSRef = llvm::make_unique<BlockScheduling>(BB);
1512
1513 BlockScheduling &BS = *BSRef.get();
1514
1515 if (!BS.tryScheduleBundle(VL, this, VL0)) {
1516 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1517 assert((!BS.getScheduleData(VL0) ||(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1519, __extension__ __PRETTY_FUNCTION__))
1518 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1519, __extension__ __PRETTY_FUNCTION__))
1519 "tryScheduleBundle should cancelScheduling on failure")(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1519, __extension__ __PRETTY_FUNCTION__))
;
1520 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1521 return;
1522 }
1523 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1524
1525 unsigned ShuffleOrOp = S.IsAltShuffle ?
1526 (unsigned) Instruction::ShuffleVector : S.Opcode;
1527 switch (ShuffleOrOp) {
1528 case Instruction::PHI: {
1529 PHINode *PH = dyn_cast<PHINode>(VL0);
1530
1531 // Check for terminator values (e.g. invoke).
1532 for (unsigned j = 0; j < VL.size(); ++j)
1533 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1534 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1535 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1536 if (Term) {
1537 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false)
;
1538 BS.cancelScheduling(VL, VL0);
1539 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1540 return;
1541 }
1542 }
1543
1544 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1545 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1546
1547 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1548 ValueList Operands;
1549 // Prepare the operand vector.
1550 for (Value *j : VL)
1551 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1552 PH->getIncomingBlock(i)));
1553
1554 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1555 }
1556 return;
1557 }
1558 case Instruction::ExtractValue:
1559 case Instruction::ExtractElement: {
1560 bool Reuse = canReuseExtract(VL, VL0);
1561 if (Reuse) {
1562 DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing or shuffling extract sequence.\n"
; } } while (false)
;
1563 ++NumOpsWantToKeepOrder[S.Opcode];
1564 } else {
1565 SmallVector<Value *, 4> ReverseVL(VL.rbegin(), VL.rend());
1566 if (canReuseExtract(ReverseVL, VL0))
1567 --NumOpsWantToKeepOrder[S.Opcode];
1568 BS.cancelScheduling(VL, VL0);
1569 }
1570 newTreeEntry(VL, Reuse, UserTreeIdx, ReuseShuffleIndicies);
1571 return;
1572 }
1573 case Instruction::Load: {
1574 // Check that a vectorized load would load the same memory as a scalar
1575 // load. For example, we don't want to vectorize loads that are smaller
1576 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1577 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1578 // from such a struct, we read/write packed bits disagreeing with the
1579 // unvectorized version.
1580 Type *ScalarTy = VL0->getType();
1581
1582 if (DL->getTypeSizeInBits(ScalarTy) !=
1583 DL->getTypeAllocSizeInBits(ScalarTy)) {
1584 BS.cancelScheduling(VL, VL0);
1585 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1586 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1587 return;
1588 }
1589
1590 // Make sure all loads in the bundle are simple - we can't vectorize
1591 // atomic or volatile loads.
1592 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1593 LoadInst *L = cast<LoadInst>(VL[i]);
1594 if (!L->isSimple()) {
1595 BS.cancelScheduling(VL, VL0);
1596 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1597 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1598 return;
1599 }
1600 }
1601
1602 // Check if the loads are consecutive, reversed, or neither.
1603 // TODO: What we really want is to sort the loads, but for now, check
1604 // the two likely directions.
1605 bool Consecutive = true;
1606 bool ReverseConsecutive = true;
1607 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1608 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1609 Consecutive = false;
1610 break;
1611 } else {
1612 ReverseConsecutive = false;
1613 }
1614 }
1615
1616 if (Consecutive) {
1617 ++NumOpsWantToKeepOrder[S.Opcode];
1618 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1619 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1620 return;
1621 }
1622
1623 // If none of the load pairs were consecutive when checked in order,
1624 // check the reverse order.
1625 if (ReverseConsecutive)
1626 for (unsigned i = VL.size() - 1; i > 0; --i)
1627 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1628 ReverseConsecutive = false;
1629 break;
1630 }
1631
1632 BS.cancelScheduling(VL, VL0);
1633 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1634
1635 if (ReverseConsecutive) {
1636 --NumOpsWantToKeepOrder[S.Opcode];
1637 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering reversed loads.\n"
; } } while (false)
;
1638 } else {
1639 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1640 }
1641 return;
1642 }
1643 case Instruction::ZExt:
1644 case Instruction::SExt:
1645 case Instruction::FPToUI:
1646 case Instruction::FPToSI:
1647 case Instruction::FPExt:
1648 case Instruction::PtrToInt:
1649 case Instruction::IntToPtr:
1650 case Instruction::SIToFP:
1651 case Instruction::UIToFP:
1652 case Instruction::Trunc:
1653 case Instruction::FPTrunc:
1654 case Instruction::BitCast: {
1655 Type *SrcTy = VL0->getOperand(0)->getType();
1656 for (unsigned i = 0; i < VL.size(); ++i) {
1657 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1658 if (Ty != SrcTy || !isValidElementType(Ty)) {
1659 BS.cancelScheduling(VL, VL0);
1660 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1661 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1662 return;
1663 }
1664 }
1665 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1666 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1667
1668 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1669 ValueList Operands;
1670 // Prepare the operand vector.
1671 for (Value *j : VL)
1672 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1673
1674 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1675 }
1676 return;
1677 }
1678 case Instruction::ICmp:
1679 case Instruction::FCmp: {
1680 // Check that all of the compares have the same predicate.
1681 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1682 Type *ComparedTy = VL0->getOperand(0)->getType();
1683 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1684 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1685 if (Cmp->getPredicate() != P0 ||
1686 Cmp->getOperand(0)->getType() != ComparedTy) {
1687 BS.cancelScheduling(VL, VL0);
1688 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1689 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1690 return;
1691 }
1692 }
1693
1694 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1695 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1696
1697 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1698 ValueList Operands;
1699 // Prepare the operand vector.
1700 for (Value *j : VL)
1701 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1702
1703 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1704 }
1705 return;
1706 }
1707 case Instruction::Select:
1708 case Instruction::Add:
1709 case Instruction::FAdd:
1710 case Instruction::Sub:
1711 case Instruction::FSub:
1712 case Instruction::Mul:
1713 case Instruction::FMul:
1714 case Instruction::UDiv:
1715 case Instruction::SDiv:
1716 case Instruction::FDiv:
1717 case Instruction::URem:
1718 case Instruction::SRem:
1719 case Instruction::FRem:
1720 case Instruction::Shl:
1721 case Instruction::LShr:
1722 case Instruction::AShr:
1723 case Instruction::And:
1724 case Instruction::Or:
1725 case Instruction::Xor:
1726 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1727 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1728
1729 // Sort operands of the instructions so that each side is more likely to
1730 // have the same opcode.
1731 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1732 ValueList Left, Right;
1733 reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1734 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1735 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1736 return;
1737 }
1738
1739 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1740 ValueList Operands;
1741 // Prepare the operand vector.
1742 for (Value *j : VL)
1743 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1744
1745 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1746 }
1747 return;
1748
1749 case Instruction::GetElementPtr: {
1750 // We don't combine GEPs with complicated (nested) indexing.
1751 for (unsigned j = 0; j < VL.size(); ++j) {
1752 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1753 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1754 BS.cancelScheduling(VL, VL0);
1755 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1756 return;
1757 }
1758 }
1759
1760 // We can't combine several GEPs into one vector if they operate on
1761 // different types.
1762 Type *Ty0 = VL0->getOperand(0)->getType();
1763 for (unsigned j = 0; j < VL.size(); ++j) {
1764 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1765 if (Ty0 != CurTy) {
1766 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1767 BS.cancelScheduling(VL, VL0);
1768 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1769 return;
1770 }
1771 }
1772
1773 // We don't combine GEPs with non-constant indexes.
1774 for (unsigned j = 0; j < VL.size(); ++j) {
1775 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1776 if (!isa<ConstantInt>(Op)) {
1777 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1778 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1779 BS.cancelScheduling(VL, VL0);
1780 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1781 return;
1782 }
1783 }
1784
1785 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1786 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1787 for (unsigned i = 0, e = 2; i < e; ++i) {
1788 ValueList Operands;
1789 // Prepare the operand vector.
1790 for (Value *j : VL)
1791 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1792
1793 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1794 }
1795 return;
1796 }
1797 case Instruction::Store: {
1798 // Check if the stores are consecutive or of we need to swizzle them.
1799 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1800 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1801 BS.cancelScheduling(VL, VL0);
1802 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1803 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1804 return;
1805 }
1806
1807 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1808 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1809
1810 ValueList Operands;
1811 for (Value *j : VL)
1812 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1813
1814 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1815 return;
1816 }
1817 case Instruction::Call: {
1818 // Check if the calls are all to the same vectorizable intrinsic.
1819 CallInst *CI = cast<CallInst>(VL0);
1820 // Check if this is an Intrinsic call or something that can be
1821 // represented by an intrinsic call
1822 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1823 if (!isTriviallyVectorizable(ID)) {
1824 BS.cancelScheduling(VL, VL0);
1825 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1826 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1827 return;
1828 }
1829 Function *Int = CI->getCalledFunction();
1830 Value *A1I = nullptr;
1831 if (hasVectorInstrinsicScalarOpd(ID, 1))
1832 A1I = CI->getArgOperand(1);
1833 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1834 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1835 if (!CI2 || CI2->getCalledFunction() != Int ||
1836 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1837 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1838 BS.cancelScheduling(VL, VL0);
1839 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1840 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1841 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1842 return;
1843 }
1844 // ctlz,cttz and powi are special intrinsics whose second argument
1845 // should be same in order for them to be vectorized.
1846 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1847 Value *A1J = CI2->getArgOperand(1);
1848 if (A1I != A1J) {
1849 BS.cancelScheduling(VL, VL0);
1850 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1851 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1852 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1853 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
;
1854 return;
1855 }
1856 }
1857 // Verify that the bundle operands are identical between the two calls.
1858 if (CI->hasOperandBundles() &&
1859 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1860 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1861 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1862 BS.cancelScheduling(VL, VL0);
1863 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1864 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1865 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1866 return;
1867 }
1868 }
1869
1870 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1871 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1872 ValueList Operands;
1873 // Prepare the operand vector.
1874 for (Value *j : VL) {
1875 CallInst *CI2 = dyn_cast<CallInst>(j);
1876 Operands.push_back(CI2->getArgOperand(i));
1877 }
1878 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1879 }
1880 return;
1881 }
1882 case Instruction::ShuffleVector:
1883 // If this is not an alternate sequence of opcode like add-sub
1884 // then do not vectorize this instruction.
1885 if (!S.IsAltShuffle) {
1886 BS.cancelScheduling(VL, VL0);
1887 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1888 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
1889 return;
1890 }
1891 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1892 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
1893
1894 // Reorder operands if reordering would enable vectorization.
1895 if (isa<BinaryOperator>(VL0)) {
1896 ValueList Left, Right;
1897 reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1898 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1899 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1900 return;
1901 }
1902
1903 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1904 ValueList Operands;
1905 // Prepare the operand vector.
1906 for (Value *j : VL)
1907 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1908
1909 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1910 }
1911 return;
1912
1913 default:
1914 BS.cancelScheduling(VL, VL0);
1915 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1916 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
1917 return;
1918 }
1919}
1920
1921unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1922 unsigned N;
1923 Type *EltTy;
1924 auto *ST = dyn_cast<StructType>(T);
1925 if (ST) {
1926 N = ST->getNumElements();
1927 EltTy = *ST->element_begin();
1928 } else {
1929 N = cast<ArrayType>(T)->getNumElements();
1930 EltTy = cast<ArrayType>(T)->getElementType();
1931 }
1932 if (!isValidElementType(EltTy))
1933 return 0;
1934 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1935 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1936 return 0;
1937 if (ST) {
1938 // Check that struct is homogeneous.
1939 for (const auto *Ty : ST->elements())
1940 if (Ty != EltTy)
1941 return 0;
1942 }
1943 return N;
1944}
1945
1946bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const {
1947 Instruction *E0 = cast<Instruction>(OpValue);
1948 assert(E0->getOpcode() == Instruction::ExtractElement ||(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1949, __extension__ __PRETTY_FUNCTION__))
1949 E0->getOpcode() == Instruction::ExtractValue)(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1949, __extension__ __PRETTY_FUNCTION__))
;
1950 assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode")(static_cast <bool> (E0->getOpcode() == getSameOpcode
(VL).Opcode && "Invalid opcode") ? void (0) : __assert_fail
("E0->getOpcode() == getSameOpcode(VL).Opcode && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1950, __extension__ __PRETTY_FUNCTION__))
;
1951 // Check if all of the extracts come from the same vector and from the
1952 // correct offset.
1953 Value *Vec = E0->getOperand(0);
1954
1955 // We have to extract from a vector/aggregate with the same number of elements.
1956 unsigned NElts;
1957 if (E0->getOpcode() == Instruction::ExtractValue) {
1958 const DataLayout &DL = E0->getModule()->getDataLayout();
1959 NElts = canMapToVector(Vec->getType(), DL);
1960 if (!NElts)
1961 return false;
1962 // Check if load can be rewritten as load of vector.
1963 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1964 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1965 return false;
1966 } else {
1967 NElts = Vec->getType()->getVectorNumElements();
1968 }
1969
1970 if (NElts != VL.size())
1971 return false;
1972
1973 // Check that all of the indices extract from the correct offset.
1974 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
1975 Instruction *Inst = cast<Instruction>(VL[I]);
1976 if (!matchExtractIndex(Inst, I, Inst->getOpcode()))
1977 return false;
1978 if (Inst->getOperand(0) != Vec)
1979 return false;
1980 }
1981
1982 return true;
1983}
1984
1985bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
1986 return I->hasOneUse() ||
1987 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
1988 return ScalarToTreeEntry.count(U) > 0;
1989 });
1990}
1991
1992int BoUpSLP::getEntryCost(TreeEntry *E) {
1993 ArrayRef<Value*> VL = E->Scalars;
1994
1995 Type *ScalarTy = VL[0]->getType();
1996 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1997 ScalarTy = SI->getValueOperand()->getType();
1998 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
1999 ScalarTy = CI->getOperand(0)->getType();
2000 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2001
2002 // If we have computed a smaller type for the expression, update VecTy so
2003 // that the costs will be accurate.
2004 if (MinBWs.count(VL[0]))
2005 VecTy = VectorType::get(
2006 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2007
2008 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2009 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2010 int ReuseShuffleCost = 0;
2011 if (NeedToShuffleReuses) {
2012 ReuseShuffleCost =
2013 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2014 }
2015 if (E->NeedToGather) {
2016 if (allConstant(VL))
2017 return 0;
2018 if (isSplat(VL)) {
2019 return ReuseShuffleCost +
2020 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2021 }
2022 if (getSameOpcode(VL).Opcode == Instruction::ExtractElement &&
2023 allSameType(VL) && allSameBlock(VL)) {
2024 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
2025 if (ShuffleKind.hasValue()) {
2026 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2027 for (auto *V : VL) {
2028 // If all users of instruction are going to be vectorized and this
2029 // instruction itself is not going to be vectorized, consider this
2030 // instruction as dead and remove its cost from the final cost of the
2031 // vectorized tree.
2032 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2033 !ScalarToTreeEntry.count(V)) {
2034 auto *IO = cast<ConstantInt>(
2035 cast<ExtractElementInst>(V)->getIndexOperand());
2036 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2037 IO->getZExtValue());
2038 }
2039 }
2040 return ReuseShuffleCost + Cost;
2041 }
2042 }
2043 return ReuseShuffleCost + getGatherCost(VL);
2044 }
2045 InstructionsState S = getSameOpcode(VL);
2046 assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")(static_cast <bool> (S.Opcode && allSameType(VL
) && allSameBlock(VL) && "Invalid VL") ? void
(0) : __assert_fail ("S.Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2046, __extension__ __PRETTY_FUNCTION__))
;
2047 Instruction *VL0 = cast<Instruction>(S.OpValue);
2048 unsigned ShuffleOrOp = S.IsAltShuffle ?
2049 (unsigned) Instruction::ShuffleVector : S.Opcode;
2050 switch (ShuffleOrOp) {
2051 case Instruction::PHI:
2052 return 0;
2053
2054 case Instruction::ExtractValue:
2055 case Instruction::ExtractElement:
2056 if (NeedToShuffleReuses) {
2057 unsigned Idx = 0;
2058 for (unsigned I : E->ReuseShuffleIndices) {
2059 if (ShuffleOrOp == Instruction::ExtractElement) {
2060 auto *IO = cast<ConstantInt>(
2061 cast<ExtractElementInst>(VL[I])->getIndexOperand());
2062 Idx = IO->getZExtValue();
2063 ReuseShuffleCost -= TTI->getVectorInstrCost(
2064 Instruction::ExtractElement, VecTy, Idx);
2065 } else {
2066 ReuseShuffleCost -= TTI->getVectorInstrCost(
2067 Instruction::ExtractElement, VecTy, Idx);
2068 ++Idx;
2069 }
2070 }
2071 Idx = ReuseShuffleNumbers;
2072 for (Value *V : VL) {
2073 if (ShuffleOrOp == Instruction::ExtractElement) {
2074 auto *IO = cast<ConstantInt>(
2075 cast<ExtractElementInst>(V)->getIndexOperand());
2076 Idx = IO->getZExtValue();
2077 } else {
2078 --Idx;
2079 }
2080 ReuseShuffleCost +=
2081 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2082 }
2083 }
2084 if (canReuseExtract(VL, S.OpValue)) {
2085 int DeadCost = ReuseShuffleCost;
2086 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2087 Instruction *E = cast<Instruction>(VL[i]);
2088 // If all users are going to be vectorized, instruction can be
2089 // considered as dead.
2090 // The same, if have only one user, it will be vectorized for sure.
2091 if (areAllUsersVectorized(E))
2092 // Take credit for instruction that will become dead.
2093 DeadCost -=
2094 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2095 }
2096 return DeadCost;
2097 }
2098 return ReuseShuffleCost + getGatherCost(VL);
2099
2100 case Instruction::ZExt:
2101 case Instruction::SExt:
2102 case Instruction::FPToUI:
2103 case Instruction::FPToSI:
2104 case Instruction::FPExt:
2105 case Instruction::PtrToInt:
2106 case Instruction::IntToPtr:
2107 case Instruction::SIToFP:
2108 case Instruction::UIToFP:
2109 case Instruction::Trunc:
2110 case Instruction::FPTrunc:
2111 case Instruction::BitCast: {
2112 Type *SrcTy = VL0->getOperand(0)->getType();
2113 if (NeedToShuffleReuses) {
2114 ReuseShuffleCost -=
2115 (ReuseShuffleNumbers - VL.size()) *
2116 TTI->getCastInstrCost(S.Opcode, ScalarTy, SrcTy, VL0);
2117 }
2118
2119 // Calculate the cost of this instruction.
2120 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2121 VL0->getType(), SrcTy, VL0);
2122
2123 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2124 int VecCost = 0;
2125 // Check if the values are candidates to demote.
2126 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2127 VecCost = ReuseShuffleCost +
2128 TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2129 }
2130 return VecCost - ScalarCost;
2131 }
2132 case Instruction::FCmp:
2133 case Instruction::ICmp:
2134 case Instruction::Select: {
2135 // Calculate the cost of this instruction.
2136 if (NeedToShuffleReuses) {
2137 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2138 TTI->getCmpSelInstrCost(S.Opcode, ScalarTy,
2139 Builder.getInt1Ty(), VL0);
2140 }
2141 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2142 int ScalarCost = VecTy->getNumElements() *
2143 TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2144 int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2145 return ReuseShuffleCost + VecCost - ScalarCost;
2146 }
2147 case Instruction::Add:
2148 case Instruction::FAdd:
2149 case Instruction::Sub:
2150 case Instruction::FSub:
2151 case Instruction::Mul:
2152 case Instruction::FMul:
2153 case Instruction::UDiv:
2154 case Instruction::SDiv:
2155 case Instruction::FDiv:
2156 case Instruction::URem:
2157 case Instruction::SRem:
2158 case Instruction::FRem:
2159 case Instruction::Shl:
2160 case Instruction::LShr:
2161 case Instruction::AShr:
2162 case Instruction::And:
2163 case Instruction::Or:
2164 case Instruction::Xor: {
2165 // Certain instructions can be cheaper to vectorize if they have a
2166 // constant second vector operand.
2167 TargetTransformInfo::OperandValueKind Op1VK =
2168 TargetTransformInfo::OK_AnyValue;
2169 TargetTransformInfo::OperandValueKind Op2VK =
2170 TargetTransformInfo::OK_UniformConstantValue;
2171 TargetTransformInfo::OperandValueProperties Op1VP =
2172 TargetTransformInfo::OP_None;
2173 TargetTransformInfo::OperandValueProperties Op2VP =
2174 TargetTransformInfo::OP_None;
2175
2176 // If all operands are exactly the same ConstantInt then set the
2177 // operand kind to OK_UniformConstantValue.
2178 // If instead not all operands are constants, then set the operand kind
2179 // to OK_AnyValue. If all operands are constants but not the same,
2180 // then set the operand kind to OK_NonUniformConstantValue.
2181 ConstantInt *CInt = nullptr;
2182 for (unsigned i = 0; i < VL.size(); ++i) {
2183 const Instruction *I = cast<Instruction>(VL[i]);
2184 if (!isa<ConstantInt>(I->getOperand(1))) {
2185 Op2VK = TargetTransformInfo::OK_AnyValue;
2186 break;
2187 }
2188 if (i == 0) {
2189 CInt = cast<ConstantInt>(I->getOperand(1));
2190 continue;
2191 }
2192 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
2193 CInt != cast<ConstantInt>(I->getOperand(1)))
2194 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2195 }
2196 // FIXME: Currently cost of model modification for division by power of
2197 // 2 is handled for X86 and AArch64. Add support for other targets.
2198 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2199 CInt->getValue().isPowerOf2())
2200 Op2VP = TargetTransformInfo::OP_PowerOf2;
2201
2202 SmallVector<const Value *, 4> Operands(VL0->operand_values());
2203 if (NeedToShuffleReuses) {
2204 ReuseShuffleCost -=
2205 (ReuseShuffleNumbers - VL.size()) *
2206 TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2207 Op2VP, Operands);
2208 }
2209 int ScalarCost =
2210 VecTy->getNumElements() *
2211 TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2212 Op2VP, Operands);
2213 int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2214 Op1VP, Op2VP, Operands);
2215 return ReuseShuffleCost + VecCost - ScalarCost;
2216 }
2217 case Instruction::GetElementPtr: {
2218 TargetTransformInfo::OperandValueKind Op1VK =
2219 TargetTransformInfo::OK_AnyValue;
2220 TargetTransformInfo::OperandValueKind Op2VK =
2221 TargetTransformInfo::OK_UniformConstantValue;
2222
2223 if (NeedToShuffleReuses) {
2224 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2225 TTI->getArithmeticInstrCost(Instruction::Add,
2226 ScalarTy, Op1VK, Op2VK);
2227 }
2228 int ScalarCost =
2229 VecTy->getNumElements() *
2230 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2231 int VecCost =
2232 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2233
2234 return ReuseShuffleCost + VecCost - ScalarCost;
2235 }
2236 case Instruction::Load: {
2237 // Cost of wide load - cost of scalar loads.
2238 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2239 if (NeedToShuffleReuses) {
2240 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2241 TTI->getMemoryOpCost(Instruction::Load, ScalarTy,
2242 alignment, 0, VL0);
2243 }
2244 int ScalarLdCost = VecTy->getNumElements() *
2245 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2246 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2247 VecTy, alignment, 0, VL0);
2248 return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2249 }
2250 case Instruction::Store: {
2251 // We know that we can merge the stores. Calculate the cost.
2252 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2253 if (NeedToShuffleReuses) {
2254 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2255 TTI->getMemoryOpCost(Instruction::Store, ScalarTy,
2256 alignment, 0, VL0);
2257 }
2258 int ScalarStCost = VecTy->getNumElements() *
2259 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2260 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2261 VecTy, alignment, 0, VL0);
2262 return ReuseShuffleCost + VecStCost - ScalarStCost;
2263 }
2264 case Instruction::Call: {
2265 CallInst *CI = cast<CallInst>(VL0);
2266 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2267
2268 // Calculate the cost of the scalar and vector calls.
2269 SmallVector<Type*, 4> ScalarTys;
2270 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2271 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2272
2273 FastMathFlags FMF;
2274 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2275 FMF = FPMO->getFastMathFlags();
2276
2277 if (NeedToShuffleReuses) {
2278 ReuseShuffleCost -=
2279 (ReuseShuffleNumbers - VL.size()) *
2280 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2281 }
2282 int ScalarCallCost = VecTy->getNumElements() *
2283 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2284
2285 SmallVector<Value *, 4> Args(CI->arg_operands());
2286 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2287 VecTy->getNumElements());
2288
2289 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2290 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2291 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
2292
2293 return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2294 }
2295 case Instruction::ShuffleVector: {
2296 TargetTransformInfo::OperandValueKind Op1VK =
2297 TargetTransformInfo::OK_AnyValue;
2298 TargetTransformInfo::OperandValueKind Op2VK =
2299 TargetTransformInfo::OK_AnyValue;
2300 int ScalarCost = 0;
2301 if (NeedToShuffleReuses) {
2302 for (unsigned Idx : E->ReuseShuffleIndices) {
2303 Instruction *I = cast<Instruction>(VL[Idx]);
2304 if (!I)
2305 continue;
2306 ReuseShuffleCost -= TTI->getArithmeticInstrCost(
2307 I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2308 }
2309 for (Value *V : VL) {
2310 Instruction *I = cast<Instruction>(V);
2311 if (!I)
2312 continue;
2313 ReuseShuffleCost += TTI->getArithmeticInstrCost(
2314 I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2315 }
2316 }
2317 int VecCost = 0;
2318 for (Value *i : VL) {
2319 Instruction *I = cast<Instruction>(i);
2320 if (!I)
2321 break;
2322 ScalarCost +=
2323 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2324 }
2325 // VecCost is equal to sum of the cost of creating 2 vectors
2326 // and the cost of creating shuffle.
2327 Instruction *I0 = cast<Instruction>(VL[0]);
2328 VecCost =
2329 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2330 Instruction *I1 = cast<Instruction>(VL[1]);
2331 VecCost +=
2332 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2333 VecCost +=
2334 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2335 return ReuseShuffleCost + VecCost - ScalarCost;
2336 }
2337 default:
2338 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2338)
;
2339 }
2340}
2341
2342bool BoUpSLP::isFullyVectorizableTinyTree() {
2343 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
2344 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
2345
2346 // We only handle trees of heights 1 and 2.
2347 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2348 return true;
2349
2350 if (VectorizableTree.size() != 2)
2351 return false;
2352
2353 // Handle splat and all-constants stores.
2354 if (!VectorizableTree[0].NeedToGather &&
2355 (allConstant(VectorizableTree[1].Scalars) ||
2356 isSplat(VectorizableTree[1].Scalars)))
2357 return true;
2358
2359 // Gathering cost would be too much for tiny trees.
2360 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2361 return false;
2362
2363 return true;
2364}
2365
2366bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2367 // We can vectorize the tree if its size is greater than or equal to the
2368 // minimum size specified by the MinTreeSize command line option.
2369 if (VectorizableTree.size() >= MinTreeSize)
2370 return false;
2371
2372 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2373 // can vectorize it if we can prove it fully vectorizable.
2374 if (isFullyVectorizableTinyTree())
2375 return false;
2376
2377 assert(VectorizableTree.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2379, __extension__ __PRETTY_FUNCTION__))
2378 ? ExternalUses.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2379, __extension__ __PRETTY_FUNCTION__))
2379 : true && "We shouldn't have any external users")(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2379, __extension__ __PRETTY_FUNCTION__))
;
2380
2381 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2382 // vectorizable.
2383 return true;
2384}
2385
2386int BoUpSLP::getSpillCost() {
2387 // Walk from the bottom of the tree to the top, tracking which values are
2388 // live. When we see a call instruction that is not part of our tree,
2389 // query TTI to see if there is a cost to keeping values live over it
2390 // (for example, if spills and fills are required).
2391 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2392 int Cost = 0;
2393
2394 SmallPtrSet<Instruction*, 4> LiveValues;
2395 Instruction *PrevInst = nullptr;
2396
2397 for (const auto &N : VectorizableTree) {
2398 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2399 if (!Inst)
2400 continue;
2401
2402 if (!PrevInst) {
2403 PrevInst = Inst;
2404 continue;
2405 }
2406
2407 // Update LiveValues.
2408 LiveValues.erase(PrevInst);
2409 for (auto &J : PrevInst->operands()) {
2410 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2411 LiveValues.insert(cast<Instruction>(&*J));
2412 }
2413
2414 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2415 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2416 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2417 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2418 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2419 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2420 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
;
2421
2422 // Now find the sequence of instructions between PrevInst and Inst.
2423 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2424 PrevInstIt =
2425 PrevInst->getIterator().getReverse();
2426 while (InstIt != PrevInstIt) {
2427 if (PrevInstIt == PrevInst->getParent()->rend()) {
2428 PrevInstIt = Inst->getParent()->rbegin();
2429 continue;
2430 }
2431
2432 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2433 SmallVector<Type*, 4> V;
2434 for (auto *II : LiveValues)
2435 V.push_back(VectorType::get(II->getType(), BundleWidth));
2436 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2437 }
2438
2439 ++PrevInstIt;
2440 }
2441
2442 PrevInst = Inst;
2443 }
2444
2445 return Cost;
2446}
2447
2448int BoUpSLP::getTreeCost() {
2449 int Cost = 0;
2450 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
2451 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
2452
2453 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2454
2455 for (TreeEntry &TE : VectorizableTree) {
2456 int C = getEntryCost(&TE);
2457 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
2458 << *TE.Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
2459 Cost += C;
2460 }
2461
2462 SmallSet<Value *, 16> ExtractCostCalculated;
2463 int ExtractCost = 0;
2464 for (ExternalUser &EU : ExternalUses) {
2465 // We only add extract cost once for the same scalar.
2466 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2467 continue;
2468
2469 // Uses by ephemeral values are free (because the ephemeral value will be
2470 // removed prior to code generation, and so the extraction will be
2471 // removed as well).
2472 if (EphValues.count(EU.User))
2473 continue;
2474
2475 // If we plan to rewrite the tree in a smaller type, we will need to sign
2476 // extend the extracted value back to the original type. Here, we account
2477 // for the extract and the added cost of the sign extend if needed.
2478 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2479 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2480 if (MinBWs.count(ScalarRoot)) {
2481 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2482 auto Extend =
2483 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2484 VecTy = VectorType::get(MinTy, BundleWidth);
2485 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2486 VecTy, EU.Lane);
2487 } else {
2488 ExtractCost +=
2489 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2490 }
2491 }
2492
2493 int SpillCost = getSpillCost();
2494 Cost += SpillCost + ExtractCost;
2495
2496 std::string Str;
2497 {
2498 raw_string_ostream OS(Str);
2499 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2500 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2501 << "SLP: Total Cost = " << Cost << ".\n";
2502 }
2503 DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
2504
2505 if (ViewSLPTree)
2506 ViewGraph(this, "SLP" + F->getName(), false, Str);
2507
2508 return Cost;
2509}
2510
2511int BoUpSLP::getGatherCost(Type *Ty,
2512 const DenseSet<unsigned> &ShuffledIndices) {
2513 int Cost = 0;
2514 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2515 if (!ShuffledIndices.count(i))
2516 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2517 if (!ShuffledIndices.empty())
2518 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2519 return Cost;
2520}
2521
2522int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2523 // Find the type of the operands in VL.
2524 Type *ScalarTy = VL[0]->getType();
2525 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2526 ScalarTy = SI->getValueOperand()->getType();
2527 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2528 // Find the cost of inserting/extracting values from the vector.
2529 // Check if the same elements are inserted several times and count them as
2530 // shuffle candidates.
2531 DenseSet<unsigned> ShuffledElements;
2532 DenseSet<Value *> UniqueElements;
2533 // Iterate in reverse order to consider insert elements with the high cost.
2534 for (unsigned I = VL.size(); I > 0; --I) {
2535 unsigned Idx = I - 1;
2536 if (!UniqueElements.insert(VL[Idx]).second)
2537 ShuffledElements.insert(Idx);
2538 }
2539 return getGatherCost(VecTy, ShuffledElements);
2540}
2541
2542// Reorder commutative operations in alternate shuffle if the resulting vectors
2543// are consecutive loads. This would allow us to vectorize the tree.
2544// If we have something like-
2545// load a[0] - load b[0]
2546// load b[1] + load a[1]
2547// load a[2] - load b[2]
2548// load a[3] + load b[3]
2549// Reordering the second load b[1] load a[1] would allow us to vectorize this
2550// code.
2551void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2552 SmallVectorImpl<Value *> &Left,
2553 SmallVectorImpl<Value *> &Right) {
2554 // Push left and right operands of binary operation into Left and Right
2555 unsigned AltOpcode = getAltOpcode(Opcode);
2556 (void)AltOpcode;
2557 for (Value *V : VL) {
2558 auto *I = cast<Instruction>(V);
2559 assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&(static_cast <bool> (sameOpcodeOrAlt(Opcode, AltOpcode,
I->getOpcode()) && "Incorrect instruction in vector"
) ? void (0) : __assert_fail ("sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2560, __extension__ __PRETTY_FUNCTION__))
2560 "Incorrect instruction in vector")(static_cast <bool> (sameOpcodeOrAlt(Opcode, AltOpcode,
I->getOpcode()) && "Incorrect instruction in vector"
) ? void (0) : __assert_fail ("sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2560, __extension__ __PRETTY_FUNCTION__))
;
2561 Left.push_back(I->getOperand(0));
2562 Right.push_back(I->getOperand(1));
2563 }
2564
2565 // Reorder if we have a commutative operation and consecutive access
2566 // are on either side of the alternate instructions.
2567 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2568 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2569 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2570 Instruction *VL1 = cast<Instruction>(VL[j]);
2571 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2572 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2573 std::swap(Left[j], Right[j]);
2574 continue;
2575 } else if (VL2->isCommutative() &&
2576 isConsecutiveAccess(L, L1, *DL, *SE)) {
2577 std::swap(Left[j + 1], Right[j + 1]);
2578 continue;
2579 }
2580 // else unchanged
2581 }
2582 }
2583 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2584 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2585 Instruction *VL1 = cast<Instruction>(VL[j]);
2586 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2587 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2588 std::swap(Left[j], Right[j]);
2589 continue;
2590 } else if (VL2->isCommutative() &&
2591 isConsecutiveAccess(L, L1, *DL, *SE)) {
2592 std::swap(Left[j + 1], Right[j + 1]);
2593 continue;
2594 }
2595 // else unchanged
2596 }
2597 }
2598 }
2599}
2600
2601// Return true if I should be commuted before adding it's left and right
2602// operands to the arrays Left and Right.
2603//
2604// The vectorizer is trying to either have all elements one side being
2605// instruction with the same opcode to enable further vectorization, or having
2606// a splat to lower the vectorizing cost.
2607static bool shouldReorderOperands(
2608 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2609 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2610 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2611 VLeft = I.getOperand(0);
2612 VRight = I.getOperand(1);
2613 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2614 if (SplatRight) {
2615 if (VRight == Right[i - 1])
2616 // Preserve SplatRight
2617 return false;
2618 if (VLeft == Right[i - 1]) {
2619 // Commuting would preserve SplatRight, but we don't want to break
2620 // SplatLeft either, i.e. preserve the original order if possible.
2621 // (FIXME: why do we care?)
2622 if (SplatLeft && VLeft == Left[i - 1])
2623 return false;
2624 return true;
2625 }
2626 }
2627 // Symmetrically handle Right side.
2628 if (SplatLeft) {
2629 if (VLeft == Left[i - 1])
2630 // Preserve SplatLeft
2631 return false;
2632 if (VRight == Left[i - 1])
2633 return true;
2634 }
2635
2636 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2637 Instruction *IRight = dyn_cast<Instruction>(VRight);
2638
2639 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2640 // it and not the right, in this case we want to commute.
2641 if (AllSameOpcodeRight) {
2642 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2643 if (IRight && RightPrevOpcode == IRight->getOpcode())
2644 // Do not commute, a match on the right preserves AllSameOpcodeRight
2645 return false;
2646 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2647 // We have a match and may want to commute, but first check if there is
2648 // not also a match on the existing operands on the Left to preserve
2649 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2650 // (FIXME: why do we care?)
2651 if (AllSameOpcodeLeft && ILeft &&
2652 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2653 return false;
2654 return true;
2655 }
2656 }
2657 // Symmetrically handle Left side.
2658 if (AllSameOpcodeLeft) {
2659 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2660 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2661 return false;
2662 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2663 return true;
2664 }
2665 return false;
2666}
2667
2668void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2669 ArrayRef<Value *> VL,
2670 SmallVectorImpl<Value *> &Left,
2671 SmallVectorImpl<Value *> &Right) {
2672 if (!VL.empty()) {
2673 // Peel the first iteration out of the loop since there's nothing
2674 // interesting to do anyway and it simplifies the checks in the loop.
2675 auto *I = cast<Instruction>(VL[0]);
2676 Value *VLeft = I->getOperand(0);
2677 Value *VRight = I->getOperand(1);
2678 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2679 // Favor having instruction to the right. FIXME: why?
2680 std::swap(VLeft, VRight);
2681 Left.push_back(VLeft);
2682 Right.push_back(VRight);
2683 }
2684
2685 // Keep track if we have instructions with all the same opcode on one side.
2686 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2687 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2688 // Keep track if we have one side with all the same value (broadcast).
2689 bool SplatLeft = true;
2690 bool SplatRight = true;
2691
2692 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2693 Instruction *I = cast<Instruction>(VL[i]);
2694 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2696, __extension__ __PRETTY_FUNCTION__))
2695 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2696, __extension__ __PRETTY_FUNCTION__))
2696 "Can only process commutative instruction")(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2696, __extension__ __PRETTY_FUNCTION__))
;
2697 // Commute to favor either a splat or maximizing having the same opcodes on
2698 // one side.
2699 Value *VLeft;
2700 Value *VRight;
2701 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2702 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2703 VRight)) {
2704 Left.push_back(VRight);
2705 Right.push_back(VLeft);
2706 } else {
2707 Left.push_back(VLeft);
2708 Right.push_back(VRight);
2709 }
2710 // Update Splat* and AllSameOpcode* after the insertion.
2711 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2712 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2713 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2714 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2715 cast<Instruction>(Left[i])->getOpcode());
2716 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2717 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2718 cast<Instruction>(Right[i])->getOpcode());
2719 }
2720
2721 // If one operand end up being broadcast, return this operand order.
2722 if (SplatRight || SplatLeft)
2723 return;
2724
2725 // Finally check if we can get longer vectorizable chain by reordering
2726 // without breaking the good operand order detected above.
2727 // E.g. If we have something like-
2728 // load a[0] load b[0]
2729 // load b[1] load a[1]
2730 // load a[2] load b[2]
2731 // load a[3] load b[3]
2732 // Reordering the second load b[1] load a[1] would allow us to vectorize
2733 // this code and we still retain AllSameOpcode property.
2734 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2735 // such as-
2736 // add a[0],c[0] load b[0]
2737 // add a[1],c[2] load b[1]
2738 // b[2] load b[2]
2739 // add a[3],c[3] load b[3]
2740 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2741 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2742 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2743 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2744 std::swap(Left[j + 1], Right[j + 1]);
2745 continue;
2746 }
2747 }
2748 }
2749 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2750 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2751 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2752 std::swap(Left[j + 1], Right[j + 1]);
2753 continue;
2754 }
2755 }
2756 }
2757 // else unchanged
2758 }
2759}
2760
2761void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2762 // Get the basic block this bundle is in. All instructions in the bundle
2763 // should be in this block.
2764 auto *Front = cast<Instruction>(OpValue);
2765 auto *BB = Front->getParent();
2766 const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2767 const unsigned AltOpcode = getAltOpcode(Opcode);
2768 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2772, __extension__ __PRETTY_FUNCTION__))
2769 return !sameOpcodeOrAlt(Opcode, AltOpcode,(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2772, __extension__ __PRETTY_FUNCTION__))
2770 cast<Instruction>(V)->getOpcode()) ||(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2772, __extension__ __PRETTY_FUNCTION__))
2771 cast<Instruction>(V)->getParent() == BB;(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2772, __extension__ __PRETTY_FUNCTION__))
2772 }))(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2772, __extension__ __PRETTY_FUNCTION__))
;
2773
2774 // The last instruction in the bundle in program order.
2775 Instruction *LastInst = nullptr;
2776
2777 // Find the last instruction. The common case should be that BB has been
2778 // scheduled, and the last instruction is VL.back(). So we start with
2779 // VL.back() and iterate over schedule data until we reach the end of the
2780 // bundle. The end of the bundle is marked by null ScheduleData.
2781 if (BlocksSchedules.count(BB)) {
2782 auto *Bundle =
2783 BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2784 if (Bundle && Bundle->isPartOfBundle())
2785 for (; Bundle; Bundle = Bundle->NextInBundle)
2786 if (Bundle->OpValue == Bundle->Inst)
2787 LastInst = Bundle->Inst;
2788 }
2789
2790 // LastInst can still be null at this point if there's either not an entry
2791 // for BB in BlocksSchedules or there's no ScheduleData available for
2792 // VL.back(). This can be the case if buildTree_rec aborts for various
2793 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2794 // size is reached, etc.). ScheduleData is initialized in the scheduling
2795 // "dry-run".
2796 //
2797 // If this happens, we can still find the last instruction by brute force. We
2798 // iterate forwards from Front (inclusive) until we either see all
2799 // instructions in the bundle or reach the end of the block. If Front is the
2800 // last instruction in program order, LastInst will be set to Front, and we
2801 // will visit all the remaining instructions in the block.
2802 //
2803 // One of the reasons we exit early from buildTree_rec is to place an upper
2804 // bound on compile-time. Thus, taking an additional compile-time hit here is
2805 // not ideal. However, this should be exceedingly rare since it requires that
2806 // we both exit early from buildTree_rec and that the bundle be out-of-order
2807 // (causing us to iterate all the way to the end of the block).
2808 if (!LastInst) {
2809 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2810 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2811 if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2812 LastInst = &I;
2813 if (Bundle.empty())
2814 break;
2815 }
2816 }
2817
2818 // Set the insertion point after the last instruction in the bundle. Set the
2819 // debug location to Front.
2820 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2821 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2822}
2823
2824Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2825 Value *Vec = UndefValue::get(Ty);
2826 // Generate the 'InsertElement' instruction.
2827 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2828 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2829 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2830 GatherSeq.insert(Insrt);
2831 CSEBlocks.insert(Insrt->getParent());
2832
2833 // Add to our 'need-to-extract' list.
2834 if (TreeEntry *E = getTreeEntry(VL[i])) {
2835 // Find which lane we need to extract.
2836 int FoundLane = -1;
2837 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2838 // Is this the lane of the scalar that we are looking for ?
2839 if (E->Scalars[Lane] == VL[i]) {
2840 FoundLane = Lane;
2841 break;
2842 }
2843 }
2844 assert(FoundLane >= 0 && "Could not find the correct lane")(static_cast <bool> (FoundLane >= 0 && "Could not find the correct lane"
) ? void (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2844, __extension__ __PRETTY_FUNCTION__))
;
2845 if (!E->ReuseShuffleIndices.empty()) {
2846 FoundLane =
2847 std::distance(E->ReuseShuffleIndices.begin(),
2848 llvm::find(E->ReuseShuffleIndices, FoundLane));
2849 }
2850 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2851 }
2852 }
2853 }
2854
2855 return Vec;
2856}
2857
2858Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2859 InstructionsState S = getSameOpcode(VL);
2860 if (S.Opcode) {
2861 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2862 if (E->isSame(VL)) {
2863 Value *V = vectorizeTree(E);
2864 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
2865 // We need to get the vectorized value but without shuffle.
2866 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
2867 V = SV->getOperand(0);
2868 } else {
2869 // Reshuffle to get only unique values.
2870 SmallVector<unsigned, 4> UniqueIdxs;
2871 SmallSet<unsigned, 4> UsedIdxs;
2872 for(unsigned Idx : E->ReuseShuffleIndices)
2873 if (UsedIdxs.insert(Idx).second)
2874 UniqueIdxs.emplace_back(Idx);
2875 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
2876 UniqueIdxs);
2877 }
2878 }
2879 return V;
2880 }
2881 }
2882 }
2883
2884 Type *ScalarTy = S.OpValue->getType();
2885 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2886 ScalarTy = SI->getValueOperand()->getType();
2887
2888 // Check that every instruction appears once in this bundle.
2889 SmallVector<unsigned, 4> ReuseShuffleIndicies;
2890 SmallVector<Value *, 4> UniqueValues;
2891 if (VL.size() > 2) {
2892 DenseMap<Value *, unsigned> UniquePositions;
2893 for (Value *V : VL) {
2894 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2895 ReuseShuffleIndicies.emplace_back(Res.first->second);
2896 if (Res.second || isa<Constant>(V))
2897 UniqueValues.emplace_back(V);
2898 }
2899 // Do not shuffle single element or if number of unique values is not power
2900 // of 2.
2901 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
2902 !llvm::isPowerOf2_32(UniqueValues.size()))
2903 ReuseShuffleIndicies.clear();
2904 else
2905 VL = UniqueValues;
2906 }
2907 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2908
2909 Value *V = Gather(VL, VecTy);
2910 if (!ReuseShuffleIndicies.empty()) {
2911 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
2912 ReuseShuffleIndicies, "shuffle");
2913 if (auto *I = dyn_cast<Instruction>(V)) {
2914 GatherSeq.insert(I);
2915 CSEBlocks.insert(I->getParent());
2916 }
2917 }
2918 return V;
2919}
2920
2921Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2922 IRBuilder<>::InsertPointGuard Guard(Builder);
2923
2924 if (E->VectorizedValue) {
1
Assuming the condition is false
2
Taking false branch
2925 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
2926 return E->VectorizedValue;
2927 }
2928
2929 InstructionsState S = getSameOpcode(E->Scalars);
2930 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2931 Type *ScalarTy = VL0->getType();
2932 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3
Taking false branch
2933 ScalarTy = SI->getValueOperand()->getType();
2934 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2935
2936 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2937
2938 if (E->NeedToGather) {
4
Assuming the condition is false
5
Taking false branch
2939 setInsertPointAfterBundle(E->Scalars, VL0);
2940 auto *V = Gather(E->Scalars, VecTy);
2941 if (NeedToShuffleReuses) {
2942 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
2943 E->ReuseShuffleIndices, "shuffle");
2944 if (auto *I = dyn_cast<Instruction>(V)) {
2945 GatherSeq.insert(I);
2946 CSEBlocks.insert(I->getParent());
2947 }
2948 }
2949 E->VectorizedValue = V;
2950 return V;
2951 }
2952
2953 unsigned ShuffleOrOp = S.IsAltShuffle ?
6
'?' condition is false
2954 (unsigned) Instruction::ShuffleVector : S.Opcode;
2955 switch (ShuffleOrOp) {
7
Control jumps to 'case Call:' at line 3302
2956 case Instruction::PHI: {
2957 PHINode *PH = dyn_cast<PHINode>(VL0);
2958 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2959 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2960 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2961 Value *V = NewPhi;
2962 if (NeedToShuffleReuses) {
2963 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
2964 E->ReuseShuffleIndices, "shuffle");
2965 }
2966 E->VectorizedValue = V;
2967
2968 // PHINodes may have multiple entries from the same block. We want to
2969 // visit every block once.
2970 SmallSet<BasicBlock*, 4> VisitedBBs;
2971
2972 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2973 ValueList Operands;
2974 BasicBlock *IBB = PH->getIncomingBlock(i);
2975
2976 if (!VisitedBBs.insert(IBB).second) {
2977 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2978 continue;
2979 }
2980
2981 // Prepare the operand vector.
2982 for (Value *V : E->Scalars)
2983 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2984
2985 Builder.SetInsertPoint(IBB->getTerminator());
2986 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2987 Value *Vec = vectorizeTree(Operands);
2988 NewPhi->addIncoming(Vec, IBB);
2989 }
2990
2991 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2992, __extension__ __PRETTY_FUNCTION__))
2992 "Invalid number of incoming values")(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2992, __extension__ __PRETTY_FUNCTION__))
;
2993 return V;
2994 }
2995
2996 case Instruction::ExtractElement: {
2997 if (canReuseExtract(E->Scalars, VL0)) {
2998 Value *V = VL0->getOperand(0);
2999 if (NeedToShuffleReuses) {
3000 Builder.SetInsertPoint(VL0);
3001 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3002 E->ReuseShuffleIndices, "shuffle");
3003 }
3004 E->VectorizedValue = V;
3005 return V;
3006 }
3007 setInsertPointAfterBundle(E->Scalars, VL0);
3008 auto *V = Gather(E->Scalars, VecTy);
3009 if (NeedToShuffleReuses) {
3010 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3011 E->ReuseShuffleIndices, "shuffle");
3012 if (auto *I = dyn_cast<Instruction>(V)) {
3013 GatherSeq.insert(I);
3014 CSEBlocks.insert(I->getParent());
3015 }
3016 }
3017 E->VectorizedValue = V;
3018 return V;
3019 }
3020 case Instruction::ExtractValue: {
3021 if (canReuseExtract(E->Scalars, VL0)) {
3022 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3023 Builder.SetInsertPoint(LI);
3024 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3025 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3026 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
3027 Value *NewV = propagateMetadata(V, E->Scalars);
3028 if (NeedToShuffleReuses) {
3029 NewV = Builder.CreateShuffleVector(
3030 NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3031 }
3032 E->VectorizedValue = NewV;
3033 return NewV;
3034 }
3035 setInsertPointAfterBundle(E->Scalars, VL0);
3036 auto *V = Gather(E->Scalars, VecTy);
3037 if (NeedToShuffleReuses) {
3038 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3039 E->ReuseShuffleIndices, "shuffle");
3040 if (auto *I = dyn_cast<Instruction>(V)) {
3041 GatherSeq.insert(I);
3042 CSEBlocks.insert(I->getParent());
3043 }
3044 }
3045 E->VectorizedValue = V;
3046 return V;
3047 }
3048 case Instruction::ZExt:
3049 case Instruction::SExt:
3050 case Instruction::FPToUI:
3051 case Instruction::FPToSI:
3052 case Instruction::FPExt:
3053 case Instruction::PtrToInt:
3054 case Instruction::IntToPtr:
3055 case Instruction::SIToFP:
3056 case Instruction::UIToFP:
3057 case Instruction::Trunc:
3058 case Instruction::FPTrunc:
3059 case Instruction::BitCast: {
3060 ValueList INVL;
3061 for (Value *V : E->Scalars)
3062 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3063
3064 setInsertPointAfterBundle(E->Scalars, VL0);
3065
3066 Value *InVec = vectorizeTree(INVL);
3067
3068 if (E->VectorizedValue) {
3069 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3070 return E->VectorizedValue;
3071 }
3072
3073 CastInst *CI = dyn_cast<CastInst>(VL0);
3074 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3075 if (NeedToShuffleReuses) {
3076 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3077 E->ReuseShuffleIndices, "shuffle");
3078 }
3079 E->VectorizedValue = V;
3080 ++NumVectorInstructions;
3081 return V;
3082 }
3083 case Instruction::FCmp:
3084 case Instruction::ICmp: {
3085 ValueList LHSV, RHSV;
3086 for (Value *V : E->Scalars) {
3087 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3088 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3089 }
3090
3091 setInsertPointAfterBundle(E->Scalars, VL0);
3092
3093 Value *L = vectorizeTree(LHSV);
3094 Value *R = vectorizeTree(RHSV);
3095
3096 if (E->VectorizedValue) {
3097 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3098 return E->VectorizedValue;
3099 }
3100
3101 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3102 Value *V;
3103 if (S.Opcode == Instruction::FCmp)
3104 V = Builder.CreateFCmp(P0, L, R);
3105 else
3106 V = Builder.CreateICmp(P0, L, R);
3107
3108 propagateIRFlags(V, E->Scalars, VL0);
3109 if (NeedToShuffleReuses) {
3110 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3111 E->ReuseShuffleIndices, "shuffle");
3112 }
3113 E->VectorizedValue = V;
3114 ++NumVectorInstructions;
3115 return V;
3116 }
3117 case Instruction::Select: {
3118 ValueList TrueVec, FalseVec, CondVec;
3119 for (Value *V : E->Scalars) {
3120 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3121 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3122 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3123 }
3124
3125 setInsertPointAfterBundle(E->Scalars, VL0);
3126
3127 Value *Cond = vectorizeTree(CondVec);
3128 Value *True = vectorizeTree(TrueVec);
3129 Value *False = vectorizeTree(FalseVec);
3130
3131 if (E->VectorizedValue) {
3132 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3133 return E->VectorizedValue;
3134 }
3135
3136 Value *V = Builder.CreateSelect(Cond, True, False);
3137 if (NeedToShuffleReuses) {
3138 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3139 E->ReuseShuffleIndices, "shuffle");
3140 }
3141 E->VectorizedValue = V;
3142 ++NumVectorInstructions;
3143 return V;
3144 }
3145 case Instruction::Add:
3146 case Instruction::FAdd:
3147 case Instruction::Sub:
3148 case Instruction::FSub:
3149 case Instruction::Mul:
3150 case Instruction::FMul:
3151 case Instruction::UDiv:
3152 case Instruction::SDiv:
3153 case Instruction::FDiv:
3154 case Instruction::URem:
3155 case Instruction::SRem:
3156 case Instruction::FRem:
3157 case Instruction::Shl:
3158 case Instruction::LShr:
3159 case Instruction::AShr:
3160 case Instruction::And:
3161 case Instruction::Or:
3162 case Instruction::Xor: {
3163 ValueList LHSVL, RHSVL;
3164 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3165 reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
3166 RHSVL);
3167 else
3168 for (Value *V : E->Scalars) {
3169 auto *I = cast<Instruction>(V);
3170 LHSVL.push_back(I->getOperand(0));
3171 RHSVL.push_back(I->getOperand(1));
3172 }
3173
3174 setInsertPointAfterBundle(E->Scalars, VL0);
3175
3176 Value *LHS = vectorizeTree(LHSVL);
3177 Value *RHS = vectorizeTree(RHSVL);
3178
3179 if (E->VectorizedValue) {
3180 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3181 return E->VectorizedValue;
3182 }
3183
3184 Value *V = Builder.CreateBinOp(
3185 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3186 propagateIRFlags(V, E->Scalars, VL0);
3187 if (auto *I = dyn_cast<Instruction>(V))
3188 V = propagateMetadata(I, E->Scalars);
3189
3190 if (NeedToShuffleReuses) {
3191 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3192 E->ReuseShuffleIndices, "shuffle");
3193 }
3194 E->VectorizedValue = V;
3195 ++NumVectorInstructions;
3196
3197 return V;
3198 }
3199 case Instruction::Load: {
3200 // Loads are inserted at the head of the tree because we don't want to
3201 // sink them all the way down past store instructions.
3202 setInsertPointAfterBundle(E->Scalars, VL0);
3203
3204 LoadInst *LI = cast<LoadInst>(VL0);
3205 Type *ScalarLoadTy = LI->getType();
3206 unsigned AS = LI->getPointerAddressSpace();
3207
3208 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3209 VecTy->getPointerTo(AS));
3210
3211 // The pointer operand uses an in-tree scalar so we add the new BitCast to
3212 // ExternalUses list to make sure that an extract will be generated in the
3213 // future.
3214 Value *PO = LI->getPointerOperand();
3215 if (getTreeEntry(PO))
3216 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3217
3218 unsigned Alignment = LI->getAlignment();
3219 LI = Builder.CreateLoad(VecPtr);
3220 if (!Alignment) {
3221 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3222 }
3223 LI->setAlignment(Alignment);
3224 Value *V = propagateMetadata(LI, E->Scalars);
3225 if (NeedToShuffleReuses) {
3226 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3227 E->ReuseShuffleIndices, "shuffle");
3228 }
3229 E->VectorizedValue = V;
3230 ++NumVectorInstructions;
3231 return V;
3232 }
3233 case Instruction::Store: {
3234 StoreInst *SI = cast<StoreInst>(VL0);
3235 unsigned Alignment = SI->getAlignment();
3236 unsigned AS = SI->getPointerAddressSpace();
3237
3238 ValueList ScalarStoreValues;
3239 for (Value *V : E->Scalars)
3240 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3241
3242 setInsertPointAfterBundle(E->Scalars, VL0);
3243
3244 Value *VecValue = vectorizeTree(ScalarStoreValues);
3245 Value *ScalarPtr = SI->getPointerOperand();
3246 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3247 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3248
3249 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3250 // ExternalUses to make sure that an extract will be generated in the
3251 // future.
3252 if (getTreeEntry(ScalarPtr))
3253 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3254
3255 if (!Alignment)
3256 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3257
3258 S->setAlignment(Alignment);
3259 Value *V = propagateMetadata(S, E->Scalars);
3260 if (NeedToShuffleReuses) {
3261 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3262 E->ReuseShuffleIndices, "shuffle");
3263 }
3264 E->VectorizedValue = V;
3265 ++NumVectorInstructions;
3266 return V;
3267 }
3268 case Instruction::GetElementPtr: {
3269 setInsertPointAfterBundle(E->Scalars, VL0);
3270
3271 ValueList Op0VL;
3272 for (Value *V : E->Scalars)
3273 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3274
3275 Value *Op0 = vectorizeTree(Op0VL);
3276
3277 std::vector<Value *> OpVecs;
3278 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3279 ++j) {
3280 ValueList OpVL;
3281 for (Value *V : E->Scalars)
3282 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3283
3284 Value *OpVec = vectorizeTree(OpVL);
3285 OpVecs.push_back(OpVec);
3286 }
3287
3288 Value *V = Builder.CreateGEP(
3289 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3290 if (Instruction *I = dyn_cast<Instruction>(V))
3291 V = propagateMetadata(I, E->Scalars);
3292
3293 if (NeedToShuffleReuses) {
3294 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3295 E->ReuseShuffleIndices, "shuffle");
3296 }
3297 E->VectorizedValue = V;
3298 ++NumVectorInstructions;
3299
3300 return V;
3301 }
3302 case Instruction::Call: {
3303 CallInst *CI = cast<CallInst>(VL0);
8
Calling 'cast'
32
Returning from 'cast'
33
'CI' initialized here
3304 setInsertPointAfterBundle(E->Scalars, VL0);
3305 Function *FI;
3306 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3307 Value *ScalarArg = nullptr;
3308 if (CI && (FI = CI->getCalledFunction())) {
34
Assuming 'CI' is null
3309 IID = FI->getIntrinsicID();
3310 }
3311 std::vector<Value *> OpVecs;
3312 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
35
Called C++ object pointer is null
3313 ValueList OpVL;
3314 // ctlz,cttz and powi are special intrinsics whose second argument is
3315 // a scalar. This argument should not be vectorized.
3316 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3317 CallInst *CEI = cast<CallInst>(VL0);
3318 ScalarArg = CEI->getArgOperand(j);
3319 OpVecs.push_back(CEI->getArgOperand(j));
3320 continue;
3321 }
3322 for (Value *V : E->Scalars) {
3323 CallInst *CEI = cast<CallInst>(V);
3324 OpVL.push_back(CEI->getArgOperand(j));
3325 }
3326
3327 Value *OpVec = vectorizeTree(OpVL);
3328 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
3329 OpVecs.push_back(OpVec);
3330 }
3331
3332 Module *M = F->getParent();
3333 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3334 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3335 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3336 SmallVector<OperandBundleDef, 1> OpBundles;
3337 CI->getOperandBundlesAsDefs(OpBundles);
3338 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3339
3340 // The scalar argument uses an in-tree scalar so we add the new vectorized
3341 // call to ExternalUses list to make sure that an extract will be
3342 // generated in the future.
3343 if (ScalarArg && getTreeEntry(ScalarArg))
3344 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3345
3346 propagateIRFlags(V, E->Scalars, VL0);
3347 if (NeedToShuffleReuses) {
3348 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3349 E->ReuseShuffleIndices, "shuffle");
3350 }
3351 E->VectorizedValue = V;
3352 ++NumVectorInstructions;
3353 return V;
3354 }
3355 case Instruction::ShuffleVector: {
3356 ValueList LHSVL, RHSVL;
3357 assert(Instruction::isBinaryOp(S.Opcode) &&(static_cast <bool> (Instruction::isBinaryOp(S.Opcode) &&
"Invalid Shuffle Vector Operand") ? void (0) : __assert_fail
("Instruction::isBinaryOp(S.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3358, __extension__ __PRETTY_FUNCTION__))
3358 "Invalid Shuffle Vector Operand")(static_cast <bool> (Instruction::isBinaryOp(S.Opcode) &&
"Invalid Shuffle Vector Operand") ? void (0) : __assert_fail
("Instruction::isBinaryOp(S.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3358, __extension__ __PRETTY_FUNCTION__))
;
3359 reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3360 setInsertPointAfterBundle(E->Scalars, VL0);
3361
3362 Value *LHS = vectorizeTree(LHSVL);
3363 Value *RHS = vectorizeTree(RHSVL);
3364
3365 if (E->VectorizedValue) {
3366 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3367 return E->VectorizedValue;
3368 }
3369
3370 // Create a vector of LHS op1 RHS
3371 Value *V0 = Builder.CreateBinOp(
3372 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3373
3374 unsigned AltOpcode = getAltOpcode(S.Opcode);
3375 // Create a vector of LHS op2 RHS
3376 Value *V1 = Builder.CreateBinOp(
3377 static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3378
3379 // Create shuffle to take alternate operations from the vector.
3380 // Also, gather up odd and even scalar ops to propagate IR flags to
3381 // each vector operation.
3382 ValueList OddScalars, EvenScalars;
3383 unsigned e = E->Scalars.size();
3384 SmallVector<Constant *, 8> Mask(e);
3385 for (unsigned i = 0; i < e; ++i) {
3386 if (isOdd(i)) {
3387 Mask[i] = Builder.getInt32(e + i);
3388 OddScalars.push_back(E->Scalars[i]);
3389 } else {
3390 Mask[i] = Builder.getInt32(i);
3391 EvenScalars.push_back(E->Scalars[i]);
3392 }
3393 }
3394
3395 Value *ShuffleMask = ConstantVector::get(Mask);
3396 propagateIRFlags(V0, EvenScalars);
3397 propagateIRFlags(V1, OddScalars);
3398
3399 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3400 if (Instruction *I = dyn_cast<Instruction>(V))
3401 V = propagateMetadata(I, E->Scalars);
3402 if (NeedToShuffleReuses) {
3403 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3404 E->ReuseShuffleIndices, "shuffle");
3405 }
3406 E->VectorizedValue = V;
3407 ++NumVectorInstructions;
3408
3409 return V;
3410 }
3411 default:
3412 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3412)
;
3413 }
3414 return nullptr;
3415}
3416
3417Value *BoUpSLP::vectorizeTree() {
3418 ExtraValueToDebugLocsMap ExternallyUsedValues;
3419 return vectorizeTree(ExternallyUsedValues);
3420}
3421
3422Value *
3423BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3424 // All blocks must be scheduled before any instructions are inserted.
3425 for (auto &BSIter : BlocksSchedules) {
3426 scheduleBlock(BSIter.second.get());
3427 }
3428
3429 Builder.SetInsertPoint(&F->getEntryBlock().front());
3430 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3431
3432 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3433 // vectorized root. InstCombine will then rewrite the entire expression. We
3434 // sign extend the extracted values below.
3435 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3436 if (MinBWs.count(ScalarRoot)) {
3437 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3438 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3439 auto BundleWidth = VectorizableTree[0].Scalars.size();
3440 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3441 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3442 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3443 VectorizableTree[0].VectorizedValue = Trunc;
3444 }
3445
3446 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
3447
3448 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3449 // specified by ScalarType.
3450 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3451 if (!MinBWs.count(ScalarRoot))
3452 return Ex;
3453 if (MinBWs[ScalarRoot].second)
3454 return Builder.CreateSExt(Ex, ScalarType);
3455 return Builder.CreateZExt(Ex, ScalarType);
3456 };
3457
3458 // Extract all of the elements with the external uses.
3459 for (const auto &ExternalUse : ExternalUses) {
3460 Value *Scalar = ExternalUse.Scalar;
3461 llvm::User *User = ExternalUse.User;
3462
3463 // Skip users that we already RAUW. This happens when one instruction
3464 // has multiple uses of the same value.
3465 if (User && !is_contained(Scalar->users(), User))
3466 continue;
3467 TreeEntry *E = getTreeEntry(Scalar);
3468 assert(E && "Invalid scalar")(static_cast <bool> (E && "Invalid scalar") ? void
(0) : __assert_fail ("E && \"Invalid scalar\"", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3468, __extension__ __PRETTY_FUNCTION__))
;
3469 assert(!E->NeedToGather && "Extracting from a gather list")(static_cast <bool> (!E->NeedToGather && "Extracting from a gather list"
) ? void (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3469, __extension__ __PRETTY_FUNCTION__))
;
3470
3471 Value *Vec = E->VectorizedValue;
3472 assert(Vec && "Can't find vectorizable value")(static_cast <bool> (Vec && "Can't find vectorizable value"
) ? void (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3472, __extension__ __PRETTY_FUNCTION__))
;
3473
3474 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3475 // If User == nullptr, the Scalar is used as extra arg. Generate
3476 // ExtractElement instruction and update the record for this scalar in
3477 // ExternallyUsedValues.
3478 if (!User) {
3479 assert(ExternallyUsedValues.count(Scalar) &&(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3481, __extension__ __PRETTY_FUNCTION__))
3480 "Scalar with nullptr as an external user must be registered in "(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3481, __extension__ __PRETTY_FUNCTION__))
3481 "ExternallyUsedValues map")(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3481, __extension__ __PRETTY_FUNCTION__))
;
3482 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3483 Builder.SetInsertPoint(VecI->getParent(),
3484 std::next(VecI->getIterator()));
3485 } else {
3486 Builder.SetInsertPoint(&F->getEntryBlock().front());
3487 }
3488 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3489 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3490 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3491 auto &Locs = ExternallyUsedValues[Scalar];
3492 ExternallyUsedValues.insert({Ex, Locs});
3493 ExternallyUsedValues.erase(Scalar);
3494 continue;
3495 }
3496
3497 // Generate extracts for out-of-tree users.
3498 // Find the insertion point for the extractelement lane.
3499 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3500 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3501 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3502 if (PH->getIncomingValue(i) == Scalar) {
3503 TerminatorInst *IncomingTerminator =
3504 PH->getIncomingBlock(i)->getTerminator();
3505 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3506 Builder.SetInsertPoint(VecI->getParent(),
3507 std::next(VecI->getIterator()));
3508 } else {
3509 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3510 }
3511 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3512 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3513 CSEBlocks.insert(PH->getIncomingBlock(i));
3514 PH->setOperand(i, Ex);
3515 }
3516 }
3517 } else {
3518 Builder.SetInsertPoint(cast<Instruction>(User));
3519 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3520 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3521 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3522 User->replaceUsesOfWith(Scalar, Ex);
3523 }
3524 } else {
3525 Builder.SetInsertPoint(&F->getEntryBlock().front());
3526 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3527 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3528 CSEBlocks.insert(&F->getEntryBlock());
3529 User->replaceUsesOfWith(Scalar, Ex);
3530 }
3531
3532 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
3533 }
3534
3535 // For each vectorized value:
3536 for (TreeEntry &EIdx : VectorizableTree) {
3537 TreeEntry *Entry = &EIdx;
3538
3539 // No need to handle users of gathered values.
3540 if (Entry->NeedToGather)
3541 continue;
3542
3543 assert(Entry->VectorizedValue && "Can't find vectorizable value")(static_cast <bool> (Entry->VectorizedValue &&
"Can't find vectorizable value") ? void (0) : __assert_fail (
"Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3543, __extension__ __PRETTY_FUNCTION__))
;
3544
3545 // For each lane:
3546 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3547 Value *Scalar = Entry->Scalars[Lane];
3548
3549 Type *Ty = Scalar->getType();
3550 if (!Ty->isVoidTy()) {
3551#ifndef NDEBUG
3552 for (User *U : Scalar->users()) {
3553 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
3554
3555 // It is legal to replace users in the ignorelist by undef.
3556 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&(static_cast <bool> ((getTreeEntry(U) || is_contained(UserIgnoreList
, U)) && "Replacing out-of-tree value with undef") ? void
(0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3557, __extension__ __PRETTY_FUNCTION__))
3557 "Replacing out-of-tree value with undef")(static_cast <bool> ((getTreeEntry(U) || is_contained(UserIgnoreList
, U)) && "Replacing out-of-tree value with undef") ? void
(0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3557, __extension__ __PRETTY_FUNCTION__))
;
3558 }
3559#endif
3560 Value *Undef = UndefValue::get(Ty);
3561 Scalar->replaceAllUsesWith(Undef);
3562 }
3563 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
3564 eraseInstruction(cast<Instruction>(Scalar));
3565 }
3566 }
3567
3568 Builder.ClearInsertionPoint();
3569
3570 return VectorizableTree[0].VectorizedValue;
3571}
3572
3573void BoUpSLP::optimizeGatherSequence() {
3574 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
3575 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
3576 // LICM InsertElementInst sequences.
3577 for (Instruction *I : GatherSeq) {
3578 if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3579 continue;
3580
3581 // Check if this block is inside a loop.
3582 Loop *L = LI->getLoopFor(I->getParent());
3583 if (!L)
3584 continue;
3585
3586 // Check if it has a preheader.
3587 BasicBlock *PreHeader = L->getLoopPreheader();
3588 if (!PreHeader)
3589 continue;
3590
3591 // If the vector or the element that we insert into it are
3592 // instructions that are defined in this basic block then we can't
3593 // hoist this instruction.
3594 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3595 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3596 if (Op0 && L->contains(Op0))
3597 continue;
3598 if (Op1 && L->contains(Op1))
3599 continue;
3600
3601 // We can hoist this instruction. Move it to the pre-header.
3602 I->moveBefore(PreHeader->getTerminator());
3603 }
3604
3605 // Make a list of all reachable blocks in our CSE queue.
3606 SmallVector<const DomTreeNode *, 8> CSEWorkList;
3607 CSEWorkList.reserve(CSEBlocks.size());
3608 for (BasicBlock *BB : CSEBlocks)
3609 if (DomTreeNode *N = DT->getNode(BB)) {
3610 assert(DT->isReachableFromEntry(N))(static_cast <bool> (DT->isReachableFromEntry(N)) ? void
(0) : __assert_fail ("DT->isReachableFromEntry(N)", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3610, __extension__ __PRETTY_FUNCTION__))
;
3611 CSEWorkList.push_back(N);
3612 }
3613
3614 // Sort blocks by domination. This ensures we visit a block after all blocks
3615 // dominating it are visited.
3616 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3617 [this](const DomTreeNode *A, const DomTreeNode *B) {
3618 return DT->properlyDominates(A, B);
3619 });
3620
3621 // Perform O(N^2) search over the gather sequences and merge identical
3622 // instructions. TODO: We can further optimize this scan if we split the
3623 // instructions into different buckets based on the insert lane.
3624 SmallVector<Instruction *, 16> Visited;
3625 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3626 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(static_cast <bool> ((I == CSEWorkList.begin() || !DT->
dominates(*I, *std::prev(I))) && "Worklist not sorted properly!"
) ? void (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3627, __extension__ __PRETTY_FUNCTION__))
3627 "Worklist not sorted properly!")(static_cast <bool> ((I == CSEWorkList.begin() || !DT->
dominates(*I, *std::prev(I))) && "Worklist not sorted properly!"
) ? void (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3627, __extension__ __PRETTY_FUNCTION__))
;
3628 BasicBlock *BB = (*I)->getBlock();
3629 // For all instructions in blocks containing gather sequences:
3630 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3631 Instruction *In = &*it++;
3632 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3633 continue;
3634
3635 // Check if we can replace this instruction with any of the
3636 // visited instructions.
3637 for (Instruction *v : Visited) {
3638 if (In->isIdenticalTo(v) &&
3639 DT->dominates(v->getParent(), In->getParent())) {
3640 In->replaceAllUsesWith(v);
3641 eraseInstruction(In);
3642 In = nullptr;
3643 break;
3644 }
3645 }
3646 if (In) {
3647 assert(!is_contained(Visited, In))(static_cast <bool> (!is_contained(Visited, In)) ? void
(0) : __assert_fail ("!is_contained(Visited, In)", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3647, __extension__ __PRETTY_FUNCTION__))
;
3648 Visited.push_back(In);
3649 }
3650 }
3651 }
3652 CSEBlocks.clear();
3653 GatherSeq.clear();
3654}
3655
3656// Groups the instructions to a bundle (which is then a single scheduling entity)
3657// and schedules instructions until the bundle gets ready.
3658bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3659 BoUpSLP *SLP, Value *OpValue) {
3660 if (isa<PHINode>(OpValue))
3661 return true;
3662
3663 // Initialize the instruction bundle.
3664 Instruction *OldScheduleEnd = ScheduleEnd;
3665 ScheduleData *PrevInBundle = nullptr;
3666 ScheduleData *Bundle = nullptr;
3667 bool ReSchedule = false;
3668 DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *OpValue
<< "\n"; } } while (false)
;
3669
3670 // Make sure that the scheduling region contains all
3671 // instructions of the bundle.
3672 for (Value *V : VL) {
3673 if (!extendSchedulingRegion(V, OpValue))
3674 return false;
3675 }
3676
3677 for (Value *V : VL) {
3678 ScheduleData *BundleMember = getScheduleData(V);
3679 assert(BundleMember &&(static_cast <bool> (BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? void (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3680, __extension__ __PRETTY_FUNCTION__))
3680 "no ScheduleData for bundle member (maybe not in same basic block)")(static_cast <bool> (BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? void (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3680, __extension__ __PRETTY_FUNCTION__))
;
3681 if (BundleMember->IsScheduled) {
3682 // A bundle member was scheduled as single instruction before and now
3683 // needs to be scheduled as part of the bundle. We just get rid of the
3684 // existing schedule.
3685 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3686 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
3687 ReSchedule = true;
3688 }
3689 assert(BundleMember->isSchedulingEntity() &&(static_cast <bool> (BundleMember->isSchedulingEntity
() && "bundle member already part of other bundle") ?
void (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3690, __extension__ __PRETTY_FUNCTION__))
3690 "bundle member already part of other bundle")(static_cast <bool> (BundleMember->isSchedulingEntity
() && "bundle member already part of other bundle") ?
void (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3690, __extension__ __PRETTY_FUNCTION__))
;
3691 if (PrevInBundle) {
3692 PrevInBundle->NextInBundle = BundleMember;
3693 } else {
3694 Bundle = BundleMember;
3695 }
3696 BundleMember->UnscheduledDepsInBundle = 0;
3697 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3698
3699 // Group the instructions to a bundle.
3700 BundleMember->FirstInBundle = Bundle;
3701 PrevInBundle = BundleMember;
3702 }
3703 if (ScheduleEnd != OldScheduleEnd) {
3704 // The scheduling region got new instructions at the lower end (or it is a
3705 // new region for the first bundle). This makes it necessary to
3706 // recalculate all dependencies.
3707 // It is seldom that this needs to be done a second time after adding the
3708 // initial bundle to the region.
3709 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3710 doForAllOpcodes(I, [](ScheduleData *SD) {
3711 SD->clearDependencies();
3712 });
3713 }
3714 ReSchedule = true;
3715 }
3716 if (ReSchedule) {
3717 resetSchedule();
3718 initialFillReadyList(ReadyInsts);
3719 }
3720
3721 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
3722 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
3723
3724 calculateDependencies(Bundle, true, SLP);
3725
3726 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3727 // means that there are no cyclic dependencies and we can schedule it.
3728 // Note that's important that we don't "schedule" the bundle yet (see
3729 // cancelScheduling).
3730 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3731
3732 ScheduleData *pickedSD = ReadyInsts.back();
3733 ReadyInsts.pop_back();
3734
3735 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3736 schedule(pickedSD, ReadyInsts);
3737 }
3738 }
3739 if (!Bundle->isReady()) {
3740 cancelScheduling(VL, OpValue);
3741 return false;
3742 }
3743 return true;
3744}
3745
3746void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3747 Value *OpValue) {
3748 if (isa<PHINode>(OpValue))
3749 return;
3750
3751 ScheduleData *Bundle = getScheduleData(OpValue);
3752 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3753 assert(!Bundle->IsScheduled &&(static_cast <bool> (!Bundle->IsScheduled &&
"Can't cancel bundle which is already scheduled") ? void (0)
: __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3754, __extension__ __PRETTY_FUNCTION__))
3754 "Can't cancel bundle which is already scheduled")(static_cast <bool> (!Bundle->IsScheduled &&
"Can't cancel bundle which is already scheduled") ? void (0)
: __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3754, __extension__ __PRETTY_FUNCTION__))
;
3755 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&(static_cast <bool> (Bundle->isSchedulingEntity() &&
Bundle->isPartOfBundle() && "tried to unbundle something which is not a bundle"
) ? void (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3756, __extension__ __PRETTY_FUNCTION__))
3756 "tried to unbundle something which is not a bundle")(static_cast <bool> (Bundle->isSchedulingEntity() &&
Bundle->isPartOfBundle() && "tried to unbundle something which is not a bundle"
) ? void (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3756, __extension__ __PRETTY_FUNCTION__))
;
3757
3758 // Un-bundle: make single instructions out of the bundle.
3759 ScheduleData *BundleMember = Bundle;
3760 while (BundleMember) {
3761 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")(static_cast <bool> (BundleMember->FirstInBundle == Bundle
&& "corrupt bundle links") ? void (0) : __assert_fail
("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3761, __extension__ __PRETTY_FUNCTION__))
;
3762 BundleMember->FirstInBundle = BundleMember;
3763 ScheduleData *Next = BundleMember->NextInBundle;
3764 BundleMember->NextInBundle = nullptr;
3765 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3766 if (BundleMember->UnscheduledDepsInBundle == 0) {
3767 ReadyInsts.insert(BundleMember);
3768 }
3769 BundleMember = Next;
3770 }
3771}
3772
3773BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3774 // Allocate a new ScheduleData for the instruction.
3775 if (ChunkPos >= ChunkSize) {
3776 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3777 ChunkPos = 0;
3778 }
3779 return &(ScheduleDataChunks.back()[ChunkPos++]);
3780}
3781
3782bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3783 Value *OpValue) {
3784 if (getScheduleData(V, isOneOf(OpValue, V)))
3785 return true;
3786 Instruction *I = dyn_cast<Instruction>(V);
3787 assert(I && "bundle member must be an instruction")(static_cast <bool> (I && "bundle member must be an instruction"
) ? void (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3787, __extension__ __PRETTY_FUNCTION__))
;
3788 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")(static_cast <bool> (!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? void (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3788, __extension__ __PRETTY_FUNCTION__))
;
3789 auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3790 ScheduleData *ISD = getScheduleData(I);
3791 if (!ISD)
3792 return false;
3793 assert(isInSchedulingRegion(ISD) &&(static_cast <bool> (isInSchedulingRegion(ISD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3794, __extension__ __PRETTY_FUNCTION__))
3794 "ScheduleData not in scheduling region")(static_cast <bool> (isInSchedulingRegion(ISD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3794, __extension__ __PRETTY_FUNCTION__))
;
3795 ScheduleData *SD = allocateScheduleDataChunks();
3796 SD->Inst = I;
3797 SD->init(SchedulingRegionID, OpValue);
3798 ExtraScheduleDataMap[I][OpValue] = SD;
3799 return true;
3800 };
3801 if (CheckSheduleForI(I))
3802 return true;
3803 if (!ScheduleStart) {
3804 // It's the first instruction in the new region.
3805 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3806 ScheduleStart = I;
3807 ScheduleEnd = I->getNextNode();
3808 if (isOneOf(OpValue, I) != I)
3809 CheckSheduleForI(I);
3810 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")(static_cast <bool> (ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? void (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3810, __extension__ __PRETTY_FUNCTION__))
;
3811 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3812 return true;
3813 }
3814 // Search up and down at the same time, because we don't know if the new
3815 // instruction is above or below the existing scheduling region.
3816 BasicBlock::reverse_iterator UpIter =
3817 ++ScheduleStart->getIterator().getReverse();
3818 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3819 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3820 BasicBlock::iterator LowerEnd = BB->end();
3821 while (true) {
3822 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3823 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3824 return false;
3825 }
3826
3827 if (UpIter != UpperEnd) {
3828 if (&*UpIter == I) {
3829 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3830 ScheduleStart = I;
3831 if (isOneOf(OpValue, I) != I)
3832 CheckSheduleForI(I);
3833 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3834 return true;
3835 }
3836 UpIter++;
3837 }
3838 if (DownIter != LowerEnd) {
3839 if (&*DownIter == I) {
3840 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3841 nullptr);
3842 ScheduleEnd = I->getNextNode();
3843 if (isOneOf(OpValue, I) != I)
3844 CheckSheduleForI(I);
3845 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")(static_cast <bool> (ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? void (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3845, __extension__ __PRETTY_FUNCTION__))
;
3846 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
3847 return true;
3848 }
3849 DownIter++;
3850 }
3851 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(static_cast <bool> ((UpIter != UpperEnd || DownIter !=
LowerEnd) && "instruction not found in block") ? void
(0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3852, __extension__ __PRETTY_FUNCTION__))
3852 "instruction not found in block")(static_cast <bool> ((UpIter != UpperEnd || DownIter !=
LowerEnd) && "instruction not found in block") ? void
(0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3852, __extension__ __PRETTY_FUNCTION__))
;
3853 }
3854 return true;
3855}
3856
3857void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3858 Instruction *ToI,
3859 ScheduleData *PrevLoadStore,
3860 ScheduleData *NextLoadStore) {
3861 ScheduleData *CurrentLoadStore = PrevLoadStore;
3862 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3863 ScheduleData *SD = ScheduleDataMap[I];
3864 if (!SD) {
3865 SD = allocateScheduleDataChunks();
3866 ScheduleDataMap[I] = SD;
3867 SD->Inst = I;
3868 }
3869 assert(!isInSchedulingRegion(SD) &&(static_cast <bool> (!isInSchedulingRegion(SD) &&
"new ScheduleData already in scheduling region") ? void (0) :
__assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3870, __extension__ __PRETTY_FUNCTION__))
3870 "new ScheduleData already in scheduling region")(static_cast <bool> (!isInSchedulingRegion(SD) &&
"new ScheduleData already in scheduling region") ? void (0) :
__assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3870, __extension__ __PRETTY_FUNCTION__))
;
3871 SD->init(SchedulingRegionID, I);
3872
3873 if (I->mayReadOrWriteMemory() &&
3874 (!isa<IntrinsicInst>(I) ||
3875 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
3876 // Update the linked list of memory accessing instructions.
3877 if (CurrentLoadStore) {
3878 CurrentLoadStore->NextLoadStore = SD;
3879 } else {
3880 FirstLoadStoreInRegion = SD;
3881 }
3882 CurrentLoadStore = SD;
3883 }
3884 }
3885 if (NextLoadStore) {
3886 if (CurrentLoadStore)
3887 CurrentLoadStore->NextLoadStore = NextLoadStore;
3888 } else {
3889 LastLoadStoreInRegion = CurrentLoadStore;
3890 }
3891}
3892
3893void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3894 bool InsertInReadyList,
3895 BoUpSLP *SLP) {
3896 assert(SD->isSchedulingEntity())(static_cast <bool> (SD->isSchedulingEntity()) ? void
(0) : __assert_fail ("SD->isSchedulingEntity()", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3896, __extension__ __PRETTY_FUNCTION__))
;
3897
3898 SmallVector<ScheduleData *, 10> WorkList;
3899 WorkList.push_back(SD);
3900
3901 while (!WorkList.empty()) {
3902 ScheduleData *SD = WorkList.back();
3903 WorkList.pop_back();
3904
3905 ScheduleData *BundleMember = SD;
3906 while (BundleMember) {
3907 assert(isInSchedulingRegion(BundleMember))(static_cast <bool> (isInSchedulingRegion(BundleMember)
) ? void (0) : __assert_fail ("isInSchedulingRegion(BundleMember)"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3907, __extension__ __PRETTY_FUNCTION__))
;
3908 if (!BundleMember->hasValidDependencies()) {
3909
3910 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
3911 BundleMember->Dependencies = 0;
3912 BundleMember->resetUnscheduledDeps();
3913
3914 // Handle def-use chain dependencies.
3915 if (BundleMember->OpValue != BundleMember->Inst) {
3916 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
3917 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3918 BundleMember->Dependencies++;
3919 ScheduleData *DestBundle = UseSD->FirstInBundle;
3920 if (!DestBundle->IsScheduled)
3921 BundleMember->incrementUnscheduledDeps(1);
3922 if (!DestBundle->hasValidDependencies())
3923 WorkList.push_back(DestBundle);
3924 }
3925 } else {
3926 for (User *U : BundleMember->Inst->users()) {
3927 if (isa<Instruction>(U)) {
3928 ScheduleData *UseSD = getScheduleData(U);
3929 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3930 BundleMember->Dependencies++;
3931 ScheduleData *DestBundle = UseSD->FirstInBundle;
3932 if (!DestBundle->IsScheduled)
3933 BundleMember->incrementUnscheduledDeps(1);
3934 if (!DestBundle->hasValidDependencies())
3935 WorkList.push_back(DestBundle);
3936 }
3937 } else {
3938 // I'm not sure if this can ever happen. But we need to be safe.
3939 // This lets the instruction/bundle never be scheduled and
3940 // eventually disable vectorization.
3941 BundleMember->Dependencies++;
3942 BundleMember->incrementUnscheduledDeps(1);
3943 }
3944 }
3945 }
3946
3947 // Handle the memory dependencies.
3948 ScheduleData *DepDest = BundleMember->NextLoadStore;
3949 if (DepDest) {
3950 Instruction *SrcInst = BundleMember->Inst;
3951 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3952 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3953 unsigned numAliased = 0;
3954 unsigned DistToSrc = 1;
3955
3956 while (DepDest) {
3957 assert(isInSchedulingRegion(DepDest))(static_cast <bool> (isInSchedulingRegion(DepDest)) ? void
(0) : __assert_fail ("isInSchedulingRegion(DepDest)", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3957, __extension__ __PRETTY_FUNCTION__))
;
3958
3959 // We have two limits to reduce the complexity:
3960 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3961 // SLP->isAliased (which is the expensive part in this loop).
3962 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3963 // the whole loop (even if the loop is fast, it's quadratic).
3964 // It's important for the loop break condition (see below) to
3965 // check this limit even between two read-only instructions.
3966 if (DistToSrc >= MaxMemDepDistance ||
3967 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3968 (numAliased >= AliasedCheckLimit ||
3969 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3970
3971 // We increment the counter only if the locations are aliased
3972 // (instead of counting all alias checks). This gives a better
3973 // balance between reduced runtime and accurate dependencies.
3974 numAliased++;
3975
3976 DepDest->MemoryDependencies.push_back(BundleMember);
3977 BundleMember->Dependencies++;
3978 ScheduleData *DestBundle = DepDest->FirstInBundle;
3979 if (!DestBundle->IsScheduled) {
3980 BundleMember->incrementUnscheduledDeps(1);
3981 }
3982 if (!DestBundle->hasValidDependencies()) {
3983 WorkList.push_back(DestBundle);
3984 }
3985 }
3986 DepDest = DepDest->NextLoadStore;
3987
3988 // Example, explaining the loop break condition: Let's assume our
3989 // starting instruction is i0 and MaxMemDepDistance = 3.
3990 //
3991 // +--------v--v--v
3992 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3993 // +--------^--^--^
3994 //
3995 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3996 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3997 // Previously we already added dependencies from i3 to i6,i7,i8
3998 // (because of MaxMemDepDistance). As we added a dependency from
3999 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4000 // and we can abort this loop at i6.
4001 if (DistToSrc >= 2 * MaxMemDepDistance)
4002 break;
4003 DistToSrc++;
4004 }
4005 }
4006 }
4007 BundleMember = BundleMember->NextInBundle;
4008 }
4009 if (InsertInReadyList && SD->isReady()) {
4010 ReadyInsts.push_back(SD);
4011 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
4012 }
4013 }
4014}
4015
4016void BoUpSLP::BlockScheduling::resetSchedule() {
4017 assert(ScheduleStart &&(static_cast <bool> (ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? void (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4018, __extension__ __PRETTY_FUNCTION__))
4018 "tried to reset schedule on block which has not been scheduled")(static_cast <bool> (ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? void (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4018, __extension__ __PRETTY_FUNCTION__))
;
4019 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4020 doForAllOpcodes(I, [&](ScheduleData *SD) {
4021 assert(isInSchedulingRegion(SD) &&(static_cast <bool> (isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4022, __extension__ __PRETTY_FUNCTION__))
4022 "ScheduleData not in scheduling region")(static_cast <bool> (isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4022, __extension__ __PRETTY_FUNCTION__))
;
4023 SD->IsScheduled = false;
4024 SD->resetUnscheduledDeps();
4025 });
4026 }
4027 ReadyInsts.clear();
4028}
4029
4030void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4031 if (!BS->ScheduleStart)
4032 return;
4033
4034 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
4035
4036 BS->resetSchedule();
4037
4038 // For the real scheduling we use a more sophisticated ready-list: it is
4039 // sorted by the original instruction location. This lets the final schedule
4040 // be as close as possible to the original instruction order.
4041 struct ScheduleDataCompare {
4042 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4043 return SD2->SchedulingPriority < SD1->SchedulingPriority;
4044 }
4045 };
4046 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4047
4048 // Ensure that all dependency data is updated and fill the ready-list with
4049 // initial instructions.
4050 int Idx = 0;
4051 int NumToSchedule = 0;
4052 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4053 I = I->getNextNode()) {
4054 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4055 assert(SD->isPartOfBundle() ==(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4057, __extension__ __PRETTY_FUNCTION__))
4056 (getTreeEntry(SD->Inst) != nullptr) &&(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4057, __extension__ __PRETTY_FUNCTION__))
4057 "scheduler and vectorizer bundle mismatch")(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4057, __extension__ __PRETTY_FUNCTION__))
;
4058 SD->FirstInBundle->SchedulingPriority = Idx++;
4059 if (SD->isSchedulingEntity()) {
4060 BS->calculateDependencies(SD, false, this);
4061 NumToSchedule++;
4062 }
4063 });
4064 }
4065 BS->initialFillReadyList(ReadyInsts);
4066
4067 Instruction *LastScheduledInst = BS->ScheduleEnd;
4068
4069 // Do the "real" scheduling.
4070 while (!ReadyInsts.empty()) {
4071 ScheduleData *picked = *ReadyInsts.begin();
4072 ReadyInsts.erase(ReadyInsts.begin());
4073
4074 // Move the scheduled instruction(s) to their dedicated places, if not
4075 // there yet.
4076 ScheduleData *BundleMember = picked;
4077 while (BundleMember) {
4078 Instruction *pickedInst = BundleMember->Inst;
4079 if (LastScheduledInst->getNextNode() != pickedInst) {
4080 BS->BB->getInstList().remove(pickedInst);
4081 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4082 pickedInst);
4083 }
4084 LastScheduledInst = pickedInst;
4085 BundleMember = BundleMember->NextInBundle;
4086 }
4087
4088 BS->schedule(picked, ReadyInsts);
4089 NumToSchedule--;
4090 }
4091 assert(NumToSchedule == 0 && "could not schedule all instructions")(static_cast <bool> (NumToSchedule == 0 && "could not schedule all instructions"
) ? void (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4091, __extension__ __PRETTY_FUNCTION__))
;
4092
4093 // Avoid duplicate scheduling of the block.
4094 BS->ScheduleStart = nullptr;
4095}
4096
4097unsigned BoUpSLP::getVectorElementSize(Value *V) {
4098 // If V is a store, just return the width of the stored value without
4099 // traversing the expression tree. This is the common case.
4100 if (auto *Store = dyn_cast<StoreInst>(V))
4101 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4102
4103 // If V is not a store, we can traverse the expression tree to find loads
4104 // that feed it. The type of the loaded value may indicate a more suitable
4105 // width than V's type. We want to base the vector element size on the width
4106 // of memory operations where possible.
4107 SmallVector<Instruction *, 16> Worklist;
4108 SmallPtrSet<Instruction *, 16> Visited;
4109 if (auto *I = dyn_cast<Instruction>(V))
4110 Worklist.push_back(I);
4111
4112 // Traverse the expression tree in bottom-up order looking for loads. If we
4113 // encounter an instruciton we don't yet handle, we give up.
4114 auto MaxWidth = 0u;
4115 auto FoundUnknownInst = false;
4116 while (!Worklist.empty() && !FoundUnknownInst) {
4117 auto *I = Worklist.pop_back_val();
4118 Visited.insert(I);
4119
4120 // We should only be looking at scalar instructions here. If the current
4121 // instruction has a vector type, give up.
4122 auto *Ty = I->getType();
4123 if (isa<VectorType>(Ty))
4124 FoundUnknownInst = true;
4125
4126 // If the current instruction is a load, update MaxWidth to reflect the
4127 // width of the loaded value.
4128 else if (isa<LoadInst>(I))
4129 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4130
4131 // Otherwise, we need to visit the operands of the instruction. We only
4132 // handle the interesting cases from buildTree here. If an operand is an
4133 // instruction we haven't yet visited, we add it to the worklist.
4134 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4135 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4136 for (Use &U : I->operands())
4137 if (auto *J = dyn_cast<Instruction>(U.get()))
4138 if (!Visited.count(J))
4139 Worklist.push_back(J);
4140 }
4141
4142 // If we don't yet handle the instruction, give up.
4143 else
4144 FoundUnknownInst = true;
4145 }
4146
4147 // If we didn't encounter a memory access in the expression tree, or if we
4148 // gave up for some reason, just return the width of V.
4149 if (!MaxWidth || FoundUnknownInst)
4150 return DL->getTypeSizeInBits(V->getType());
4151
4152 // Otherwise, return the maximum width we found.
4153 return MaxWidth;
4154}
4155
4156// Determine if a value V in a vectorizable expression Expr can be demoted to a
4157// smaller type with a truncation. We collect the values that will be demoted
4158// in ToDemote and additional roots that require investigating in Roots.
4159static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
4160 SmallVectorImpl<Value *> &ToDemote,
4161 SmallVectorImpl<Value *> &Roots) {
4162 // We can always demote constants.
4163 if (isa<Constant>(V)) {
4164 ToDemote.push_back(V);
4165 return true;
4166 }
4167
4168 // If the value is not an instruction in the expression with only one use, it
4169 // cannot be demoted.
4170 auto *I = dyn_cast<Instruction>(V);
4171 if (!I || !I->hasOneUse() || !Expr.count(I))
4172 return false;
4173
4174 switch (I->getOpcode()) {
4175
4176 // We can always demote truncations and extensions. Since truncations can
4177 // seed additional demotion, we save the truncated value.
4178 case Instruction::Trunc:
4179 Roots.push_back(I->getOperand(0));
4180 break;
4181 case Instruction::ZExt:
4182 case Instruction::SExt:
4183 break;
4184
4185 // We can demote certain binary operations if we can demote both of their
4186 // operands.
4187 case Instruction::Add:
4188 case Instruction::Sub:
4189 case Instruction::Mul:
4190 case Instruction::And:
4191 case Instruction::Or:
4192 case Instruction::Xor:
4193 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4194 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4195 return false;
4196 break;
4197
4198 // We can demote selects if we can demote their true and false values.
4199 case Instruction::Select: {
4200 SelectInst *SI = cast<SelectInst>(I);
4201 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4202 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4203 return false;
4204 break;
4205 }
4206
4207 // We can demote phis if we can demote all their incoming operands. Note that
4208 // we don't need to worry about cycles since we ensure single use above.
4209 case Instruction::PHI: {
4210 PHINode *PN = cast<PHINode>(I);
4211 for (Value *IncValue : PN->incoming_values())
4212 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4213 return false;
4214 break;
4215 }
4216
4217 // Otherwise, conservatively give up.
4218 default:
4219 return false;
4220 }
4221
4222 // Record the value that we can demote.
4223 ToDemote.push_back(V);
4224 return true;
4225}
4226
4227void BoUpSLP::computeMinimumValueSizes() {
4228 // If there are no external uses, the expression tree must be rooted by a
4229 // store. We can't demote in-memory values, so there is nothing to do here.
4230 if (ExternalUses.empty())
4231 return;
4232
4233 // We only attempt to truncate integer expressions.
4234 auto &TreeRoot = VectorizableTree[0].Scalars;
4235 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4236 if (!TreeRootIT)
4237 return;
4238
4239 // If the expression is not rooted by a store, these roots should have
4240 // external uses. We will rely on InstCombine to rewrite the expression in
4241 // the narrower type. However, InstCombine only rewrites single-use values.
4242 // This means that if a tree entry other than a root is used externally, it
4243 // must have multiple uses and InstCombine will not rewrite it. The code
4244 // below ensures that only the roots are used externally.
4245 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4246 for (auto &EU : ExternalUses)
4247 if (!Expr.erase(EU.Scalar))
4248 return;
4249 if (!Expr.empty())
4250 return;
4251
4252 // Collect the scalar values of the vectorizable expression. We will use this
4253 // context to determine which values can be demoted. If we see a truncation,
4254 // we mark it as seeding another demotion.
4255 for (auto &Entry : VectorizableTree)
4256 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4257
4258 // Ensure the roots of the vectorizable tree don't form a cycle. They must
4259 // have a single external user that is not in the vectorizable tree.
4260 for (auto *Root : TreeRoot)
4261 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4262 return;
4263
4264 // Conservatively determine if we can actually truncate the roots of the
4265 // expression. Collect the values that can be demoted in ToDemote and
4266 // additional roots that require investigating in Roots.
4267 SmallVector<Value *, 32> ToDemote;
4268 SmallVector<Value *, 4> Roots;
4269 for (auto *Root : TreeRoot) {
4270 // Do not include top zext/sext/trunc operations to those to be demoted, it
4271 // produces noise cast<vect>, trunc <vect>, exctract <vect>, cast <extract>
4272 // sequence.
4273 if (isa<Constant>(Root))
4274 continue;
4275 auto *I = dyn_cast<Instruction>(Root);
4276 if (!I || !I->hasOneUse() || !Expr.count(I))
4277 return;
4278 if (isa<ZExtInst>(I) || isa<SExtInst>(I))
4279 continue;
4280 if (auto *TI = dyn_cast<TruncInst>(I)) {
4281 Roots.push_back(TI->getOperand(0));
4282 continue;
4283 }
4284 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4285 return;
4286 }
4287
4288 // The maximum bit width required to represent all the values that can be
4289 // demoted without loss of precision. It would be safe to truncate the roots
4290 // of the expression to this width.
4291 auto MaxBitWidth = 8u;
4292
4293 // We first check if all the bits of the roots are demanded. If they're not,
4294 // we can truncate the roots to this narrower type.
4295 for (auto *Root : TreeRoot) {
4296 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4297 MaxBitWidth = std::max<unsigned>(
4298 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4299 }
4300
4301 // True if the roots can be zero-extended back to their original type, rather
4302 // than sign-extended. We know that if the leading bits are not demanded, we
4303 // can safely zero-extend. So we initialize IsKnownPositive to True.
4304 bool IsKnownPositive = true;
4305
4306 // If all the bits of the roots are demanded, we can try a little harder to
4307 // compute a narrower type. This can happen, for example, if the roots are
4308 // getelementptr indices. InstCombine promotes these indices to the pointer
4309 // width. Thus, all their bits are technically demanded even though the
4310 // address computation might be vectorized in a smaller type.
4311 //
4312 // We start by looking at each entry that can be demoted. We compute the
4313 // maximum bit width required to store the scalar by using ValueTracking to
4314 // compute the number of high-order bits we can truncate.
4315 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
4316 MaxBitWidth = 8u;
4317
4318 // Determine if the sign bit of all the roots is known to be zero. If not,
4319 // IsKnownPositive is set to False.
4320 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4321 KnownBits Known = computeKnownBits(R, *DL);
4322 return Known.isNonNegative();
4323 });
4324
4325 // Determine the maximum number of bits required to store the scalar
4326 // values.
4327 for (auto *Scalar : ToDemote) {
4328 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4329 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4330 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4331 }
4332
4333 // If we can't prove that the sign bit is zero, we must add one to the
4334 // maximum bit width to account for the unknown sign bit. This preserves
4335 // the existing sign bit so we can safely sign-extend the root back to the
4336 // original type. Otherwise, if we know the sign bit is zero, we will
4337 // zero-extend the root instead.
4338 //
4339 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4340 // one to the maximum bit width will yield a larger-than-necessary
4341 // type. In general, we need to add an extra bit only if we can't
4342 // prove that the upper bit of the original type is equal to the
4343 // upper bit of the proposed smaller type. If these two bits are the
4344 // same (either zero or one) we know that sign-extending from the
4345 // smaller type will result in the same value. Here, since we can't
4346 // yet prove this, we are just making the proposed smaller type
4347 // larger to ensure correctness.
4348 if (!IsKnownPositive)
4349 ++MaxBitWidth;
4350 }
4351
4352 // Round MaxBitWidth up to the next power-of-two.
4353 if (!isPowerOf2_64(MaxBitWidth))
4354 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4355
4356 // If the maximum bit width we compute is less than the with of the roots'
4357 // type, we can proceed with the narrowing. Otherwise, do nothing.
4358 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4359 return;
4360
4361 // If we can truncate the root, we must collect additional values that might
4362 // be demoted as a result. That is, those seeded by truncations we will
4363 // modify.
4364 while (!Roots.empty())
4365 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4366
4367 // Finally, map the values we can demote to the maximum bit with we computed.
4368 for (auto *Scalar : ToDemote)
4369 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4370}
4371
4372namespace {
4373
4374/// The SLPVectorizer Pass.
4375struct SLPVectorizer : public FunctionPass {
4376 SLPVectorizerPass Impl;
4377
4378 /// Pass identification, replacement for typeid
4379 static char ID;
4380
4381 explicit SLPVectorizer() : FunctionPass(ID) {
4382 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4383 }
4384
4385 bool doInitialization(Module &M) override {
4386 return false;
4387 }
4388
4389 bool runOnFunction(Function &F) override {
4390 if (skipFunction(F))
4391 return false;
4392
4393 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4394 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4395 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4396 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4397 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4398 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4399 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4400 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4401 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4402 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4403
4404 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4405 }
4406
4407 void getAnalysisUsage(AnalysisUsage &AU) const override {
4408 FunctionPass::getAnalysisUsage(AU);
4409 AU.addRequired<AssumptionCacheTracker>();
4410 AU.addRequired<ScalarEvolutionWrapperPass>();
4411 AU.addRequired<AAResultsWrapperPass>();
4412 AU.addRequired<TargetTransformInfoWrapperPass>();
4413 AU.addRequired<LoopInfoWrapperPass>();
4414 AU.addRequired<DominatorTreeWrapperPass>();
4415 AU.addRequired<DemandedBitsWrapperPass>();
4416 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4417 AU.addPreserved<LoopInfoWrapperPass>();
4418 AU.addPreserved<DominatorTreeWrapperPass>();
4419 AU.addPreserved<AAResultsWrapperPass>();
4420 AU.addPreserved<GlobalsAAWrapperPass>();
4421 AU.setPreservesCFG();
4422 }
4423};
4424
4425} // end anonymous namespace
4426
4427PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4428 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4429 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4430 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4431 auto *AA = &AM.getResult<AAManager>(F);
4432 auto *LI = &AM.getResult<LoopAnalysis>(F);
4433 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4434 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4435 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4436 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4437
4438 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4439 if (!Changed)
4440 return PreservedAnalyses::all();
4441
4442 PreservedAnalyses PA;
4443 PA.preserveSet<CFGAnalyses>();
4444 PA.preserve<AAManager>();
4445 PA.preserve<GlobalsAA>();
4446 return PA;
4447}
4448
4449bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4450 TargetTransformInfo *TTI_,
4451 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4452 LoopInfo *LI_, DominatorTree *DT_,
4453 AssumptionCache *AC_, DemandedBits *DB_,
4454 OptimizationRemarkEmitter *ORE_) {
4455 SE = SE_;
4456 TTI = TTI_;
4457 TLI = TLI_;
4458 AA = AA_;
4459 LI = LI_;
4460 DT = DT_;
4461 AC = AC_;
4462 DB = DB_;
4463 DL = &F.getParent()->getDataLayout();
4464
4465 Stores.clear();
4466 GEPs.clear();
4467 bool Changed = false;
4468
4469 // If the target claims to have no vector registers don't attempt
4470 // vectorization.
4471 if (!TTI->getNumberOfRegisters(true))
4472 return false;
4473
4474 // Don't vectorize when the attribute NoImplicitFloat is used.
4475 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4476 return false;
4477
4478 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
4479
4480 // Use the bottom up slp vectorizer to construct chains that start with
4481 // store instructions.
4482 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4483
4484 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4485 // delete instructions.
4486
4487 // Scan the blocks in the function in post order.
4488 for (auto BB : post_order(&F.getEntryBlock())) {
4489 collectSeedInstructions(BB);
4490
4491 // Vectorize trees that end at stores.
4492 if (!Stores.empty()) {
4493 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
4494 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
4495 Changed |= vectorizeStoreChains(R);
4496 }
4497
4498 // Vectorize trees that end at reductions.
4499 Changed |= vectorizeChainsInBlock(BB, R);
4500
4501 // Vectorize the index computations of getelementptr instructions. This
4502 // is primarily intended to catch gather-like idioms ending at
4503 // non-consecutive loads.
4504 if (!GEPs.empty()) {
4505 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
4506 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
4507 Changed |= vectorizeGEPIndices(BB, R);
4508 }
4509 }
4510
4511 if (Changed) {
4512 R.optimizeGatherSequence();
4513 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
4514 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
4515 }
4516 return Changed;
4517}
4518
4519/// \brief Check that the Values in the slice in VL array are still existent in
4520/// the WeakTrackingVH array.
4521/// Vectorization of part of the VL array may cause later values in the VL array
4522/// to become invalid. We track when this has happened in the WeakTrackingVH
4523/// array.
4524static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4525 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4526 unsigned SliceSize) {
4527 VL = VL.slice(SliceBegin, SliceSize);
4528 VH = VH.slice(SliceBegin, SliceSize);
4529 return !std::equal(VL.begin(), VL.end(), VH.begin());
4530}
4531
4532bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4533 unsigned VecRegSize) {
4534 const unsigned ChainLen = Chain.size();
4535 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
4536 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
4537 const unsigned Sz = R.getVectorElementSize(Chain[0]);
4538 const unsigned VF = VecRegSize / Sz;
4539
4540 if (!isPowerOf2_32(Sz) || VF < 2)
4541 return false;
4542
4543 // Keep track of values that were deleted by vectorizing in the loop below.
4544 const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4545
4546 bool Changed = false;
4547 // Look for profitable vectorizable trees at all offsets, starting at zero.
4548 for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4549
4550 // Check that a previous iteration of this loop did not delete the Value.
4551 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4552 continue;
4553
4554 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
4555 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
4556 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4557
4558 R.buildTree(Operands);
4559 if (R.isTreeTinyAndNotFullyVectorizable())
4560 continue;
4561
4562 R.computeMinimumValueSizes();
4563
4564 int Cost = R.getTreeCost();
4565
4566 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
4567 if (Cost < -SLPCostThreshold) {
4568 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
4569
4570 using namespace ore;
4571
4572 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "StoresVectorized",
4573 cast<StoreInst>(Chain[i]))
4574 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4575 << " and with tree size "
4576 << NV("TreeSize", R.getTreeSize()));
4577
4578 R.vectorizeTree();
4579
4580 // Move to the next bundle.
4581 i += VF - 1;
4582 Changed = true;
4583 }
4584 }
4585
4586 return Changed;
4587}
4588
4589bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4590 BoUpSLP &R) {
4591 SetVector<StoreInst *> Heads;
4592 SmallDenseSet<StoreInst *> Tails;
4593 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4594
4595 // We may run into multiple chains that merge into a single chain. We mark the
4596 // stores that we vectorized so that we don't visit the same store twice.
4597 BoUpSLP::ValueSet VectorizedStores;
4598 bool Changed = false;
4599
4600 // Do a quadratic search on all of the given stores in reverse order and find
4601 // all of the pairs of stores that follow each other.
4602 SmallVector<unsigned, 16> IndexQueue;
4603 unsigned E = Stores.size();
4604 IndexQueue.resize(E - 1);
4605 for (unsigned I = E; I > 0; --I) {
4606 unsigned Idx = I - 1;
4607 // If a store has multiple consecutive store candidates, search Stores
4608 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4609 // This is because usually pairing with immediate succeeding or preceding
4610 // candidate create the best chance to find slp vectorization opportunity.
4611 unsigned Offset = 1;
4612 unsigned Cnt = 0;
4613 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4614 if (Idx >= Offset) {
4615 IndexQueue[Cnt] = Idx - Offset;
4616 ++Cnt;
4617 }
4618 if (Idx + Offset < E) {
4619 IndexQueue[Cnt] = Idx + Offset;
4620 ++Cnt;
4621 }
4622 }
4623
4624 for (auto K : IndexQueue) {
4625 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4626 Tails.insert(Stores[Idx]);
4627 Heads.insert(Stores[K]);
4628 ConsecutiveChain[Stores[K]] = Stores[Idx];
4629 break;
4630 }
4631 }
4632 }
4633
4634 // For stores that start but don't end a link in the chain:
4635 for (auto *SI : llvm::reverse(Heads)) {
4636 if (Tails.count(SI))
4637 continue;
4638
4639 // We found a store instr that starts a chain. Now follow the chain and try
4640 // to vectorize it.
4641 BoUpSLP::ValueList Operands;
4642 StoreInst *I = SI;
4643 // Collect the chain into a list.
4644 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4645 Operands.push_back(I);
4646 // Move to the next value in the chain.
4647 I = ConsecutiveChain[I];
4648 }
4649
4650 // FIXME: Is division-by-2 the correct step? Should we assert that the
4651 // register size is a power-of-2?
4652 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4653 Size /= 2) {
4654 if (vectorizeStoreChain(Operands, R, Size)) {
4655 // Mark the vectorized stores so that we don't vectorize them again.
4656 VectorizedStores.insert(Operands.begin(), Operands.end());
4657 Changed = true;
4658 break;
4659 }
4660 }
4661 }
4662
4663 return Changed;
4664}
4665
4666void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4667 // Initialize the collections. We will make a single pass over the block.
4668 Stores.clear();
4669 GEPs.clear();
4670
4671 // Visit the store and getelementptr instructions in BB and organize them in
4672 // Stores and GEPs according to the underlying objects of their pointer
4673 // operands.
4674 for (Instruction &I : *BB) {
4675 // Ignore store instructions that are volatile or have a pointer operand
4676 // that doesn't point to a scalar type.
4677 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4678 if (!SI->isSimple())
4679 continue;
4680 if (!isValidElementType(SI->getValueOperand()->getType()))
4681 continue;
4682 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4683 }
4684
4685 // Ignore getelementptr instructions that have more than one index, a
4686 // constant index, or a pointer operand that doesn't point to a scalar
4687 // type.
4688 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4689 auto Idx = GEP->idx_begin()->get();
4690 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4691 continue;
4692 if (!isValidElementType(Idx->getType()))
4693 continue;
4694 if (GEP->getType()->isVectorTy())
4695 continue;
4696 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4697 }
4698 }
4699}
4700
4701bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4702 if (!A || !B)
4703 return false;
4704 Value *VL[] = { A, B };
4705 return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4706}
4707
4708bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4709 int UserCost, bool AllowReorder) {
4710 if (VL.size() < 2)
4711 return false;
4712
4713 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4714 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
4715
4716 // Check that all of the parts are scalar instructions of the same type.
4717 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4718 if (!I0)
4719 return false;
4720
4721 unsigned Opcode0 = I0->getOpcode();
4722
4723 unsigned Sz = R.getVectorElementSize(I0);
4724 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4725 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4726 if (MaxVF < 2) {
4727 R.getORE()->emit([&]() {
4728 return OptimizationRemarkMissed(
4729 SV_NAME"slp-vectorizer", "SmallVF", I0)
4730 << "Cannot SLP vectorize list: vectorization factor "
4731 << "less than 2 is not supported";
4732 });
4733 return false;
4734 }
4735
4736 for (Value *V : VL) {
4737 Type *Ty = V->getType();
4738 if (!isValidElementType(Ty)) {
4739 // NOTE: the following will give user internal llvm type name, which may not be useful
4740 R.getORE()->emit([&]() {
4741 std::string type_str;
4742 llvm::raw_string_ostream rso(type_str);
4743 Ty->print(rso);
4744 return OptimizationRemarkMissed(
4745 SV_NAME"slp-vectorizer", "UnsupportedType", I0)
4746 << "Cannot SLP vectorize list: type "
4747 << rso.str() + " is unsupported by vectorizer";
4748 });
4749 return false;
4750 }
4751 Instruction *Inst = dyn_cast<Instruction>(V);
4752
4753 if (!Inst)
4754 return false;
4755 if (Inst->getOpcode() != Opcode0) {
4756 R.getORE()->emit([&]() {
4757 return OptimizationRemarkMissed(
4758 SV_NAME"slp-vectorizer", "InequableTypes", I0)
4759 << "Cannot SLP vectorize list: not all of the "
4760 << "parts of scalar instructions are of the same type: "
4761 << ore::NV("Instruction1Opcode", I0) << " and "
4762 << ore::NV("Instruction2Opcode", Inst);
4763 });
4764 return false;
4765 }
4766 }
4767
4768 bool Changed = false;
4769 bool CandidateFound = false;
4770 int MinCost = SLPCostThreshold;
4771
4772 // Keep track of values that were deleted by vectorizing in the loop below.
4773 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4774
4775 unsigned NextInst = 0, MaxInst = VL.size();
4776 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4777 VF /= 2) {
4778 // No actual vectorization should happen, if number of parts is the same as
4779 // provided vectorization factor (i.e. the scalar type is used for vector
4780 // code during codegen).
4781 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4782 if (TTI->getNumberOfParts(VecTy) == VF)
4783 continue;
4784 for (unsigned I = NextInst; I < MaxInst; ++I) {
4785 unsigned OpsWidth = 0;
4786
4787 if (I + VF > MaxInst)
4788 OpsWidth = MaxInst - I;
4789 else
4790 OpsWidth = VF;
4791
4792 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4793 break;
4794
4795 // Check that a previous iteration of this loop did not delete the Value.
4796 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4797 continue;
4798
4799 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4800 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
4801 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4802
4803 R.buildTree(Ops);
4804 // TODO: check if we can allow reordering for more cases.
4805 if (AllowReorder && R.shouldReorder()) {
4806 // Conceptually, there is nothing actually preventing us from trying to
4807 // reorder a larger list. In fact, we do exactly this when vectorizing
4808 // reductions. However, at this point, we only expect to get here when
4809 // there are exactly two operations.
4810 assert(Ops.size() == 2)(static_cast <bool> (Ops.size() == 2) ? void (0) : __assert_fail
("Ops.size() == 2", "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4810, __extension__ __PRETTY_FUNCTION__))
;
4811 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4812 R.buildTree(ReorderedOps, None);
4813 }
4814 if (R.isTreeTinyAndNotFullyVectorizable())
4815 continue;
4816
4817 R.computeMinimumValueSizes();
4818 int Cost = R.getTreeCost() - UserCost;
4819 CandidateFound = true;
4820 MinCost = std::min(MinCost, Cost);
4821
4822 if (Cost < -SLPCostThreshold) {
4823 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
4824 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedList",
4825 cast<Instruction>(Ops[0]))
4826 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4827 << " and with tree size "
4828 << ore::NV("TreeSize", R.getTreeSize()));
4829
4830 R.vectorizeTree();
4831 // Move to the next bundle.
4832 I += VF - 1;
4833 NextInst = I + 1;
4834 Changed = true;
4835 }
4836 }
4837 }
4838
4839 if (!Changed && CandidateFound) {
4840 R.getORE()->emit([&]() {
4841 return OptimizationRemarkMissed(
4842 SV_NAME"slp-vectorizer", "NotBeneficial", I0)
4843 << "List vectorization was possible but not beneficial with cost "
4844 << ore::NV("Cost", MinCost) << " >= "
4845 << ore::NV("Treshold", -SLPCostThreshold);
4846 });
4847 } else if (!Changed) {
4848 R.getORE()->emit([&]() {
4849 return OptimizationRemarkMissed(
4850 SV_NAME"slp-vectorizer", "NotPossible", I0)
4851 << "Cannot SLP vectorize list: vectorization was impossible"
4852 << " with available vectorization factors";
4853 });
4854 }
4855 return Changed;
4856}
4857
4858bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4859 if (!I)
4860 return false;
4861
4862 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4863 return false;
4864
4865 Value *P = I->getParent();
4866
4867 // Vectorize in current basic block only.
4868 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4869 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4870 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4871 return false;
4872
4873 // Try to vectorize V.
4874 if (tryToVectorizePair(Op0, Op1, R))
4875 return true;
4876
4877 auto *A = dyn_cast<BinaryOperator>(Op0);
4878 auto *B = dyn_cast<BinaryOperator>(Op1);
4879 // Try to skip B.
4880 if (B && B->hasOneUse()) {
4881 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4882 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4883 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4884 return true;
4885 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4886 return true;
4887 }
4888
4889 // Try to skip A.
4890 if (A && A->hasOneUse()) {
4891 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4892 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4893 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4894 return true;
4895 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4896 return true;
4897 }
4898 return false;
4899}
4900
4901/// \brief Generate a shuffle mask to be used in a reduction tree.
4902///
4903/// \param VecLen The length of the vector to be reduced.
4904/// \param NumEltsToRdx The number of elements that should be reduced in the
4905/// vector.
4906/// \param IsPairwise Whether the reduction is a pairwise or splitting
4907/// reduction. A pairwise reduction will generate a mask of
4908/// <0,2,...> or <1,3,..> while a splitting reduction will generate
4909/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4910/// \param IsLeft True will generate a mask of even elements, odd otherwise.
4911static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4912 bool IsPairwise, bool IsLeft,
4913 IRBuilder<> &Builder) {
4914 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(static_cast <bool> ((IsPairwise || !IsLeft) &&
"Don't support a <0,1,undef,...> mask") ? void (0) : __assert_fail
("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4914, __extension__ __PRETTY_FUNCTION__))
;
4915
4916 SmallVector<Constant *, 32> ShuffleMask(
4917 VecLen, UndefValue::get(Builder.getInt32Ty()));
4918
4919 if (IsPairwise)
4920 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4921 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4922 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4923 else
4924 // Move the upper half of the vector to the lower half.
4925 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4926 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4927
4928 return ConstantVector::get(ShuffleMask);
4929}
4930
4931namespace {
4932
4933/// Model horizontal reductions.
4934///
4935/// A horizontal reduction is a tree of reduction operations (currently add and
4936/// fadd) that has operations that can be put into a vector as its leaf.
4937/// For example, this tree:
4938///
4939/// mul mul mul mul
4940/// \ / \ /
4941/// + +
4942/// \ /
4943/// +
4944/// This tree has "mul" as its reduced values and "+" as its reduction
4945/// operations. A reduction might be feeding into a store or a binary operation
4946/// feeding a phi.
4947/// ...
4948/// \ /
4949/// +
4950/// |
4951/// phi +=
4952///
4953/// Or:
4954/// ...
4955/// \ /
4956/// +
4957/// |
4958/// *p =
4959///
4960class HorizontalReduction {
4961 using ReductionOpsType = SmallVector<Value *, 16>;
4962 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
4963 ReductionOpsListType ReductionOps;
4964 SmallVector<Value *, 32> ReducedVals;
4965 // Use map vector to make stable output.
4966 MapVector<Instruction *, Value *> ExtraArgs;
4967
4968 /// Kind of the reduction data.
4969 enum ReductionKind {
4970 RK_None, /// Not a reduction.
4971 RK_Arithmetic, /// Binary reduction data.
4972 RK_Min, /// Minimum reduction data.
4973 RK_UMin, /// Unsigned minimum reduction data.
4974 RK_Max, /// Maximum reduction data.
4975 RK_UMax, /// Unsigned maximum reduction data.
4976 };
4977
4978 /// Contains info about operation, like its opcode, left and right operands.
4979 class OperationData {
4980 /// Opcode of the instruction.
4981 unsigned Opcode = 0;
4982
4983 /// Left operand of the reduction operation.
4984 Value *LHS = nullptr;
4985
4986 /// Right operand of the reduction operation.
4987 Value *RHS = nullptr;
4988
4989 /// Kind of the reduction operation.
4990 ReductionKind Kind = RK_None;
4991
4992 /// True if float point min/max reduction has no NaNs.
4993 bool NoNaN = false;
4994
4995 /// Checks if the reduction operation can be vectorized.
4996 bool isVectorizable() const {
4997 return LHS && RHS &&
4998 // We currently only support adds && min/max reductions.
4999 ((Kind == RK_Arithmetic &&
5000 (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
5001 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5002 (Kind == RK_Min || Kind == RK_Max)) ||
5003 (Opcode == Instruction::ICmp &&
5004 (Kind == RK_UMin || Kind == RK_UMax)));
5005 }
5006
5007 /// Creates reduction operation with the current opcode.
5008 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5009 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5010, __extension__ __PRETTY_FUNCTION__))
5010 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5010, __extension__ __PRETTY_FUNCTION__))
;
5011 Value *Cmp;
5012 switch (Kind) {
5013 case RK_Arithmetic:
5014 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5015 Name);
5016 case RK_Min:
5017 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5018 : Builder.CreateFCmpOLT(LHS, RHS);
5019 break;
5020 case RK_Max:
5021 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5022 : Builder.CreateFCmpOGT(LHS, RHS);
5023 break;
5024 case RK_UMin:
5025 assert(Opcode == Instruction::ICmp && "Expected integer types.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Expected integer types.") ? void (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5025, __extension__ __PRETTY_FUNCTION__))
;
5026 Cmp = Builder.CreateICmpULT(LHS, RHS);
5027 break;
5028 case RK_UMax:
5029 assert(Opcode == Instruction::ICmp && "Expected integer types.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Expected integer types.") ? void (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5029, __extension__ __PRETTY_FUNCTION__))
;
5030 Cmp = Builder.CreateICmpUGT(LHS, RHS);
5031 break;
5032 case RK_None:
5033 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5033)
;
5034 }
5035 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5036 }
5037
5038 public:
5039 explicit OperationData() = default;
5040
5041 /// Construction for reduced values. They are identified by opcode only and
5042 /// don't have associated LHS/RHS values.
5043 explicit OperationData(Value *V) {
5044 if (auto *I = dyn_cast<Instruction>(V))
5045 Opcode = I->getOpcode();
5046 }
5047
5048 /// Constructor for reduction operations with opcode and its left and
5049 /// right operands.
5050 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5051 bool NoNaN = false)
5052 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5053 assert(Kind != RK_None && "One of the reduction operations is expected.")(static_cast <bool> (Kind != RK_None && "One of the reduction operations is expected."
) ? void (0) : __assert_fail ("Kind != RK_None && \"One of the reduction operations is expected.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5053, __extension__ __PRETTY_FUNCTION__))
;
5054 }
5055
5056 explicit operator bool() const { return Opcode; }
5057
5058 /// Get the index of the first operand.
5059 unsigned getFirstOperandIndex() const {
5060 assert(!!*this && "The opcode is not set.")(static_cast <bool> (!!*this && "The opcode is not set."
) ? void (0) : __assert_fail ("!!*this && \"The opcode is not set.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5060, __extension__ __PRETTY_FUNCTION__))
;
5061 switch (Kind) {
5062 case RK_Min:
5063 case RK_UMin:
5064 case RK_Max:
5065 case RK_UMax:
5066 return 1;
5067 case RK_Arithmetic:
5068 case RK_None:
5069 break;
5070 }
5071 return 0;
5072 }
5073
5074 /// Total number of operands in the reduction operation.
5075 unsigned getNumberOfOperands() const {
5076 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5077, __extension__ __PRETTY_FUNCTION__))
5077 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5077, __extension__ __PRETTY_FUNCTION__))
;
5078 switch (Kind) {
5079 case RK_Arithmetic:
5080 return 2;
5081 case RK_Min:
5082 case RK_UMin:
5083 case RK_Max:
5084 case RK_UMax:
5085 return 3;
5086 case RK_None:
5087 break;
5088 }
5089 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5089)
;
5090 }
5091
5092 /// Checks if the operation has the same parent as \p P.
5093 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5094 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5095, __extension__ __PRETTY_FUNCTION__))
5095 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5095, __extension__ __PRETTY_FUNCTION__))
;
5096 if (!IsRedOp)
5097 return I->getParent() == P;
5098 switch (Kind) {
5099 case RK_Arithmetic:
5100 // Arithmetic reduction operation must be used once only.
5101 return I->getParent() == P;
5102 case RK_Min:
5103 case RK_UMin:
5104 case RK_Max:
5105 case RK_UMax: {
5106 // SelectInst must be used twice while the condition op must have single
5107 // use only.
5108 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5109 return I->getParent() == P && Cmp && Cmp->getParent() == P;
5110 }
5111 case RK_None:
5112 break;
5113 }
5114 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5114)
;
5115 }
5116 /// Expected number of uses for reduction operations/reduced values.
5117 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5118 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5119, __extension__ __PRETTY_FUNCTION__))
5119 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5119, __extension__ __PRETTY_FUNCTION__))
;
5120 switch (Kind) {
5121 case RK_Arithmetic:
5122 return I->hasOneUse();
5123 case RK_Min:
5124 case RK_UMin:
5125 case RK_Max:
5126 case RK_UMax:
5127 return I->hasNUses(2) &&
5128 (!IsReductionOp ||
5129 cast<SelectInst>(I)->getCondition()->hasOneUse());
5130 case RK_None:
5131 break;
5132 }
5133 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5133)
;
5134 }
5135
5136 /// Initializes the list of reduction operations.
5137 void initReductionOps(ReductionOpsListType &ReductionOps) {
5138 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5139, __extension__ __PRETTY_FUNCTION__))
5139 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5139, __extension__ __PRETTY_FUNCTION__))
;
5140 switch (Kind) {
5141 case RK_Arithmetic:
5142 ReductionOps.assign(1, ReductionOpsType());
5143 break;
5144 case RK_Min:
5145 case RK_UMin:
5146 case RK_Max:
5147 case RK_UMax:
5148 ReductionOps.assign(2, ReductionOpsType());
5149 break;
5150 case RK_None:
5151 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5151)
;
5152 }
5153 }
5154 /// Add all reduction operations for the reduction instruction \p I.
5155 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5156 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5157, __extension__ __PRETTY_FUNCTION__))
5157 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5157, __extension__ __PRETTY_FUNCTION__))
;
5158 switch (Kind) {
5159 case RK_Arithmetic:
5160 ReductionOps[0].emplace_back(I);
5161 break;
5162 case RK_Min:
5163 case RK_UMin:
5164 case RK_Max:
5165 case RK_UMax:
5166 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5167 ReductionOps[1].emplace_back(I);
5168 break;
5169 case RK_None:
5170 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5170)
;
5171 }
5172 }
5173
5174 /// Checks if instruction is associative and can be vectorized.
5175 bool isAssociative(Instruction *I) const {
5176 assert(Kind != RK_None && *this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && *this &&
LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5177, __extension__ __PRETTY_FUNCTION__))
5177 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && *this &&
LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5177, __extension__ __PRETTY_FUNCTION__))
;
5178 switch (Kind) {
5179 case RK_Arithmetic:
5180 return I->isAssociative();
5181 case RK_Min:
5182 case RK_Max:
5183 return Opcode == Instruction::ICmp ||
5184 cast<Instruction>(I->getOperand(0))->isFast();
5185 case RK_UMin:
5186 case RK_UMax:
5187 assert(Opcode == Instruction::ICmp &&(static_cast <bool> (Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.") ? void (0) : __assert_fail
("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5188, __extension__ __PRETTY_FUNCTION__))
5188 "Only integer compare operation is expected.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.") ? void (0) : __assert_fail
("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5188, __extension__ __PRETTY_FUNCTION__))
;
5189 return true;
5190 case RK_None:
5191 break;
5192 }
5193 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5193)
;
5194 }
5195
5196 /// Checks if the reduction operation can be vectorized.
5197 bool isVectorizable(Instruction *I) const {
5198 return isVectorizable() && isAssociative(I);
5199 }
5200
5201 /// Checks if two operation data are both a reduction op or both a reduced
5202 /// value.
5203 bool operator==(const OperationData &OD) {
5204 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&(static_cast <bool> (((Kind != OD.Kind) || ((!LHS == !OD
.LHS) && (!RHS == !OD.RHS))) && "One of the comparing operations is incorrect."
) ? void (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5205, __extension__ __PRETTY_FUNCTION__))
5205 "One of the comparing operations is incorrect.")(static_cast <bool> (((Kind != OD.Kind) || ((!LHS == !OD
.LHS) && (!RHS == !OD.RHS))) && "One of the comparing operations is incorrect."
) ? void (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5205, __extension__ __PRETTY_FUNCTION__))
;
5206 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5207 }
5208 bool operator!=(const OperationData &OD) { return !(*this == OD); }
5209 void clear() {
5210 Opcode = 0;
5211 LHS = nullptr;
5212 RHS = nullptr;
5213 Kind = RK_None;
5214 NoNaN = false;
5215 }
5216
5217 /// Get the opcode of the reduction operation.
5218 unsigned getOpcode() const {
5219 assert(isVectorizable() && "Expected vectorizable operation.")(static_cast <bool> (isVectorizable() && "Expected vectorizable operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected vectorizable operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5219, __extension__ __PRETTY_FUNCTION__))
;
5220 return Opcode;
5221 }
5222
5223 /// Get kind of reduction data.
5224 ReductionKind getKind() const { return Kind; }
5225 Value *getLHS() const { return LHS; }
5226 Value *getRHS() const { return RHS; }
5227 Type *getConditionType() const {
5228 switch (Kind) {
5229 case RK_Arithmetic:
5230 return nullptr;
5231 case RK_Min:
5232 case RK_Max:
5233 case RK_UMin:
5234 case RK_UMax:
5235 return CmpInst::makeCmpResultType(LHS->getType());
5236 case RK_None:
5237 break;
5238 }
5239 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5239)
;
5240 }
5241
5242 /// Creates reduction operation with the current opcode with the IR flags
5243 /// from \p ReductionOps.
5244 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5245 const ReductionOpsListType &ReductionOps) const {
5246 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5247, __extension__ __PRETTY_FUNCTION__))
5247 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5247, __extension__ __PRETTY_FUNCTION__))
;
5248 auto *Op = createOp(Builder, Name);
5249 switch (Kind) {
5250 case RK_Arithmetic:
5251 propagateIRFlags(Op, ReductionOps[0]);
5252 return Op;
5253 case RK_Min:
5254 case RK_Max:
5255 case RK_UMin:
5256 case RK_UMax:
5257 if (auto *SI = dyn_cast<SelectInst>(Op))
5258 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5259 propagateIRFlags(Op, ReductionOps[1]);
5260 return Op;
5261 case RK_None:
5262 break;
5263 }
5264 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5264)
;
5265 }
5266 /// Creates reduction operation with the current opcode with the IR flags
5267 /// from \p I.
5268 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5269 Instruction *I) const {
5270 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5271, __extension__ __PRETTY_FUNCTION__))
5271 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5271, __extension__ __PRETTY_FUNCTION__))
;
5272 auto *Op = createOp(Builder, Name);
5273 switch (Kind) {
5274 case RK_Arithmetic:
5275 propagateIRFlags(Op, I);
5276 return Op;
5277 case RK_Min:
5278 case RK_Max:
5279 case RK_UMin:
5280 case RK_UMax:
5281 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5282 propagateIRFlags(SI->getCondition(),
5283 cast<SelectInst>(I)->getCondition());
5284 }
5285 propagateIRFlags(Op, I);
5286 return Op;
5287 case RK_None:
5288 break;
5289 }
5290 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5290)
;
5291 }
5292
5293 TargetTransformInfo::ReductionFlags getFlags() const {
5294 TargetTransformInfo::ReductionFlags Flags;
5295 Flags.NoNaN = NoNaN;
5296 switch (Kind) {
5297 case RK_Arithmetic:
5298 break;
5299 case RK_Min:
5300 Flags.IsSigned = Opcode == Instruction::ICmp;
5301 Flags.IsMaxOp = false;
5302 break;
5303 case RK_Max:
5304 Flags.IsSigned = Opcode == Instruction::ICmp;
5305 Flags.IsMaxOp = true;
5306 break;
5307 case RK_UMin:
5308 Flags.IsSigned = false;
5309 Flags.IsMaxOp = false;
5310 break;
5311 case RK_UMax:
5312 Flags.IsSigned = false;
5313 Flags.IsMaxOp = true;
5314 break;
5315 case RK_None:
5316 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5316)
;
5317 }
5318 return Flags;
5319 }
5320 };
5321
5322 Instruction *ReductionRoot = nullptr;
5323
5324 /// The operation data of the reduction operation.
5325 OperationData ReductionData;
5326
5327 /// The operation data of the values we perform a reduction on.
5328 OperationData ReducedValueData;
5329
5330 /// Should we model this reduction as a pairwise reduction tree or a tree that
5331 /// splits the vector in halves and adds those halves.
5332 bool IsPairwiseReduction = false;
5333
5334 /// Checks if the ParentStackElem.first should be marked as a reduction
5335 /// operation with an extra argument or as extra argument itself.
5336 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5337 Value *ExtraArg) {
5338 if (ExtraArgs.count(ParentStackElem.first)) {
5339 ExtraArgs[ParentStackElem.first] = nullptr;
5340 // We ran into something like:
5341 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5342 // The whole ParentStackElem.first should be considered as an extra value
5343 // in this case.
5344 // Do not perform analysis of remaining operands of ParentStackElem.first
5345 // instruction, this whole instruction is an extra argument.
5346 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5347 } else {
5348 // We ran into something like:
5349 // ParentStackElem.first += ... + ExtraArg + ...
5350 ExtraArgs[ParentStackElem.first] = ExtraArg;
5351 }
5352 }
5353
5354 static OperationData getOperationData(Value *V) {
5355 if (!V)
5356 return OperationData();
5357
5358 Value *LHS;
5359 Value *RHS;
5360 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5361 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5362 RK_Arithmetic);
5363 }
5364 if (auto *Select = dyn_cast<SelectInst>(V)) {
5365 // Look for a min/max pattern.
5366 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5367 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5368 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5369 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5370 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5371 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5372 return OperationData(
5373 Instruction::FCmp, LHS, RHS, RK_Min,
5374 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5375 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5376 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5377 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5378 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5379 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5380 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5381 return OperationData(
5382 Instruction::FCmp, LHS, RHS, RK_Max,
5383 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5384 }
5385 }
5386 return OperationData(V);
5387 }
5388
5389public:
5390 HorizontalReduction() = default;
5391
5392 /// \brief Try to find a reduction tree.
5393 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5394 assert((!Phi || is_contained(Phi->operands(), B)) &&(static_cast <bool> ((!Phi || is_contained(Phi->operands
(), B)) && "Thi phi needs to use the binary operator"
) ? void (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5395, __extension__ __PRETTY_FUNCTION__))
5395 "Thi phi needs to use the binary operator")(static_cast <bool> ((!Phi || is_contained(Phi->operands
(), B)) && "Thi phi needs to use the binary operator"
) ? void (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5395, __extension__ __PRETTY_FUNCTION__))
;
5396
5397 ReductionData = getOperationData(B);
5398
5399 // We could have a initial reductions that is not an add.
5400 // r *= v1 + v2 + v3 + v4
5401 // In such a case start looking for a tree rooted in the first '+'.
5402 if (Phi) {
5403 if (ReductionData.getLHS() == Phi) {
5404 Phi = nullptr;
5405 B = dyn_cast<Instruction>(ReductionData.getRHS());
5406 ReductionData = getOperationData(B);
5407 } else if (ReductionData.getRHS() == Phi) {
5408 Phi = nullptr;
5409 B = dyn_cast<Instruction>(ReductionData.getLHS());
5410 ReductionData = getOperationData(B);
5411 }
5412 }
5413
5414 if (!ReductionData.isVectorizable(B))
5415 return false;
5416
5417 Type *Ty = B->getType();
5418 if (!isValidElementType(Ty))
5419 return false;
5420
5421 ReducedValueData.clear();
5422 ReductionRoot = B;
5423
5424 // Post order traverse the reduction tree starting at B. We only handle true
5425 // trees containing only binary operators.
5426 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5427 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5428 ReductionData.initReductionOps(ReductionOps);
5429 while (!Stack.empty()) {
5430 Instruction *TreeN = Stack.back().first;
5431 unsigned EdgeToVist = Stack.back().second++;
5432 OperationData OpData = getOperationData(TreeN);
5433 bool IsReducedValue = OpData != ReductionData;
5434
5435 // Postorder vist.
5436 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5437 if (IsReducedValue)
5438 ReducedVals.push_back(TreeN);
5439 else {
5440 auto I = ExtraArgs.find(TreeN);
5441 if (I != ExtraArgs.end() && !I->second) {
5442 // Check if TreeN is an extra argument of its parent operation.
5443 if (Stack.size() <= 1) {
5444 // TreeN can't be an extra argument as it is a root reduction
5445 // operation.
5446 return false;
5447 }
5448 // Yes, TreeN is an extra argument, do not add it to a list of
5449 // reduction operations.
5450 // Stack[Stack.size() - 2] always points to the parent operation.
5451 markExtraArg(Stack[Stack.size() - 2], TreeN);
5452 ExtraArgs.erase(TreeN);
5453 } else
5454 ReductionData.addReductionOps(TreeN, ReductionOps);
5455 }
5456 // Retract.
5457 Stack.pop_back();
5458 continue;
5459 }
5460
5461 // Visit left or right.
5462 Value *NextV = TreeN->getOperand(EdgeToVist);
5463 if (NextV != Phi) {
5464 auto *I = dyn_cast<Instruction>(NextV);
5465 OpData = getOperationData(I);
5466 // Continue analysis if the next operand is a reduction operation or
5467 // (possibly) a reduced value. If the reduced value opcode is not set,
5468 // the first met operation != reduction operation is considered as the
5469 // reduced value class.
5470 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5471 OpData == ReductionData)) {
5472 const bool IsReductionOperation = OpData == ReductionData;
5473 // Only handle trees in the current basic block.
5474 if (!ReductionData.hasSameParent(I, B->getParent(),
5475 IsReductionOperation)) {
5476 // I is an extra argument for TreeN (its parent operation).
5477 markExtraArg(Stack.back(), I);
5478 continue;
5479 }
5480
5481 // Each tree node needs to have minimal number of users except for the
5482 // ultimate reduction.
5483 if (!ReductionData.hasRequiredNumberOfUses(I,
5484 OpData == ReductionData) &&
5485 I != B) {
5486 // I is an extra argument for TreeN (its parent operation).
5487 markExtraArg(Stack.back(), I);
5488 continue;
5489 }
5490
5491 if (IsReductionOperation) {
5492 // We need to be able to reassociate the reduction operations.
5493 if (!OpData.isAssociative(I)) {
5494 // I is an extra argument for TreeN (its parent operation).
5495 markExtraArg(Stack.back(), I);
5496 continue;
5497 }
5498 } else if (ReducedValueData &&
5499 ReducedValueData != OpData) {
5500 // Make sure that the opcodes of the operations that we are going to
5501 // reduce match.
5502 // I is an extra argument for TreeN (its parent operation).
5503 markExtraArg(Stack.back(), I);
5504 continue;
5505 } else if (!ReducedValueData)
5506 ReducedValueData = OpData;
5507
5508 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5509 continue;
5510 }
5511 }
5512 // NextV is an extra argument for TreeN (its parent operation).
5513 markExtraArg(Stack.back(), NextV);
5514 }
5515 return true;
5516 }
5517
5518 /// \brief Attempt to vectorize the tree found by
5519 /// matchAssociativeReduction.
5520 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5521 if (ReducedVals.empty())
5522 return false;
5523
5524 // If there is a sufficient number of reduction values, reduce
5525 // to a nearby power-of-2. Can safely generate oversized
5526 // vectors and rely on the backend to split them to legal sizes.
5527 unsigned NumReducedVals = ReducedVals.size();
5528 if (NumReducedVals < 4)
5529 return false;
5530
5531 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5532
5533 Value *VectorizedTree = nullptr;
5534 IRBuilder<> Builder(ReductionRoot);
5535 FastMathFlags Unsafe;
5536 Unsafe.setFast();
5537 Builder.setFastMathFlags(Unsafe);
5538 unsigned i = 0;
5539
5540 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5541 // The same extra argument may be used several time, so log each attempt
5542 // to use it.
5543 for (auto &Pair : ExtraArgs)
5544 ExternallyUsedValues[Pair.second].push_back(Pair.first);
5545 SmallVector<Value *, 16> IgnoreList;
5546 for (auto &V : ReductionOps)
5547 IgnoreList.append(V.begin(), V.end());
5548 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5549 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5550 V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5551 if (V.shouldReorder()) {
5552 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
5553 V.buildTree(Reversed, ExternallyUsedValues, IgnoreList);
5554 }
5555 if (V.isTreeTinyAndNotFullyVectorizable())
5556 break;
5557
5558 V.computeMinimumValueSizes();
5559
5560 // Estimate cost.
5561 int Cost =
5562 V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5563 if (Cost >= -SLPCostThreshold) {
5564 V.getORE()->emit([&]() {
5565 return OptimizationRemarkMissed(
5566 SV_NAME"slp-vectorizer", "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5567 << "Vectorizing horizontal reduction is possible"
5568 << "but not beneficial with cost "
5569 << ore::NV("Cost", Cost) << " and threshold "
5570 << ore::NV("Threshold", -SLPCostThreshold);
5571 });
5572 break;
5573 }
5574
5575 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
5576 << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
;
5577 V.getORE()->emit([&]() {
5578 return OptimizationRemark(
5579 SV_NAME"slp-vectorizer", "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5580 << "Vectorized horizontal reduction with cost "
5581 << ore::NV("Cost", Cost) << " and with tree size "
5582 << ore::NV("TreeSize", V.getTreeSize());
5583 });
5584
5585 // Vectorize a tree.
5586 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5587 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5588
5589 // Emit a reduction.
5590 Value *ReducedSubTree =
5591 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5592 if (VectorizedTree) {
5593 Builder.SetCurrentDebugLocation(Loc);
5594 OperationData VectReductionData(ReductionData.getOpcode(),
5595 VectorizedTree, ReducedSubTree,
5596 ReductionData.getKind());
5597 VectorizedTree =
5598 VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5599 } else
5600 VectorizedTree = ReducedSubTree;
5601 i += ReduxWidth;
5602 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5603 }
5604
5605 if (VectorizedTree) {
5606 // Finish the reduction.
5607 for (; i < NumReducedVals; ++i) {
5608 auto *I = cast<Instruction>(ReducedVals[i]);
5609 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5610 OperationData VectReductionData(ReductionData.getOpcode(),
5611 VectorizedTree, I,
5612 ReductionData.getKind());
5613 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5614 }
5615 for (auto &Pair : ExternallyUsedValues) {
5616 assert(!Pair.second.empty() &&(static_cast <bool> (!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? void (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5617, __extension__ __PRETTY_FUNCTION__))
5617 "At least one DebugLoc must be inserted")(static_cast <bool> (!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? void (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5617, __extension__ __PRETTY_FUNCTION__))
;
5618 // Add each externally used value to the final reduction.
5619 for (auto *I : Pair.second) {
5620 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5621 OperationData VectReductionData(ReductionData.getOpcode(),
5622 VectorizedTree, Pair.first,
5623 ReductionData.getKind());
5624 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5625 }
5626 }
5627 // Update users.
5628 ReductionRoot->replaceAllUsesWith(VectorizedTree);
5629 }
5630 return VectorizedTree != nullptr;
5631 }
5632
5633 unsigned numReductionValues() const {
5634 return ReducedVals.size();
5635 }
5636
5637private:
5638 /// \brief Calculate the cost of a reduction.
5639 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5640 unsigned ReduxWidth) {
5641 Type *ScalarTy = FirstReducedVal->getType();
5642 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5643
5644 int PairwiseRdxCost;
5645 int SplittingRdxCost;
5646 switch (ReductionData.getKind()) {
5647 case RK_Arithmetic:
5648 PairwiseRdxCost =
5649 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5650 /*IsPairwiseForm=*/true);
5651 SplittingRdxCost =
5652 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5653 /*IsPairwiseForm=*/false);
5654 break;
5655 case RK_Min:
5656 case RK_Max:
5657 case RK_UMin:
5658 case RK_UMax: {
5659 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5660 bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5661 ReductionData.getKind() == RK_UMax;
5662 PairwiseRdxCost =
5663 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5664 /*IsPairwiseForm=*/true, IsUnsigned);
5665 SplittingRdxCost =
5666 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5667 /*IsPairwiseForm=*/false, IsUnsigned);
5668 break;
5669 }
5670 case RK_None:
5671 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-7~svn325118/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5671)
;
5672 }
5673
5674 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5675 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5676
5677 int ScalarReduxCost;
5678 switch (ReductionData.getKind()) {
5679 case RK_Arithmetic:
5680 ScalarReduxCost =
5681 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5682 break;
5683 case RK_Min:
5684 case RK_Max:
5685 case RK_UMin:
5686 case RK_UMax:
5687 ScalarReduxCost =
5688 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5689 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5690 CmpInst::makeCmpResultType(ScalarTy));