Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 3895, column 11
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SLPVectorizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn329677/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-7~svn329677/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn329677/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/x86_64-linux-gnu/c++/7.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.3.0/../../../../include/c++/7.3.0/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn329677/build-llvm/lib/Transforms/Vectorize -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-04-11-031539-24776-1 -x c++ /build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp
1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11// stores that can be put together into vector-stores. Next, it attempts to
12// construct vectorizable tree using the use-def chains. If a profitable tree
13// was found, the SLP vectorizer performs vectorization on the tree.
14//
15// The pass is inspired by the work described in the paper:
16// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/DenseSet.h"
24#include "llvm/ADT/MapVector.h"
25#include "llvm/ADT/None.h"
26#include "llvm/ADT/Optional.h"
27#include "llvm/ADT/PostOrderIterator.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SetVector.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/iterator.h"
35#include "llvm/ADT/iterator_range.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/Analysis/CodeMetrics.h"
38#include "llvm/Analysis/DemandedBits.h"
39#include "llvm/Analysis/GlobalsModRef.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PassManager.h"
69#include "llvm/IR/PatternMatch.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/IR/Verifier.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/DOTGraphTraits.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/ErrorHandling.h"
83#include "llvm/Support/GraphWriter.h"
84#include "llvm/Support/KnownBits.h"
85#include "llvm/Support/MathExtras.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Transforms/Utils/LoopUtils.h"
88#include "llvm/Transforms/Vectorize.h"
89#include <algorithm>
90#include <cassert>
91#include <cstdint>
92#include <iterator>
93#include <memory>
94#include <set>
95#include <string>
96#include <tuple>
97#include <utility>
98#include <vector>
99
100using namespace llvm;
101using namespace llvm::PatternMatch;
102using namespace slpvectorizer;
103
104#define SV_NAME"slp-vectorizer" "slp-vectorizer"
105#define DEBUG_TYPE"SLP" "SLP"
106
107STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, {false}}
;
108
109static cl::opt<int>
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
112 "number "));
113
114static cl::opt<bool>
115ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
117
118static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120 cl::desc(
121 "Attempt to vectorize horizontal reductions feeding into a store"));
122
123static cl::opt<int>
124MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
126
127/// Limits the size of scheduling regions in a block.
128/// It avoid long compile times for _very_ large blocks where vector
129/// instructions are spread over a wide range.
130/// This limit is way higher than needed by real-world functions.
131static cl::opt<int>
132ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
134
135static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
138
139static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
142
143static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
146
147static cl::opt<bool>
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
150
151// Limit the number of alias checks. The limit is chosen so that
152// it has no negative effect on the llvm benchmarks.
153static const unsigned AliasedCheckLimit = 10;
154
155// Another limit for the alias checks: The maximum distance between load/store
156// instructions where alias checks are done.
157// This limit is useful for very large basic blocks.
158static const unsigned MaxMemDepDistance = 160;
159
160/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161/// regions to be handled.
162static const int MinScheduleRegionSize = 16;
163
164/// \brief Predicate for the element types that the SLP vectorizer supports.
165///
166/// The most important thing to filter here are types which are invalid in LLVM
167/// vectors. We also filter target specific types which have absolutely no
168/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169/// avoids spending time checking the cost model and realizing that they will
170/// be inevitably scalarized.
171static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
174}
175
176/// \returns true if all of the instructions in \p VL are in the same block or
177/// false otherwise.
178static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180 if (!I0)
181 return false;
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
185 if (!I)
186 return false;
187
188 if (BB != I->getParent())
189 return false;
190 }
191 return true;
192}
193
194/// \returns True if all of the values in \p VL are constants.
195static bool allConstant(ArrayRef<Value *> VL) {
196 for (Value *i : VL)
197 if (!isa<Constant>(i))
198 return false;
199 return true;
200}
201
202/// \returns True if all of the values in \p VL are identical.
203static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
205 if (VL[i] != VL[0])
206 return false;
207 return true;
208}
209
210/// Checks if the vector of instructions can be represented as a shuffle, like:
211/// %x0 = extractelement <4 x i8> %x, i32 0
212/// %x3 = extractelement <4 x i8> %x, i32 3
213/// %y1 = extractelement <4 x i8> %y, i32 1
214/// %y2 = extractelement <4 x i8> %y, i32 2
215/// %x0x0 = mul i8 %x0, %x0
216/// %x3x3 = mul i8 %x3, %x3
217/// %y1y1 = mul i8 %y1, %y1
218/// %y2y2 = mul i8 %y2, %y2
219/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223/// ret <4 x i8> %ins4
224/// can be transformed into:
225/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226/// i32 6>
227/// %2 = mul <4 x i8> %1, %1
228/// ret <4 x i8> %2
229/// We convert this initially to something like:
230/// %x0 = extractelement <4 x i8> %x, i32 0
231/// %x3 = extractelement <4 x i8> %x, i32 3
232/// %y1 = extractelement <4 x i8> %y, i32 1
233/// %y2 = extractelement <4 x i8> %y, i32 2
234/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238/// %5 = mul <4 x i8> %4, %4
239/// %6 = extractelement <4 x i8> %5, i32 0
240/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241/// %7 = extractelement <4 x i8> %5, i32 1
242/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243/// %8 = extractelement <4 x i8> %5, i32 2
244/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245/// %9 = extractelement <4 x i8> %5, i32 3
246/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247/// ret <4 x i8> %ins4
248/// InstCombiner transforms this into a shuffle and vector mul
249static Optional<TargetTransformInfo::ShuffleKind>
250isShuffle(ArrayRef<Value *> VL) {
251 auto *EI0 = cast<ExtractElementInst>(VL[0]);
252 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253 Value *Vec1 = nullptr;
254 Value *Vec2 = nullptr;
255 enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256 ShuffleMode CommonShuffleMode = Unknown;
257 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258 auto *EI = cast<ExtractElementInst>(VL[I]);
259 auto *Vec = EI->getVectorOperand();
260 // All vector operands must have the same number of vector elements.
261 if (Vec->getType()->getVectorNumElements() != Size)
262 return None;
263 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264 if (!Idx)
265 return None;
266 // Undefined behavior if Idx is negative or >= Size.
267 if (Idx->getValue().uge(Size))
268 continue;
269 unsigned IntIdx = Idx->getValue().getZExtValue();
270 // We can extractelement from undef vector.
271 if (isa<UndefValue>(Vec))
272 continue;
273 // For correct shuffling we have to have at most 2 different vector operands
274 // in all extractelement instructions.
275 if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276 return None;
277 if (CommonShuffleMode == Permute)
278 continue;
279 // If the extract index is not the same as the operation number, it is a
280 // permutation.
281 if (IntIdx != I) {
282 CommonShuffleMode = Permute;
283 continue;
284 }
285 // Check the shuffle mode for the current operation.
286 if (!Vec1)
287 Vec1 = Vec;
288 else if (Vec != Vec1)
289 Vec2 = Vec;
290 // Example: shufflevector A, B, <0,5,2,7>
291 // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292 // I is even and IntIdx for B == I - FirstAlternate shuffle.
293 // Example: shufflevector A, B, <4,1,6,3>
294 // I is even and IntIdx for A == I - SecondAlternate shuffle.
295 // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296 const bool IIsEven = I & 1;
297 const bool CurrVecIsA = Vec == Vec1;
298 const bool IIsOdd = !IIsEven;
299 const bool CurrVecIsB = !CurrVecIsA;
300 ShuffleMode CurrentShuffleMode =
301 ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302 : SecondAlternate;
303 // Common mode is not set or the same as the shuffle mode of the current
304 // operation - alternate.
305 if (CommonShuffleMode == Unknown)
306 CommonShuffleMode = CurrentShuffleMode;
307 // Common shuffle mode is not the same as the shuffle mode of the current
308 // operation - permutation.
309 if (CommonShuffleMode != CurrentShuffleMode)
310 CommonShuffleMode = Permute;
311 }
312 // If we're not crossing lanes in different vectors, consider it as blending.
313 if ((CommonShuffleMode == FirstAlternate ||
314 CommonShuffleMode == SecondAlternate) &&
315 Vec2)
316 return TargetTransformInfo::SK_Alternate;
317 // If Vec2 was never used, we have a permutation of a single vector, otherwise
318 // we have permutation of 2 vectors.
319 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
320 : TargetTransformInfo::SK_PermuteSingleSrc;
321}
322
323///\returns Opcode that can be clubbed with \p Op to create an alternate
324/// sequence which can later be merged as a ShuffleVector instruction.
325static unsigned getAltOpcode(unsigned Op) {
326 switch (Op) {
327 case Instruction::FAdd:
328 return Instruction::FSub;
329 case Instruction::FSub:
330 return Instruction::FAdd;
331 case Instruction::Add:
332 return Instruction::Sub;
333 case Instruction::Sub:
334 return Instruction::Add;
335 default:
336 return 0;
337 }
338}
339
340static bool isOdd(unsigned Value) {
341 return Value & 1;
342}
343
344static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345 unsigned CheckedOpcode) {
346 return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347}
348
349/// Chooses the correct key for scheduling data. If \p Op has the same (or
350/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
351/// OpValue.
352static Value *isOneOf(Value *OpValue, Value *Op) {
353 auto *I = dyn_cast<Instruction>(Op);
354 if (!I)
355 return OpValue;
356 auto *OpInst = cast<Instruction>(OpValue);
357 unsigned OpInstOpcode = OpInst->getOpcode();
358 unsigned IOpcode = I->getOpcode();
359 if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
360 return Op;
361 return OpValue;
362}
363
364namespace {
365
366/// Contains data for the instructions going to be vectorized.
367struct RawInstructionsData {
368 /// Main Opcode of the instructions going to be vectorized.
369 unsigned Opcode = 0;
370
371 /// The list of instructions have some instructions with alternate opcodes.
372 bool HasAltOpcodes = false;
373};
374
375} // end anonymous namespace
376
377/// Checks the list of the vectorized instructions \p VL and returns info about
378/// this list.
379static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380 auto *I0 = dyn_cast<Instruction>(VL[0]);
381 if (!I0)
382 return {};
383 RawInstructionsData Res;
384 unsigned Opcode = I0->getOpcode();
385 // Walk through the list of the vectorized instructions
386 // in order to check its structure described by RawInstructionsData.
387 for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388 auto *I = dyn_cast<Instruction>(VL[Cnt]);
389 if (!I)
390 return {};
391 if (Opcode != I->getOpcode())
392 Res.HasAltOpcodes = true;
393 }
394 Res.Opcode = Opcode;
395 return Res;
396}
397
398namespace {
399
400/// Main data required for vectorization of instructions.
401struct InstructionsState {
402 /// The very first instruction in the list with the main opcode.
403 Value *OpValue = nullptr;
404
405 /// The main opcode for the list of instructions.
406 unsigned Opcode = 0;
407
408 /// Some of the instructions in the list have alternate opcodes.
409 bool IsAltShuffle = false;
410
411 InstructionsState() = default;
412 InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413 : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
414};
415
416} // end anonymous namespace
417
418/// \returns analysis of the Instructions in \p VL described in
419/// InstructionsState, the Opcode that we suppose the whole list
420/// could be vectorized even if its structure is diverse.
421static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422 auto Res = getMainOpcode(VL);
423 unsigned Opcode = Res.Opcode;
424 if (!Res.HasAltOpcodes)
425 return InstructionsState(VL[0], Opcode, false);
426 auto *OpInst = cast<Instruction>(VL[0]);
427 unsigned AltOpcode = getAltOpcode(Opcode);
428 // Examine each element in the list instructions VL to determine
429 // if some operations there could be considered as an alternative
430 // (for example as subtraction relates to addition operation).
431 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432 auto *I = cast<Instruction>(VL[Cnt]);
433 unsigned InstOpcode = I->getOpcode();
434 if ((Res.HasAltOpcodes &&
435 InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436 (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437 return InstructionsState(OpInst, 0, false);
438 }
439 }
440 return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
441}
442
443/// \returns true if all of the values in \p VL have the same type or false
444/// otherwise.
445static bool allSameType(ArrayRef<Value *> VL) {
446 Type *Ty = VL[0]->getType();
447 for (int i = 1, e = VL.size(); i < e; i++)
448 if (VL[i]->getType() != Ty)
449 return false;
450
451 return true;
452}
453
454/// \returns True if Extract{Value,Element} instruction extracts element Idx.
455static Optional<unsigned> getExtractIndex(Instruction *E) {
456 unsigned Opcode = E->getOpcode();
457 assert((Opcode == Instruction::ExtractElement ||(static_cast <bool> ((Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? void (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 459, __extension__ __PRETTY_FUNCTION__))
458 Opcode == Instruction::ExtractValue) &&(static_cast <bool> ((Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? void (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 459, __extension__ __PRETTY_FUNCTION__))
459 "Expected extractelement or extractvalue instruction.")(static_cast <bool> ((Opcode == Instruction::ExtractElement
|| Opcode == Instruction::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? void (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 459, __extension__ __PRETTY_FUNCTION__))
;
460 if (Opcode == Instruction::ExtractElement) {
461 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
462 if (!CI)
463 return None;
464 return CI->getZExtValue();
465 }
466 ExtractValueInst *EI = cast<ExtractValueInst>(E);
467 if (EI->getNumIndices() != 1)
468 return None;
469 return *EI->idx_begin();
470}
471
472/// \returns True if in-tree use also needs extract. This refers to
473/// possible scalar operand in vectorized instruction.
474static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
475 TargetLibraryInfo *TLI) {
476 unsigned Opcode = UserInst->getOpcode();
477 switch (Opcode) {
478 case Instruction::Load: {
479 LoadInst *LI = cast<LoadInst>(UserInst);
480 return (LI->getPointerOperand() == Scalar);
481 }
482 case Instruction::Store: {
483 StoreInst *SI = cast<StoreInst>(UserInst);
484 return (SI->getPointerOperand() == Scalar);
485 }
486 case Instruction::Call: {
487 CallInst *CI = cast<CallInst>(UserInst);
488 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
489 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
490 return (CI->getArgOperand(1) == Scalar);
491 }
492 LLVM_FALLTHROUGH[[clang::fallthrough]];
493 }
494 default:
495 return false;
496 }
497}
498
499/// \returns the AA location that is being access by the instruction.
500static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
501 if (StoreInst *SI = dyn_cast<StoreInst>(I))
502 return MemoryLocation::get(SI);
503 if (LoadInst *LI = dyn_cast<LoadInst>(I))
504 return MemoryLocation::get(LI);
505 return MemoryLocation();
506}
507
508/// \returns True if the instruction is not a volatile or atomic load/store.
509static bool isSimple(Instruction *I) {
510 if (LoadInst *LI = dyn_cast<LoadInst>(I))
511 return LI->isSimple();
512 if (StoreInst *SI = dyn_cast<StoreInst>(I))
513 return SI->isSimple();
514 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
515 return !MI->isVolatile();
516 return true;
517}
518
519namespace llvm {
520
521namespace slpvectorizer {
522
523/// Bottom Up SLP Vectorizer.
524class BoUpSLP {
525public:
526 using ValueList = SmallVector<Value *, 8>;
527 using InstrList = SmallVector<Instruction *, 16>;
528 using ValueSet = SmallPtrSet<Value *, 16>;
529 using StoreList = SmallVector<StoreInst *, 8>;
530 using ExtraValueToDebugLocsMap =
531 MapVector<Value *, SmallVector<Instruction *, 2>>;
532
533 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
534 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
535 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
536 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
537 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
538 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
539 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
540 // Use the vector register size specified by the target unless overridden
541 // by a command-line option.
542 // TODO: It would be better to limit the vectorization factor based on
543 // data type rather than just register size. For example, x86 AVX has
544 // 256-bit registers, but it does not support integer operations
545 // at that width (that requires AVX2).
546 if (MaxVectorRegSizeOption.getNumOccurrences())
547 MaxVecRegSize = MaxVectorRegSizeOption;
548 else
549 MaxVecRegSize = TTI->getRegisterBitWidth(true);
550
551 if (MinVectorRegSizeOption.getNumOccurrences())
552 MinVecRegSize = MinVectorRegSizeOption;
553 else
554 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
555 }
556
557 /// \brief Vectorize the tree that starts with the elements in \p VL.
558 /// Returns the vectorized root.
559 Value *vectorizeTree();
560
561 /// Vectorize the tree but with the list of externally used values \p
562 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
563 /// generated extractvalue instructions.
564 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
565
566 /// \returns the cost incurred by unwanted spills and fills, caused by
567 /// holding live values over call sites.
568 int getSpillCost();
569
570 /// \returns the vectorization cost of the subtree that starts at \p VL.
571 /// A negative number means that this is profitable.
572 int getTreeCost();
573
574 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
576 void buildTree(ArrayRef<Value *> Roots,
577 ArrayRef<Value *> UserIgnoreLst = None);
578
579 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
580 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
581 /// into account (anf updating it, if required) list of externally used
582 /// values stored in \p ExternallyUsedValues.
583 void buildTree(ArrayRef<Value *> Roots,
584 ExtraValueToDebugLocsMap &ExternallyUsedValues,
585 ArrayRef<Value *> UserIgnoreLst = None);
586
587 /// Clear the internal data structures that are created by 'buildTree'.
588 void deleteTree() {
589 VectorizableTree.clear();
590 ScalarToTreeEntry.clear();
591 MustGather.clear();
592 ExternalUses.clear();
593 NumOpsWantToKeepOrder.clear();
594 NumOpsWantToKeepOriginalOrder = 0;
595 for (auto &Iter : BlocksSchedules) {
596 BlockScheduling *BS = Iter.second.get();
597 BS->clear();
598 }
599 MinBWs.clear();
600 }
601
602 unsigned getTreeSize() const { return VectorizableTree.size(); }
603
604 /// \brief Perform LICM and CSE on the newly generated gather sequences.
605 void optimizeGatherSequence();
606
607 /// \returns The best order of instructions for vectorization.
608 Optional<ArrayRef<unsigned>> bestOrder() const {
609 auto I = std::max_element(
610 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
611 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
612 const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
613 return D1.second < D2.second;
614 });
615 if (I == NumOpsWantToKeepOrder.end() ||
616 I->getSecond() <= NumOpsWantToKeepOriginalOrder)
617 return None;
618
619 return makeArrayRef(I->getFirst());
620 }
621
622 /// \return The vector element size in bits to use when vectorizing the
623 /// expression tree ending at \p V. If V is a store, the size is the width of
624 /// the stored value. Otherwise, the size is the width of the largest loaded
625 /// value reaching V. This method is used by the vectorizer to calculate
626 /// vectorization factors.
627 unsigned getVectorElementSize(Value *V);
628
629 /// Compute the minimum type sizes required to represent the entries in a
630 /// vectorizable tree.
631 void computeMinimumValueSizes();
632
633 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
634 unsigned getMaxVecRegSize() const {
635 return MaxVecRegSize;
636 }
637
638 // \returns minimum vector register size as set by cl::opt.
639 unsigned getMinVecRegSize() const {
640 return MinVecRegSize;
641 }
642
643 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
644 ///
645 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
646 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
647
648 /// \returns True if the VectorizableTree is both tiny and not fully
649 /// vectorizable. We do not vectorize such trees.
650 bool isTreeTinyAndNotFullyVectorizable();
651
652 OptimizationRemarkEmitter *getORE() { return ORE; }
653
654private:
655 struct TreeEntry;
656
657 /// Checks if all users of \p I are the part of the vectorization tree.
658 bool areAllUsersVectorized(Instruction *I) const;
659
660 /// \returns the cost of the vectorizable entry.
661 int getEntryCost(TreeEntry *E);
662
663 /// This is the recursive part of buildTree.
664 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
665
666 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
667 /// be vectorized to use the original vector (or aggregate "bitcast" to a
668 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
669 /// returns false, setting \p CurrentOrder to either an empty vector or a
670 /// non-identity permutation that allows to reuse extract instructions.
671 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
672 SmallVectorImpl<unsigned> &CurrentOrder) const;
673
674 /// Vectorize a single entry in the tree.
675 Value *vectorizeTree(TreeEntry *E);
676
677 /// Vectorize a single entry in the tree, starting in \p VL.
678 Value *vectorizeTree(ArrayRef<Value *> VL);
679
680 /// \returns the scalarization cost for this type. Scalarization in this
681 /// context means the creation of vectors from a group of scalars.
682 int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
683
684 /// \returns the scalarization cost for this list of values. Assuming that
685 /// this subtree gets vectorized, we may need to extract the values from the
686 /// roots. This method calculates the cost of extracting the values.
687 int getGatherCost(ArrayRef<Value *> VL);
688
689 /// \brief Set the Builder insert point to one after the last instruction in
690 /// the bundle
691 void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
692
693 /// \returns a vector from a collection of scalars in \p VL.
694 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
695
696 /// \returns whether the VectorizableTree is fully vectorizable and will
697 /// be beneficial even the tree height is tiny.
698 bool isFullyVectorizableTinyTree();
699
700 /// \reorder commutative operands in alt shuffle if they result in
701 /// vectorized code.
702 void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
703 SmallVectorImpl<Value *> &Left,
704 SmallVectorImpl<Value *> &Right);
705
706 /// \reorder commutative operands to get better probability of
707 /// generating vectorized code.
708 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
709 SmallVectorImpl<Value *> &Left,
710 SmallVectorImpl<Value *> &Right);
711 struct TreeEntry {
712 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
713
714 /// \returns true if the scalars in VL are equal to this entry.
715 bool isSame(ArrayRef<Value *> VL) const {
716 if (VL.size() == Scalars.size())
717 return std::equal(VL.begin(), VL.end(), Scalars.begin());
718 return VL.size() == ReuseShuffleIndices.size() &&
719 std::equal(
720 VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
721 [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
722 }
723
724 /// A vector of scalars.
725 ValueList Scalars;
726
727 /// The Scalars are vectorized into this value. It is initialized to Null.
728 Value *VectorizedValue = nullptr;
729
730 /// Do we need to gather this sequence ?
731 bool NeedToGather = false;
732
733 /// Does this sequence require some shuffling?
734 SmallVector<unsigned, 4> ReuseShuffleIndices;
735
736 /// Does this entry require reordering?
737 ArrayRef<unsigned> ReorderIndices;
738
739 /// Points back to the VectorizableTree.
740 ///
741 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
742 /// to be a pointer and needs to be able to initialize the child iterator.
743 /// Thus we need a reference back to the container to translate the indices
744 /// to entries.
745 std::vector<TreeEntry> &Container;
746
747 /// The TreeEntry index containing the user of this entry. We can actually
748 /// have multiple users so the data structure is not truly a tree.
749 SmallVector<int, 1> UserTreeIndices;
750 };
751
752 /// Create a new VectorizableTree entry.
753 void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
754 ArrayRef<unsigned> ReuseShuffleIndices = None,
755 ArrayRef<unsigned> ReorderIndices = None) {
756 VectorizableTree.emplace_back(VectorizableTree);
757 int idx = VectorizableTree.size() - 1;
758 TreeEntry *Last = &VectorizableTree[idx];
759 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
760 Last->NeedToGather = !Vectorized;
761 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
762 ReuseShuffleIndices.end());
763 Last->ReorderIndices = ReorderIndices;
764 if (Vectorized) {
765 for (int i = 0, e = VL.size(); i != e; ++i) {
766 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!")(static_cast <bool> (!getTreeEntry(VL[i]) && "Scalar already in tree!"
) ? void (0) : __assert_fail ("!getTreeEntry(VL[i]) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 766, __extension__ __PRETTY_FUNCTION__))
;
767 ScalarToTreeEntry[VL[i]] = idx;
768 }
769 } else {
770 MustGather.insert(VL.begin(), VL.end());
771 }
772
773 if (UserTreeIdx >= 0)
774 Last->UserTreeIndices.push_back(UserTreeIdx);
775 UserTreeIdx = idx;
776 }
777
778 /// -- Vectorization State --
779 /// Holds all of the tree entries.
780 std::vector<TreeEntry> VectorizableTree;
781
782 TreeEntry *getTreeEntry(Value *V) {
783 auto I = ScalarToTreeEntry.find(V);
784 if (I != ScalarToTreeEntry.end())
785 return &VectorizableTree[I->second];
786 return nullptr;
787 }
788
789 /// Maps a specific scalar to its tree entry.
790 SmallDenseMap<Value*, int> ScalarToTreeEntry;
791
792 /// A list of scalars that we found that we need to keep as scalars.
793 ValueSet MustGather;
794
795 /// This POD struct describes one external user in the vectorized tree.
796 struct ExternalUser {
797 ExternalUser(Value *S, llvm::User *U, int L)
798 : Scalar(S), User(U), Lane(L) {}
799
800 // Which scalar in our function.
801 Value *Scalar;
802
803 // Which user that uses the scalar.
804 llvm::User *User;
805
806 // Which lane does the scalar belong to.
807 int Lane;
808 };
809 using UserList = SmallVector<ExternalUser, 16>;
810
811 /// Checks if two instructions may access the same memory.
812 ///
813 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
814 /// is invariant in the calling loop.
815 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
816 Instruction *Inst2) {
817 // First check if the result is already in the cache.
818 AliasCacheKey key = std::make_pair(Inst1, Inst2);
819 Optional<bool> &result = AliasCache[key];
820 if (result.hasValue()) {
821 return result.getValue();
822 }
823 MemoryLocation Loc2 = getLocation(Inst2, AA);
824 bool aliased = true;
825 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
826 // Do the alias check.
827 aliased = AA->alias(Loc1, Loc2);
828 }
829 // Store the result in the cache.
830 result = aliased;
831 return aliased;
832 }
833
834 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
835
836 /// Cache for alias results.
837 /// TODO: consider moving this to the AliasAnalysis itself.
838 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
839
840 /// Removes an instruction from its block and eventually deletes it.
841 /// It's like Instruction::eraseFromParent() except that the actual deletion
842 /// is delayed until BoUpSLP is destructed.
843 /// This is required to ensure that there are no incorrect collisions in the
844 /// AliasCache, which can happen if a new instruction is allocated at the
845 /// same address as a previously deleted instruction.
846 void eraseInstruction(Instruction *I) {
847 I->removeFromParent();
848 I->dropAllReferences();
849 DeletedInstructions.emplace_back(I);
850 }
851
852 /// Temporary store for deleted instructions. Instructions will be deleted
853 /// eventually when the BoUpSLP is destructed.
854 SmallVector<unique_value, 8> DeletedInstructions;
855
856 /// A list of values that need to extracted out of the tree.
857 /// This list holds pairs of (Internal Scalar : External User). External User
858 /// can be nullptr, it means that this Internal Scalar will be used later,
859 /// after vectorization.
860 UserList ExternalUses;
861
862 /// Values used only by @llvm.assume calls.
863 SmallPtrSet<const Value *, 32> EphValues;
864
865 /// Holds all of the instructions that we gathered.
866 SetVector<Instruction *> GatherSeq;
867
868 /// A list of blocks that we are going to CSE.
869 SetVector<BasicBlock *> CSEBlocks;
870
871 /// Contains all scheduling relevant data for an instruction.
872 /// A ScheduleData either represents a single instruction or a member of an
873 /// instruction bundle (= a group of instructions which is combined into a
874 /// vector instruction).
875 struct ScheduleData {
876 // The initial value for the dependency counters. It means that the
877 // dependencies are not calculated yet.
878 enum { InvalidDeps = -1 };
879
880 ScheduleData() = default;
881
882 void init(int BlockSchedulingRegionID, Value *OpVal) {
883 FirstInBundle = this;
884 NextInBundle = nullptr;
885 NextLoadStore = nullptr;
886 IsScheduled = false;
887 SchedulingRegionID = BlockSchedulingRegionID;
888 UnscheduledDepsInBundle = UnscheduledDeps;
889 clearDependencies();
890 OpValue = OpVal;
891 }
892
893 /// Returns true if the dependency information has been calculated.
894 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
895
896 /// Returns true for single instructions and for bundle representatives
897 /// (= the head of a bundle).
898 bool isSchedulingEntity() const { return FirstInBundle == this; }
899
900 /// Returns true if it represents an instruction bundle and not only a
901 /// single instruction.
902 bool isPartOfBundle() const {
903 return NextInBundle != nullptr || FirstInBundle != this;
904 }
905
906 /// Returns true if it is ready for scheduling, i.e. it has no more
907 /// unscheduled depending instructions/bundles.
908 bool isReady() const {
909 assert(isSchedulingEntity() &&(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 910, __extension__ __PRETTY_FUNCTION__))
910 "can't consider non-scheduling entity for ready list")(static_cast <bool> (isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? void (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 910, __extension__ __PRETTY_FUNCTION__))
;
911 return UnscheduledDepsInBundle == 0 && !IsScheduled;
912 }
913
914 /// Modifies the number of unscheduled dependencies, also updating it for
915 /// the whole bundle.
916 int incrementUnscheduledDeps(int Incr) {
917 UnscheduledDeps += Incr;
918 return FirstInBundle->UnscheduledDepsInBundle += Incr;
919 }
920
921 /// Sets the number of unscheduled dependencies to the number of
922 /// dependencies.
923 void resetUnscheduledDeps() {
924 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
925 }
926
927 /// Clears all dependency information.
928 void clearDependencies() {
929 Dependencies = InvalidDeps;
930 resetUnscheduledDeps();
931 MemoryDependencies.clear();
932 }
933
934 void dump(raw_ostream &os) const {
935 if (!isSchedulingEntity()) {
936 os << "/ " << *Inst;
937 } else if (NextInBundle) {
938 os << '[' << *Inst;
939 ScheduleData *SD = NextInBundle;
940 while (SD) {
941 os << ';' << *SD->Inst;
942 SD = SD->NextInBundle;
943 }
944 os << ']';
945 } else {
946 os << *Inst;
947 }
948 }
949
950 Instruction *Inst = nullptr;
951
952 /// Points to the head in an instruction bundle (and always to this for
953 /// single instructions).
954 ScheduleData *FirstInBundle = nullptr;
955
956 /// Single linked list of all instructions in a bundle. Null if it is a
957 /// single instruction.
958 ScheduleData *NextInBundle = nullptr;
959
960 /// Single linked list of all memory instructions (e.g. load, store, call)
961 /// in the block - until the end of the scheduling region.
962 ScheduleData *NextLoadStore = nullptr;
963
964 /// The dependent memory instructions.
965 /// This list is derived on demand in calculateDependencies().
966 SmallVector<ScheduleData *, 4> MemoryDependencies;
967
968 /// This ScheduleData is in the current scheduling region if this matches
969 /// the current SchedulingRegionID of BlockScheduling.
970 int SchedulingRegionID = 0;
971
972 /// Used for getting a "good" final ordering of instructions.
973 int SchedulingPriority = 0;
974
975 /// The number of dependencies. Constitutes of the number of users of the
976 /// instruction plus the number of dependent memory instructions (if any).
977 /// This value is calculated on demand.
978 /// If InvalidDeps, the number of dependencies is not calculated yet.
979 int Dependencies = InvalidDeps;
980
981 /// The number of dependencies minus the number of dependencies of scheduled
982 /// instructions. As soon as this is zero, the instruction/bundle gets ready
983 /// for scheduling.
984 /// Note that this is negative as long as Dependencies is not calculated.
985 int UnscheduledDeps = InvalidDeps;
986
987 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
988 /// single instructions.
989 int UnscheduledDepsInBundle = InvalidDeps;
990
991 /// True if this instruction is scheduled (or considered as scheduled in the
992 /// dry-run).
993 bool IsScheduled = false;
994
995 /// Opcode of the current instruction in the schedule data.
996 Value *OpValue = nullptr;
997 };
998
999#ifndef NDEBUG
1000 friend inline raw_ostream &operator<<(raw_ostream &os,
1001 const BoUpSLP::ScheduleData &SD) {
1002 SD.dump(os);
1003 return os;
1004 }
1005#endif
1006
1007 friend struct GraphTraits<BoUpSLP *>;
1008 friend struct DOTGraphTraits<BoUpSLP *>;
1009
1010 /// Contains all scheduling data for a basic block.
1011 struct BlockScheduling {
1012 BlockScheduling(BasicBlock *BB)
1013 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
1014
1015 void clear() {
1016 ReadyInsts.clear();
1017 ScheduleStart = nullptr;
1018 ScheduleEnd = nullptr;
1019 FirstLoadStoreInRegion = nullptr;
1020 LastLoadStoreInRegion = nullptr;
1021
1022 // Reduce the maximum schedule region size by the size of the
1023 // previous scheduling run.
1024 ScheduleRegionSizeLimit -= ScheduleRegionSize;
1025 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1026 ScheduleRegionSizeLimit = MinScheduleRegionSize;
1027 ScheduleRegionSize = 0;
1028
1029 // Make a new scheduling region, i.e. all existing ScheduleData is not
1030 // in the new region yet.
1031 ++SchedulingRegionID;
1032 }
1033
1034 ScheduleData *getScheduleData(Value *V) {
1035 ScheduleData *SD = ScheduleDataMap[V];
1036 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1037 return SD;
1038 return nullptr;
1039 }
1040
1041 ScheduleData *getScheduleData(Value *V, Value *Key) {
1042 if (V == Key)
1043 return getScheduleData(V);
1044 auto I = ExtraScheduleDataMap.find(V);
1045 if (I != ExtraScheduleDataMap.end()) {
1046 ScheduleData *SD = I->second[Key];
1047 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1048 return SD;
1049 }
1050 return nullptr;
1051 }
1052
1053 bool isInSchedulingRegion(ScheduleData *SD) {
1054 return SD->SchedulingRegionID == SchedulingRegionID;
1055 }
1056
1057 /// Marks an instruction as scheduled and puts all dependent ready
1058 /// instructions into the ready-list.
1059 template <typename ReadyListType>
1060 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1061 SD->IsScheduled = true;
1062 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
1063
1064 ScheduleData *BundleMember = SD;
1065 while (BundleMember) {
1066 if (BundleMember->Inst != BundleMember->OpValue) {
1067 BundleMember = BundleMember->NextInBundle;
1068 continue;
1069 }
1070 // Handle the def-use chain dependencies.
1071 for (Use &U : BundleMember->Inst->operands()) {
1072 auto *I = dyn_cast<Instruction>(U.get());
1073 if (!I)
1074 continue;
1075 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1076 if (OpDef && OpDef->hasValidDependencies() &&
1077 OpDef->incrementUnscheduledDeps(-1) == 0) {
1078 // There are no more unscheduled dependencies after
1079 // decrementing, so we can put the dependent instruction
1080 // into the ready list.
1081 ScheduleData *DepBundle = OpDef->FirstInBundle;
1082 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1083, __extension__ __PRETTY_FUNCTION__))
1083 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1083, __extension__ __PRETTY_FUNCTION__))
;
1084 ReadyList.insert(DepBundle);
1085 DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
1086 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
1087 }
1088 });
1089 }
1090 // Handle the memory dependencies.
1091 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1092 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1093 // There are no more unscheduled dependencies after decrementing,
1094 // so we can put the dependent instruction into the ready list.
1095 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1096 assert(!DepBundle->IsScheduled &&(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1097, __extension__ __PRETTY_FUNCTION__))
1097 "already scheduled bundle gets ready")(static_cast <bool> (!DepBundle->IsScheduled &&
"already scheduled bundle gets ready") ? void (0) : __assert_fail
("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1097, __extension__ __PRETTY_FUNCTION__))
;
1098 ReadyList.insert(DepBundle);
1099 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundledo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
1100 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
1101 }
1102 }
1103 BundleMember = BundleMember->NextInBundle;
1104 }
1105 }
1106
1107 void doForAllOpcodes(Value *V,
1108 function_ref<void(ScheduleData *SD)> Action) {
1109 if (ScheduleData *SD = getScheduleData(V))
1110 Action(SD);
1111 auto I = ExtraScheduleDataMap.find(V);
1112 if (I != ExtraScheduleDataMap.end())
1113 for (auto &P : I->second)
1114 if (P.second->SchedulingRegionID == SchedulingRegionID)
1115 Action(P.second);
1116 }
1117
1118 /// Put all instructions into the ReadyList which are ready for scheduling.
1119 template <typename ReadyListType>
1120 void initialFillReadyList(ReadyListType &ReadyList) {
1121 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1122 doForAllOpcodes(I, [&](ScheduleData *SD) {
1123 if (SD->isSchedulingEntity() && SD->isReady()) {
1124 ReadyList.insert(SD);
1125 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
1126 }
1127 });
1128 }
1129 }
1130
1131 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1132 /// cyclic dependencies. This is only a dry-run, no instructions are
1133 /// actually moved at this stage.
1134 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1135
1136 /// Un-bundles a group of instructions.
1137 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1138
1139 /// Allocates schedule data chunk.
1140 ScheduleData *allocateScheduleDataChunks();
1141
1142 /// Extends the scheduling region so that V is inside the region.
1143 /// \returns true if the region size is within the limit.
1144 bool extendSchedulingRegion(Value *V, Value *OpValue);
1145
1146 /// Initialize the ScheduleData structures for new instructions in the
1147 /// scheduling region.
1148 void initScheduleData(Instruction *FromI, Instruction *ToI,
1149 ScheduleData *PrevLoadStore,
1150 ScheduleData *NextLoadStore);
1151
1152 /// Updates the dependency information of a bundle and of all instructions/
1153 /// bundles which depend on the original bundle.
1154 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1155 BoUpSLP *SLP);
1156
1157 /// Sets all instruction in the scheduling region to un-scheduled.
1158 void resetSchedule();
1159
1160 BasicBlock *BB;
1161
1162 /// Simple memory allocation for ScheduleData.
1163 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1164
1165 /// The size of a ScheduleData array in ScheduleDataChunks.
1166 int ChunkSize;
1167
1168 /// The allocator position in the current chunk, which is the last entry
1169 /// of ScheduleDataChunks.
1170 int ChunkPos;
1171
1172 /// Attaches ScheduleData to Instruction.
1173 /// Note that the mapping survives during all vectorization iterations, i.e.
1174 /// ScheduleData structures are recycled.
1175 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1176
1177 /// Attaches ScheduleData to Instruction with the leading key.
1178 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1179 ExtraScheduleDataMap;
1180
1181 struct ReadyList : SmallVector<ScheduleData *, 8> {
1182 void insert(ScheduleData *SD) { push_back(SD); }
1183 };
1184
1185 /// The ready-list for scheduling (only used for the dry-run).
1186 ReadyList ReadyInsts;
1187
1188 /// The first instruction of the scheduling region.
1189 Instruction *ScheduleStart = nullptr;
1190
1191 /// The first instruction _after_ the scheduling region.
1192 Instruction *ScheduleEnd = nullptr;
1193
1194 /// The first memory accessing instruction in the scheduling region
1195 /// (can be null).
1196 ScheduleData *FirstLoadStoreInRegion = nullptr;
1197
1198 /// The last memory accessing instruction in the scheduling region
1199 /// (can be null).
1200 ScheduleData *LastLoadStoreInRegion = nullptr;
1201
1202 /// The current size of the scheduling region.
1203 int ScheduleRegionSize = 0;
1204
1205 /// The maximum size allowed for the scheduling region.
1206 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1207
1208 /// The ID of the scheduling region. For a new vectorization iteration this
1209 /// is incremented which "removes" all ScheduleData from the region.
1210 // Make sure that the initial SchedulingRegionID is greater than the
1211 // initial SchedulingRegionID in ScheduleData (which is 0).
1212 int SchedulingRegionID = 1;
1213 };
1214
1215 /// Attaches the BlockScheduling structures to basic blocks.
1216 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1217
1218 /// Performs the "real" scheduling. Done before vectorization is actually
1219 /// performed in a basic block.
1220 void scheduleBlock(BlockScheduling *BS);
1221
1222 /// List of users to ignore during scheduling and that don't need extracting.
1223 ArrayRef<Value *> UserIgnoreList;
1224
1225 using OrdersType = SmallVector<unsigned, 4>;
1226 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
1227 /// sorted SmallVectors of unsigned.
1228 struct OrdersTypeDenseMapInfo {
1229 static OrdersType getEmptyKey() {
1230 OrdersType V;
1231 V.push_back(~1U);
1232 return V;
1233 }
1234
1235 static OrdersType getTombstoneKey() {
1236 OrdersType V;
1237 V.push_back(~2U);
1238 return V;
1239 }
1240
1241 static unsigned getHashValue(const OrdersType &V) {
1242 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1243 }
1244
1245 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
1246 return LHS == RHS;
1247 }
1248 };
1249
1250 /// Contains orders of operations along with the number of bundles that have
1251 /// operations in this order. It stores only those orders that require
1252 /// reordering, if reordering is not required it is counted using \a
1253 /// NumOpsWantToKeepOriginalOrder.
1254 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder;
1255 /// Number of bundles that do not require reordering.
1256 unsigned NumOpsWantToKeepOriginalOrder = 0;
1257
1258 // Analysis and block reference.
1259 Function *F;
1260 ScalarEvolution *SE;
1261 TargetTransformInfo *TTI;
1262 TargetLibraryInfo *TLI;
1263 AliasAnalysis *AA;
1264 LoopInfo *LI;
1265 DominatorTree *DT;
1266 AssumptionCache *AC;
1267 DemandedBits *DB;
1268 const DataLayout *DL;
1269 OptimizationRemarkEmitter *ORE;
1270
1271 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1272 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1273
1274 /// Instruction builder to construct the vectorized tree.
1275 IRBuilder<> Builder;
1276
1277 /// A map of scalar integer values to the smallest bit width with which they
1278 /// can legally be represented. The values map to (width, signed) pairs,
1279 /// where "width" indicates the minimum bit width and "signed" is True if the
1280 /// value must be signed-extended, rather than zero-extended, back to its
1281 /// original width.
1282 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1283};
1284
1285} // end namespace slpvectorizer
1286
1287template <> struct GraphTraits<BoUpSLP *> {
1288 using TreeEntry = BoUpSLP::TreeEntry;
1289
1290 /// NodeRef has to be a pointer per the GraphWriter.
1291 using NodeRef = TreeEntry *;
1292
1293 /// \brief Add the VectorizableTree to the index iterator to be able to return
1294 /// TreeEntry pointers.
1295 struct ChildIteratorType
1296 : public iterator_adaptor_base<ChildIteratorType,
1297 SmallVector<int, 1>::iterator> {
1298 std::vector<TreeEntry> &VectorizableTree;
1299
1300 ChildIteratorType(SmallVector<int, 1>::iterator W,
1301 std::vector<TreeEntry> &VT)
1302 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1303
1304 NodeRef operator*() { return &VectorizableTree[*I]; }
1305 };
1306
1307 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1308
1309 static ChildIteratorType child_begin(NodeRef N) {
1310 return {N->UserTreeIndices.begin(), N->Container};
1311 }
1312
1313 static ChildIteratorType child_end(NodeRef N) {
1314 return {N->UserTreeIndices.end(), N->Container};
1315 }
1316
1317 /// For the node iterator we just need to turn the TreeEntry iterator into a
1318 /// TreeEntry* iterator so that it dereferences to NodeRef.
1319 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1320
1321 static nodes_iterator nodes_begin(BoUpSLP *R) {
1322 return nodes_iterator(R->VectorizableTree.begin());
1323 }
1324
1325 static nodes_iterator nodes_end(BoUpSLP *R) {
1326 return nodes_iterator(R->VectorizableTree.end());
1327 }
1328
1329 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1330};
1331
1332template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1333 using TreeEntry = BoUpSLP::TreeEntry;
1334
1335 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1336
1337 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1338 std::string Str;
1339 raw_string_ostream OS(Str);
1340 if (isSplat(Entry->Scalars)) {
1341 OS << "<splat> " << *Entry->Scalars[0];
1342 return Str;
1343 }
1344 for (auto V : Entry->Scalars) {
1345 OS << *V;
1346 if (std::any_of(
1347 R->ExternalUses.begin(), R->ExternalUses.end(),
1348 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1349 OS << " <extract>";
1350 OS << "\n";
1351 }
1352 return Str;
1353 }
1354
1355 static std::string getNodeAttributes(const TreeEntry *Entry,
1356 const BoUpSLP *) {
1357 if (Entry->NeedToGather)
1358 return "color=red";
1359 return "";
1360 }
1361};
1362
1363} // end namespace llvm
1364
1365void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1366 ArrayRef<Value *> UserIgnoreLst) {
1367 ExtraValueToDebugLocsMap ExternallyUsedValues;
1368 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1369}
1370
1371void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1372 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1373 ArrayRef<Value *> UserIgnoreLst) {
1374 deleteTree();
1375 UserIgnoreList = UserIgnoreLst;
1376 if (!allSameType(Roots))
1377 return;
1378 buildTree_rec(Roots, 0, -1);
1379
1380 // Collect the values that we need to extract from the tree.
1381 for (TreeEntry &EIdx : VectorizableTree) {
1382 TreeEntry *Entry = &EIdx;
1383
1384 // No need to handle users of gathered values.
1385 if (Entry->NeedToGather)
1386 continue;
1387
1388 // For each lane:
1389 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1390 Value *Scalar = Entry->Scalars[Lane];
1391 int FoundLane = Lane;
1392 if (!Entry->ReuseShuffleIndices.empty()) {
1393 FoundLane =
1394 std::distance(Entry->ReuseShuffleIndices.begin(),
1395 llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1396 }
1397
1398 // Check if the scalar is externally used as an extra arg.
1399 auto ExtI = ExternallyUsedValues.find(Scalar);
1400 if (ExtI != ExternallyUsedValues.end()) {
1401 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
1402 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
1403 ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1404 }
1405 for (User *U : Scalar->users()) {
1406 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
1407
1408 Instruction *UserInst = dyn_cast<Instruction>(U);
1409 if (!UserInst)
1410 continue;
1411
1412 // Skip in-tree scalars that become vectors
1413 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1414 Value *UseScalar = UseEntry->Scalars[0];
1415 // Some in-tree scalars will remain as scalar in vectorized
1416 // instructions. If that is the case, the one in Lane 0 will
1417 // be used.
1418 if (UseScalar != U ||
1419 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1420 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1421 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
1422 assert(!UseEntry->NeedToGather && "Bad state")(static_cast <bool> (!UseEntry->NeedToGather &&
"Bad state") ? void (0) : __assert_fail ("!UseEntry->NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1422, __extension__ __PRETTY_FUNCTION__))
;
1423 continue;
1424 }
1425 }
1426
1427 // Ignore users in the user ignore list.
1428 if (is_contained(UserIgnoreList, UserInst))
1429 continue;
1430
1431 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1432 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
1433 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1434 }
1435 }
1436 }
1437}
1438
1439void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1440 int UserTreeIdx) {
1441 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(static_cast <bool> ((allConstant(VL) || allSameType(VL
)) && "Invalid types!") ? void (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1441, __extension__ __PRETTY_FUNCTION__))
;
1442
1443 InstructionsState S = getSameOpcode(VL);
1444 if (Depth == RecursionMaxDepth) {
1
Taking false branch
1445 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
1446 newTreeEntry(VL, false, UserTreeIdx);
1447 return;
1448 }
1449
1450 // Don't handle vectors.
1451 if (S.OpValue->getType()->isVectorTy()) {
2
Taking false branch
1452 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
1453 newTreeEntry(VL, false, UserTreeIdx);
1454 return;
1455 }
1456
1457 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
3
Taking false branch
1458 if (SI->getValueOperand()->getType()->isVectorTy()) {
1459 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1460 newTreeEntry(VL, false, UserTreeIdx);
1461 return;
1462 }
1463
1464 // If all of the operands are identical or constant we have a simple solution.
1465 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
4
Assuming the condition is false
5
Assuming the condition is false
6
Assuming the condition is false
7
Assuming the condition is false
8
Taking false branch
1466 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1467 newTreeEntry(VL, false, UserTreeIdx);
1468 return;
1469 }
1470
1471 // We now know that this is a vector of instructions of the same type from
1472 // the same block.
1473
1474 // Don't vectorize ephemeral values.
1475 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
9
Assuming 'i' is equal to 'e'
10
Loop condition is false. Execution continues on line 1485
1476 if (EphValues.count(VL[i])) {
1477 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1478 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1479 newTreeEntry(VL, false, UserTreeIdx);
1480 return;
1481 }
1482 }
1483
1484 // Check if this is a duplicate of another entry.
1485 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
11
Taking false branch
1486 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*S.OpValue << ".\n"; } } while (false)
;
1487 if (!E->isSame(VL)) {
1488 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1489 newTreeEntry(VL, false, UserTreeIdx);
1490 return;
1491 }
1492 // Record the reuse of the tree node. FIXME, currently this is only used to
1493 // properly draw the graph rather than for the actual vectorization.
1494 E->UserTreeIndices.push_back(UserTreeIdx);
1495 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
;
1496 return;
1497 }
1498
1499 // Check that none of the instructions in the bundle are already in the tree.
1500 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
12
Assuming 'i' is equal to 'e'
13
Loop condition is false. Execution continues on line 1514
1501 auto *I = dyn_cast<Instruction>(VL[i]);
1502 if (!I)
1503 continue;
1504 if (getTreeEntry(I)) {
1505 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1506 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1507 newTreeEntry(VL, false, UserTreeIdx);
1508 return;
1509 }
1510 }
1511
1512 // If any of the scalars is marked as a value that needs to stay scalar, then
1513 // we need to gather the scalars.
1514 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
14
Assuming 'i' is equal to 'e'
15
Loop condition is false. Execution continues on line 1524
1515 if (MustGather.count(VL[i])) {
1516 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1517 newTreeEntry(VL, false, UserTreeIdx);
1518 return;
1519 }
1520 }
1521
1522 // Check that all of the users of the scalars that we want to vectorize are
1523 // schedulable.
1524 auto *VL0 = cast<Instruction>(S.OpValue);
1525 BasicBlock *BB = VL0->getParent();
1526
1527 if (!DT->isReachableFromEntry(BB)) {
16
Assuming the condition is false
17
Taking false branch
1528 // Don't go into unreachable blocks. They may contain instructions with
1529 // dependency cycles which confuse the final scheduling.
1530 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1531 newTreeEntry(VL, false, UserTreeIdx);
1532 return;
1533 }
1534
1535 // Check that every instruction appears once in this bundle.
1536 SmallVector<unsigned, 4> ReuseShuffleIndicies;
1537 SmallVector<Value *, 4> UniqueValues;
1538 DenseMap<Value *, unsigned> UniquePositions;
1539 for (Value *V : VL) {
18
Assuming '__begin1' is equal to '__end1'
1540 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1541 ReuseShuffleIndicies.emplace_back(Res.first->second);
1542 if (Res.second)
1543 UniqueValues.emplace_back(V);
1544 }
1545 if (UniqueValues.size() == VL.size()) {
19
Taking true branch
1546 ReuseShuffleIndicies.clear();
1547 } else {
1548 DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Shuffle for reused scalars.\n"
; } } while (false)
;
1549 if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1550 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1551 newTreeEntry(VL, false, UserTreeIdx);
1552 return;
1553 }
1554 VL = UniqueValues;
1555 }
1556
1557 auto &BSRef = BlocksSchedules[BB];
1558 if (!BSRef)
20
Taking false branch
1559 BSRef = llvm::make_unique<BlockScheduling>(BB);
1560
1561 BlockScheduling &BS = *BSRef.get();
1562
1563 if (!BS.tryScheduleBundle(VL, this, VL0)) {
21
Calling 'BlockScheduling::tryScheduleBundle'
1564 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1565 assert((!BS.getScheduleData(VL0) ||(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1567, __extension__ __PRETTY_FUNCTION__))
1566 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1567, __extension__ __PRETTY_FUNCTION__))
1567 "tryScheduleBundle should cancelScheduling on failure")(static_cast <bool> ((!BS.getScheduleData(VL0) || !BS.getScheduleData
(VL0)->isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? void (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1567, __extension__ __PRETTY_FUNCTION__))
;
1568 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1569 return;
1570 }
1571 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1572
1573 unsigned ShuffleOrOp = S.IsAltShuffle ?
1574 (unsigned) Instruction::ShuffleVector : S.Opcode;
1575 switch (ShuffleOrOp) {
1576 case Instruction::PHI: {
1577 PHINode *PH = dyn_cast<PHINode>(VL0);
1578
1579 // Check for terminator values (e.g. invoke).
1580 for (unsigned j = 0; j < VL.size(); ++j)
1581 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1582 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1583 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1584 if (Term) {
1585 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false)
;
1586 BS.cancelScheduling(VL, VL0);
1587 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1588 return;
1589 }
1590 }
1591
1592 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1593 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1594
1595 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1596 ValueList Operands;
1597 // Prepare the operand vector.
1598 for (Value *j : VL)
1599 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1600 PH->getIncomingBlock(i)));
1601
1602 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1603 }
1604 return;
1605 }
1606 case Instruction::ExtractValue:
1607 case Instruction::ExtractElement: {
1608 OrdersType CurrentOrder;
1609 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
1610 if (Reuse) {
1611 DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing or shuffling extract sequence.\n"
; } } while (false)
;
1612 ++NumOpsWantToKeepOriginalOrder;
1613 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1614 ReuseShuffleIndicies);
1615 return;
1616 }
1617 if (!CurrentOrder.empty()) {
1618 DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1619 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1620 "with order";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1621 for (unsigned Idx : CurrentOrder)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1622 dbgs() << " " << Idx;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1623 dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1624 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
;
1625 // Insert new order with initial value 0, if it does not exist,
1626 // otherwise return the iterator to the existing one.
1627 auto StoredCurrentOrderAndNum =
1628 NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1629 ++StoredCurrentOrderAndNum->getSecond();
1630 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx, ReuseShuffleIndicies,
1631 StoredCurrentOrderAndNum->getFirst());
1632 return;
1633 }
1634 DEBUG(dbgs() << "SLP: Gather extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gather extract sequence.\n";
} } while (false)
;
1635 newTreeEntry(VL, /*Vectorized=*/false, UserTreeIdx, ReuseShuffleIndicies);
1636 BS.cancelScheduling(VL, VL0);
1637 return;
1638 }
1639 case Instruction::Load: {
1640 // Check that a vectorized load would load the same memory as a scalar
1641 // load. For example, we don't want to vectorize loads that are smaller
1642 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1643 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1644 // from such a struct, we read/write packed bits disagreeing with the
1645 // unvectorized version.
1646 Type *ScalarTy = VL0->getType();
1647
1648 if (DL->getTypeSizeInBits(ScalarTy) !=
1649 DL->getTypeAllocSizeInBits(ScalarTy)) {
1650 BS.cancelScheduling(VL, VL0);
1651 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1652 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1653 return;
1654 }
1655
1656 // Make sure all loads in the bundle are simple - we can't vectorize
1657 // atomic or volatile loads.
1658 SmallVector<Value *, 4> PointerOps(VL.size());
1659 auto POIter = PointerOps.begin();
1660 for (Value *V : VL) {
1661 auto *L = cast<LoadInst>(V);
1662 if (!L->isSimple()) {
1663 BS.cancelScheduling(VL, VL0);
1664 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1665 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1666 return;
1667 }
1668 *POIter = L->getPointerOperand();
1669 ++POIter;
1670 }
1671
1672 OrdersType CurrentOrder;
1673 // Check the order of pointer operands.
1674 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
1675 Value *Ptr0;
1676 Value *PtrN;
1677 if (CurrentOrder.empty()) {
1678 Ptr0 = PointerOps.front();
1679 PtrN = PointerOps.back();
1680 } else {
1681 Ptr0 = PointerOps[CurrentOrder.front()];
1682 PtrN = PointerOps[CurrentOrder.back()];
1683 }
1684 const SCEV *Scev0 = SE->getSCEV(Ptr0);
1685 const SCEV *ScevN = SE->getSCEV(PtrN);
1686 const auto *Diff =
1687 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
1688 uint64_t Size = DL->getTypeAllocSize(ScalarTy);
1689 // Check that the sorted loads are consecutive.
1690 if (Diff && Diff->getAPInt().getZExtValue() == (VL.size() - 1) * Size) {
1691 if (CurrentOrder.empty()) {
1692 // Original loads are consecutive and does not require reordering.
1693 ++NumOpsWantToKeepOriginalOrder;
1694 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1695 ReuseShuffleIndicies);
1696 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1697 } else {
1698 // Need to reorder.
1699 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1700 ++I->getSecond();
1701 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1702 ReuseShuffleIndicies, I->getFirst());
1703 DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of jumbled loads.\n"
; } } while (false)
;
1704 }
1705 return;
1706 }
1707 }
1708
1709 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1710 BS.cancelScheduling(VL, VL0);
1711 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1712 return;
1713 }
1714 case Instruction::ZExt:
1715 case Instruction::SExt:
1716 case Instruction::FPToUI:
1717 case Instruction::FPToSI:
1718 case Instruction::FPExt:
1719 case Instruction::PtrToInt:
1720 case Instruction::IntToPtr:
1721 case Instruction::SIToFP:
1722 case Instruction::UIToFP:
1723 case Instruction::Trunc:
1724 case Instruction::FPTrunc:
1725 case Instruction::BitCast: {
1726 Type *SrcTy = VL0->getOperand(0)->getType();
1727 for (unsigned i = 0; i < VL.size(); ++i) {
1728 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1729 if (Ty != SrcTy || !isValidElementType(Ty)) {
1730 BS.cancelScheduling(VL, VL0);
1731 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1732 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1733 return;
1734 }
1735 }
1736 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1737 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1738
1739 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1740 ValueList Operands;
1741 // Prepare the operand vector.
1742 for (Value *j : VL)
1743 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1744
1745 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1746 }
1747 return;
1748 }
1749 case Instruction::ICmp:
1750 case Instruction::FCmp: {
1751 // Check that all of the compares have the same predicate.
1752 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1753 Type *ComparedTy = VL0->getOperand(0)->getType();
1754 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1755 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1756 if (Cmp->getPredicate() != P0 ||
1757 Cmp->getOperand(0)->getType() != ComparedTy) {
1758 BS.cancelScheduling(VL, VL0);
1759 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1760 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1761 return;
1762 }
1763 }
1764
1765 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1766 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1767
1768 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1769 ValueList Operands;
1770 // Prepare the operand vector.
1771 for (Value *j : VL)
1772 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1773
1774 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1775 }
1776 return;
1777 }
1778 case Instruction::Select:
1779 case Instruction::Add:
1780 case Instruction::FAdd:
1781 case Instruction::Sub:
1782 case Instruction::FSub:
1783 case Instruction::Mul:
1784 case Instruction::FMul:
1785 case Instruction::UDiv:
1786 case Instruction::SDiv:
1787 case Instruction::FDiv:
1788 case Instruction::URem:
1789 case Instruction::SRem:
1790 case Instruction::FRem:
1791 case Instruction::Shl:
1792 case Instruction::LShr:
1793 case Instruction::AShr:
1794 case Instruction::And:
1795 case Instruction::Or:
1796 case Instruction::Xor:
1797 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1798 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1799
1800 // Sort operands of the instructions so that each side is more likely to
1801 // have the same opcode.
1802 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1803 ValueList Left, Right;
1804 reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1805 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1806 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1807 return;
1808 }
1809
1810 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1811 ValueList Operands;
1812 // Prepare the operand vector.
1813 for (Value *j : VL)
1814 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1815
1816 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1817 }
1818 return;
1819
1820 case Instruction::GetElementPtr: {
1821 // We don't combine GEPs with complicated (nested) indexing.
1822 for (unsigned j = 0; j < VL.size(); ++j) {
1823 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1824 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1825 BS.cancelScheduling(VL, VL0);
1826 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1827 return;
1828 }
1829 }
1830
1831 // We can't combine several GEPs into one vector if they operate on
1832 // different types.
1833 Type *Ty0 = VL0->getOperand(0)->getType();
1834 for (unsigned j = 0; j < VL.size(); ++j) {
1835 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1836 if (Ty0 != CurTy) {
1837 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1838 BS.cancelScheduling(VL, VL0);
1839 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1840 return;
1841 }
1842 }
1843
1844 // We don't combine GEPs with non-constant indexes.
1845 for (unsigned j = 0; j < VL.size(); ++j) {
1846 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1847 if (!isa<ConstantInt>(Op)) {
1848 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1849 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1850 BS.cancelScheduling(VL, VL0);
1851 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1852 return;
1853 }
1854 }
1855
1856 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1857 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1858 for (unsigned i = 0, e = 2; i < e; ++i) {
1859 ValueList Operands;
1860 // Prepare the operand vector.
1861 for (Value *j : VL)
1862 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1863
1864 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1865 }
1866 return;
1867 }
1868 case Instruction::Store: {
1869 // Check if the stores are consecutive or of we need to swizzle them.
1870 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1871 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1872 BS.cancelScheduling(VL, VL0);
1873 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1874 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1875 return;
1876 }
1877
1878 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1879 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1880
1881 ValueList Operands;
1882 for (Value *j : VL)
1883 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1884
1885 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1886 return;
1887 }
1888 case Instruction::Call: {
1889 // Check if the calls are all to the same vectorizable intrinsic.
1890 CallInst *CI = cast<CallInst>(VL0);
1891 // Check if this is an Intrinsic call or something that can be
1892 // represented by an intrinsic call
1893 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1894 if (!isTriviallyVectorizable(ID)) {
1895 BS.cancelScheduling(VL, VL0);
1896 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1897 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1898 return;
1899 }
1900 Function *Int = CI->getCalledFunction();
1901 Value *A1I = nullptr;
1902 if (hasVectorInstrinsicScalarOpd(ID, 1))
1903 A1I = CI->getArgOperand(1);
1904 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1905 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1906 if (!CI2 || CI2->getCalledFunction() != Int ||
1907 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1908 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1909 BS.cancelScheduling(VL, VL0);
1910 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1911 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1912 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1913 return;
1914 }
1915 // ctlz,cttz and powi are special intrinsics whose second argument
1916 // should be same in order for them to be vectorized.
1917 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1918 Value *A1J = CI2->getArgOperand(1);
1919 if (A1I != A1J) {
1920 BS.cancelScheduling(VL, VL0);
1921 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1922 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1923 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1924 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
;
1925 return;
1926 }
1927 }
1928 // Verify that the bundle operands are identical between the two calls.
1929 if (CI->hasOperandBundles() &&
1930 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1931 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1932 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1933 BS.cancelScheduling(VL, VL0);
1934 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1935 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1936 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1937 return;
1938 }
1939 }
1940
1941 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1942 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1943 ValueList Operands;
1944 // Prepare the operand vector.
1945 for (Value *j : VL) {
1946 CallInst *CI2 = dyn_cast<CallInst>(j);
1947 Operands.push_back(CI2->getArgOperand(i));
1948 }
1949 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1950 }
1951 return;
1952 }
1953 case Instruction::ShuffleVector:
1954 // If this is not an alternate sequence of opcode like add-sub
1955 // then do not vectorize this instruction.
1956 if (!S.IsAltShuffle) {
1957 BS.cancelScheduling(VL, VL0);
1958 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1959 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
1960 return;
1961 }
1962 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1963 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
1964
1965 // Reorder operands if reordering would enable vectorization.
1966 if (isa<BinaryOperator>(VL0)) {
1967 ValueList Left, Right;
1968 reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1969 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1970 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1971 return;
1972 }
1973
1974 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1975 ValueList Operands;
1976 // Prepare the operand vector.
1977 for (Value *j : VL)
1978 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1979
1980 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1981 }
1982 return;
1983
1984 default:
1985 BS.cancelScheduling(VL, VL0);
1986 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1987 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
1988 return;
1989 }
1990}
1991
1992unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1993 unsigned N;
1994 Type *EltTy;
1995 auto *ST = dyn_cast<StructType>(T);
1996 if (ST) {
1997 N = ST->getNumElements();
1998 EltTy = *ST->element_begin();
1999 } else {
2000 N = cast<ArrayType>(T)->getNumElements();
2001 EltTy = cast<ArrayType>(T)->getElementType();
2002 }
2003 if (!isValidElementType(EltTy))
2004 return 0;
2005 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
2006 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
2007 return 0;
2008 if (ST) {
2009 // Check that struct is homogeneous.
2010 for (const auto *Ty : ST->elements())
2011 if (Ty != EltTy)
2012 return 0;
2013 }
2014 return N;
2015}
2016
2017bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
2018 SmallVectorImpl<unsigned> &CurrentOrder) const {
2019 Instruction *E0 = cast<Instruction>(OpValue);
2020 assert(E0->getOpcode() == Instruction::ExtractElement ||(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2021, __extension__ __PRETTY_FUNCTION__))
2021 E0->getOpcode() == Instruction::ExtractValue)(static_cast <bool> (E0->getOpcode() == Instruction::
ExtractElement || E0->getOpcode() == Instruction::ExtractValue
) ? void (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2021, __extension__ __PRETTY_FUNCTION__))
;
2022 assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode")(static_cast <bool> (E0->getOpcode() == getSameOpcode
(VL).Opcode && "Invalid opcode") ? void (0) : __assert_fail
("E0->getOpcode() == getSameOpcode(VL).Opcode && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2022, __extension__ __PRETTY_FUNCTION__))
;
2023 // Check if all of the extracts come from the same vector and from the
2024 // correct offset.
2025 Value *Vec = E0->getOperand(0);
2026
2027 CurrentOrder.clear();
2028
2029 // We have to extract from a vector/aggregate with the same number of elements.
2030 unsigned NElts;
2031 if (E0->getOpcode() == Instruction::ExtractValue) {
2032 const DataLayout &DL = E0->getModule()->getDataLayout();
2033 NElts = canMapToVector(Vec->getType(), DL);
2034 if (!NElts)
2035 return false;
2036 // Check if load can be rewritten as load of vector.
2037 LoadInst *LI = dyn_cast<LoadInst>(Vec);
2038 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2039 return false;
2040 } else {
2041 NElts = Vec->getType()->getVectorNumElements();
2042 }
2043
2044 if (NElts != VL.size())
2045 return false;
2046
2047 // Check that all of the indices extract from the correct offset.
2048 bool ShouldKeepOrder = true;
2049 unsigned E = VL.size();
2050 // Assign to all items the initial value E + 1 so we can check if the extract
2051 // instruction index was used already.
2052 // Also, later we can check that all the indices are used and we have a
2053 // consecutive access in the extract instructions, by checking that no
2054 // element of CurrentOrder still has value E + 1.
2055 CurrentOrder.assign(E, E + 1);
2056 unsigned I = 0;
2057 for (; I < E; ++I) {
2058 auto *Inst = cast<Instruction>(VL[I]);
2059 if (Inst->getOperand(0) != Vec)
2060 break;
2061 Optional<unsigned> Idx = getExtractIndex(Inst);
2062 if (!Idx)
2063 break;
2064 const unsigned ExtIdx = *Idx;
2065 if (ExtIdx != I) {
2066 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
2067 break;
2068 ShouldKeepOrder = false;
2069 CurrentOrder[ExtIdx] = I;
2070 } else {
2071 if (CurrentOrder[I] != E + 1)
2072 break;
2073 CurrentOrder[I] = I;
2074 }
2075 }
2076 if (I < E) {
2077 CurrentOrder.clear();
2078 return false;
2079 }
2080
2081 return ShouldKeepOrder;
2082}
2083
2084bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2085 return I->hasOneUse() ||
2086 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2087 return ScalarToTreeEntry.count(U) > 0;
2088 });
2089}
2090
2091int BoUpSLP::getEntryCost(TreeEntry *E) {
2092 ArrayRef<Value*> VL = E->Scalars;
2093
2094 Type *ScalarTy = VL[0]->getType();
2095 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2096 ScalarTy = SI->getValueOperand()->getType();
2097 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2098 ScalarTy = CI->getOperand(0)->getType();
2099 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2100
2101 // If we have computed a smaller type for the expression, update VecTy so
2102 // that the costs will be accurate.
2103 if (MinBWs.count(VL[0]))
2104 VecTy = VectorType::get(
2105 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2106
2107 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2108 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2109 int ReuseShuffleCost = 0;
2110 if (NeedToShuffleReuses) {
2111 ReuseShuffleCost =
2112 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2113 }
2114 if (E->NeedToGather) {
2115 if (allConstant(VL))
2116 return 0;
2117 if (isSplat(VL)) {
2118 return ReuseShuffleCost +
2119 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2120 }
2121 if (getSameOpcode(VL).Opcode == Instruction::ExtractElement &&
2122 allSameType(VL) && allSameBlock(VL)) {
2123 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
2124 if (ShuffleKind.hasValue()) {
2125 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2126 for (auto *V : VL) {
2127 // If all users of instruction are going to be vectorized and this
2128 // instruction itself is not going to be vectorized, consider this
2129 // instruction as dead and remove its cost from the final cost of the
2130 // vectorized tree.
2131 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2132 !ScalarToTreeEntry.count(V)) {
2133 auto *IO = cast<ConstantInt>(
2134 cast<ExtractElementInst>(V)->getIndexOperand());
2135 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2136 IO->getZExtValue());
2137 }
2138 }
2139 return ReuseShuffleCost + Cost;
2140 }
2141 }
2142 return ReuseShuffleCost + getGatherCost(VL);
2143 }
2144 InstructionsState S = getSameOpcode(VL);
2145 assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")(static_cast <bool> (S.Opcode && allSameType(VL
) && allSameBlock(VL) && "Invalid VL") ? void
(0) : __assert_fail ("S.Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2145, __extension__ __PRETTY_FUNCTION__))
;
2146 Instruction *VL0 = cast<Instruction>(S.OpValue);
2147 unsigned ShuffleOrOp = S.IsAltShuffle ?
2148 (unsigned) Instruction::ShuffleVector : S.Opcode;
2149 switch (ShuffleOrOp) {
2150 case Instruction::PHI:
2151 return 0;
2152
2153 case Instruction::ExtractValue:
2154 case Instruction::ExtractElement:
2155 if (NeedToShuffleReuses) {
2156 unsigned Idx = 0;
2157 for (unsigned I : E->ReuseShuffleIndices) {
2158 if (ShuffleOrOp == Instruction::ExtractElement) {
2159 auto *IO = cast<ConstantInt>(
2160 cast<ExtractElementInst>(VL[I])->getIndexOperand());
2161 Idx = IO->getZExtValue();
2162 ReuseShuffleCost -= TTI->getVectorInstrCost(
2163 Instruction::ExtractElement, VecTy, Idx);
2164 } else {
2165 ReuseShuffleCost -= TTI->getVectorInstrCost(
2166 Instruction::ExtractElement, VecTy, Idx);
2167 ++Idx;
2168 }
2169 }
2170 Idx = ReuseShuffleNumbers;
2171 for (Value *V : VL) {
2172 if (ShuffleOrOp == Instruction::ExtractElement) {
2173 auto *IO = cast<ConstantInt>(
2174 cast<ExtractElementInst>(V)->getIndexOperand());
2175 Idx = IO->getZExtValue();
2176 } else {
2177 --Idx;
2178 }
2179 ReuseShuffleCost +=
2180 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2181 }
2182 }
2183 if (!E->NeedToGather) {
2184 int DeadCost = ReuseShuffleCost;
2185 if (!E->ReorderIndices.empty()) {
2186 // TODO: Merge this shuffle with the ReuseShuffleCost.
2187 DeadCost += TTI->getShuffleCost(
2188 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2189 }
2190 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2191 Instruction *E = cast<Instruction>(VL[i]);
2192 // If all users are going to be vectorized, instruction can be
2193 // considered as dead.
2194 // The same, if have only one user, it will be vectorized for sure.
2195 if (areAllUsersVectorized(E))
2196 // Take credit for instruction that will become dead.
2197 DeadCost -=
2198 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2199 }
2200 return DeadCost;
2201 }
2202 return ReuseShuffleCost + getGatherCost(VL);
2203
2204 case Instruction::ZExt:
2205 case Instruction::SExt:
2206 case Instruction::FPToUI:
2207 case Instruction::FPToSI:
2208 case Instruction::FPExt:
2209 case Instruction::PtrToInt:
2210 case Instruction::IntToPtr:
2211 case Instruction::SIToFP:
2212 case Instruction::UIToFP:
2213 case Instruction::Trunc:
2214 case Instruction::FPTrunc:
2215 case Instruction::BitCast: {
2216 Type *SrcTy = VL0->getOperand(0)->getType();
2217 if (NeedToShuffleReuses) {
2218 ReuseShuffleCost -=
2219 (ReuseShuffleNumbers - VL.size()) *
2220 TTI->getCastInstrCost(S.Opcode, ScalarTy, SrcTy, VL0);
2221 }
2222
2223 // Calculate the cost of this instruction.
2224 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2225 VL0->getType(), SrcTy, VL0);
2226
2227 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2228 int VecCost = 0;
2229 // Check if the values are candidates to demote.
2230 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2231 VecCost = ReuseShuffleCost +
2232 TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2233 }
2234 return VecCost - ScalarCost;
2235 }
2236 case Instruction::FCmp:
2237 case Instruction::ICmp:
2238 case Instruction::Select: {
2239 // Calculate the cost of this instruction.
2240 if (NeedToShuffleReuses) {
2241 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2242 TTI->getCmpSelInstrCost(S.Opcode, ScalarTy,
2243 Builder.getInt1Ty(), VL0);
2244 }
2245 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2246 int ScalarCost = VecTy->getNumElements() *
2247 TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2248 int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2249 return ReuseShuffleCost + VecCost - ScalarCost;
2250 }
2251 case Instruction::Add:
2252 case Instruction::FAdd:
2253 case Instruction::Sub:
2254 case Instruction::FSub:
2255 case Instruction::Mul:
2256 case Instruction::FMul:
2257 case Instruction::UDiv:
2258 case Instruction::SDiv:
2259 case Instruction::FDiv:
2260 case Instruction::URem:
2261 case Instruction::SRem:
2262 case Instruction::FRem:
2263 case Instruction::Shl:
2264 case Instruction::LShr:
2265 case Instruction::AShr:
2266 case Instruction::And:
2267 case Instruction::Or:
2268 case Instruction::Xor: {
2269 // Certain instructions can be cheaper to vectorize if they have a
2270 // constant second vector operand.
2271 TargetTransformInfo::OperandValueKind Op1VK =
2272 TargetTransformInfo::OK_AnyValue;
2273 TargetTransformInfo::OperandValueKind Op2VK =
2274 TargetTransformInfo::OK_UniformConstantValue;
2275 TargetTransformInfo::OperandValueProperties Op1VP =
2276 TargetTransformInfo::OP_None;
2277 TargetTransformInfo::OperandValueProperties Op2VP =
2278 TargetTransformInfo::OP_None;
2279
2280 // If all operands are exactly the same ConstantInt then set the
2281 // operand kind to OK_UniformConstantValue.
2282 // If instead not all operands are constants, then set the operand kind
2283 // to OK_AnyValue. If all operands are constants but not the same,
2284 // then set the operand kind to OK_NonUniformConstantValue.
2285 ConstantInt *CInt = nullptr;
2286 for (unsigned i = 0; i < VL.size(); ++i) {
2287 const Instruction *I = cast<Instruction>(VL[i]);
2288 if (!isa<ConstantInt>(I->getOperand(1))) {
2289 Op2VK = TargetTransformInfo::OK_AnyValue;
2290 break;
2291 }
2292 if (i == 0) {
2293 CInt = cast<ConstantInt>(I->getOperand(1));
2294 continue;
2295 }
2296 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
2297 CInt != cast<ConstantInt>(I->getOperand(1)))
2298 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2299 }
2300 // FIXME: Currently cost of model modification for division by power of
2301 // 2 is handled for X86 and AArch64. Add support for other targets.
2302 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2303 CInt->getValue().isPowerOf2())
2304 Op2VP = TargetTransformInfo::OP_PowerOf2;
2305
2306 SmallVector<const Value *, 4> Operands(VL0->operand_values());
2307 if (NeedToShuffleReuses) {
2308 ReuseShuffleCost -=
2309 (ReuseShuffleNumbers - VL.size()) *
2310 TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2311 Op2VP, Operands);
2312 }
2313 int ScalarCost =
2314 VecTy->getNumElements() *
2315 TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2316 Op2VP, Operands);
2317 int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2318 Op1VP, Op2VP, Operands);
2319 return ReuseShuffleCost + VecCost - ScalarCost;
2320 }
2321 case Instruction::GetElementPtr: {
2322 TargetTransformInfo::OperandValueKind Op1VK =
2323 TargetTransformInfo::OK_AnyValue;
2324 TargetTransformInfo::OperandValueKind Op2VK =
2325 TargetTransformInfo::OK_UniformConstantValue;
2326
2327 if (NeedToShuffleReuses) {
2328 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2329 TTI->getArithmeticInstrCost(Instruction::Add,
2330 ScalarTy, Op1VK, Op2VK);
2331 }
2332 int ScalarCost =
2333 VecTy->getNumElements() *
2334 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2335 int VecCost =
2336 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2337
2338 return ReuseShuffleCost + VecCost - ScalarCost;
2339 }
2340 case Instruction::Load: {
2341 // Cost of wide load - cost of scalar loads.
2342 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2343 if (NeedToShuffleReuses) {
2344 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2345 TTI->getMemoryOpCost(Instruction::Load, ScalarTy,
2346 alignment, 0, VL0);
2347 }
2348 int ScalarLdCost = VecTy->getNumElements() *
2349 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2350 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2351 VecTy, alignment, 0, VL0);
2352 if (!E->ReorderIndices.empty()) {
2353 // TODO: Merge this shuffle with the ReuseShuffleCost.
2354 VecLdCost += TTI->getShuffleCost(
2355 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2356 }
2357 return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2358 }
2359 case Instruction::Store: {
2360 // We know that we can merge the stores. Calculate the cost.
2361 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2362 if (NeedToShuffleReuses) {
2363 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) *
2364 TTI->getMemoryOpCost(Instruction::Store, ScalarTy,
2365 alignment, 0, VL0);
2366 }
2367 int ScalarStCost = VecTy->getNumElements() *
2368 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2369 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2370 VecTy, alignment, 0, VL0);
2371 return ReuseShuffleCost + VecStCost - ScalarStCost;
2372 }
2373 case Instruction::Call: {
2374 CallInst *CI = cast<CallInst>(VL0);
2375 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2376
2377 // Calculate the cost of the scalar and vector calls.
2378 SmallVector<Type*, 4> ScalarTys;
2379 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2380 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2381
2382 FastMathFlags FMF;
2383 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2384 FMF = FPMO->getFastMathFlags();
2385
2386 if (NeedToShuffleReuses) {
2387 ReuseShuffleCost -=
2388 (ReuseShuffleNumbers - VL.size()) *
2389 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2390 }
2391 int ScalarCallCost = VecTy->getNumElements() *
2392 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2393
2394 SmallVector<Value *, 4> Args(CI->arg_operands());
2395 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2396 VecTy->getNumElements());
2397
2398 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2399 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2400 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
2401
2402 return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2403 }
2404 case Instruction::ShuffleVector: {
2405 TargetTransformInfo::OperandValueKind Op1VK =
2406 TargetTransformInfo::OK_AnyValue;
2407 TargetTransformInfo::OperandValueKind Op2VK =
2408 TargetTransformInfo::OK_AnyValue;
2409 int ScalarCost = 0;
2410 if (NeedToShuffleReuses) {
2411 for (unsigned Idx : E->ReuseShuffleIndices) {
2412 Instruction *I = cast<Instruction>(VL[Idx]);
2413 if (!I)
2414 continue;
2415 ReuseShuffleCost -= TTI->getArithmeticInstrCost(
2416 I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2417 }
2418 for (Value *V : VL) {
2419 Instruction *I = cast<Instruction>(V);
2420 if (!I)
2421 continue;
2422 ReuseShuffleCost += TTI->getArithmeticInstrCost(
2423 I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2424 }
2425 }
2426 int VecCost = 0;
2427 for (Value *i : VL) {
2428 Instruction *I = cast<Instruction>(i);
2429 if (!I)
2430 break;
2431 ScalarCost +=
2432 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2433 }
2434 // VecCost is equal to sum of the cost of creating 2 vectors
2435 // and the cost of creating shuffle.
2436 Instruction *I0 = cast<Instruction>(VL[0]);
2437 VecCost =
2438 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2439 Instruction *I1 = cast<Instruction>(VL[1]);
2440 VecCost +=
2441 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2442 VecCost +=
2443 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2444 return ReuseShuffleCost + VecCost - ScalarCost;
2445 }
2446 default:
2447 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2447)
;
2448 }
2449}
2450
2451bool BoUpSLP::isFullyVectorizableTinyTree() {
2452 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
2453 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
2454
2455 // We only handle trees of heights 1 and 2.
2456 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2457 return true;
2458
2459 if (VectorizableTree.size() != 2)
2460 return false;
2461
2462 // Handle splat and all-constants stores.
2463 if (!VectorizableTree[0].NeedToGather &&
2464 (allConstant(VectorizableTree[1].Scalars) ||
2465 isSplat(VectorizableTree[1].Scalars)))
2466 return true;
2467
2468 // Gathering cost would be too much for tiny trees.
2469 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2470 return false;
2471
2472 return true;
2473}
2474
2475bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2476 // We can vectorize the tree if its size is greater than or equal to the
2477 // minimum size specified by the MinTreeSize command line option.
2478 if (VectorizableTree.size() >= MinTreeSize)
2479 return false;
2480
2481 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2482 // can vectorize it if we can prove it fully vectorizable.
2483 if (isFullyVectorizableTinyTree())
2484 return false;
2485
2486 assert(VectorizableTree.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2488, __extension__ __PRETTY_FUNCTION__))
2487 ? ExternalUses.empty()(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2488, __extension__ __PRETTY_FUNCTION__))
2488 : true && "We shouldn't have any external users")(static_cast <bool> (VectorizableTree.empty() ? ExternalUses
.empty() : true && "We shouldn't have any external users"
) ? void (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2488, __extension__ __PRETTY_FUNCTION__))
;
2489
2490 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2491 // vectorizable.
2492 return true;
2493}
2494
2495int BoUpSLP::getSpillCost() {
2496 // Walk from the bottom of the tree to the top, tracking which values are
2497 // live. When we see a call instruction that is not part of our tree,
2498 // query TTI to see if there is a cost to keeping values live over it
2499 // (for example, if spills and fills are required).
2500 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2501 int Cost = 0;
2502
2503 SmallPtrSet<Instruction*, 4> LiveValues;
2504 Instruction *PrevInst = nullptr;
2505
2506 for (const auto &N : VectorizableTree) {
2507 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2508 if (!Inst)
2509 continue;
2510
2511 if (!PrevInst) {
2512 PrevInst = Inst;
2513 continue;
2514 }
2515
2516 // Update LiveValues.
2517 LiveValues.erase(PrevInst);
2518 for (auto &J : PrevInst->operands()) {
2519 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2520 LiveValues.insert(cast<Instruction>(&*J));
2521 }
2522
2523 DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2524 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2525 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2526 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2527 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2528 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2529 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
;
2530
2531 // Now find the sequence of instructions between PrevInst and Inst.
2532 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2533 PrevInstIt =
2534 PrevInst->getIterator().getReverse();
2535 while (InstIt != PrevInstIt) {
2536 if (PrevInstIt == PrevInst->getParent()->rend()) {
2537 PrevInstIt = Inst->getParent()->rbegin();
2538 continue;
2539 }
2540
2541 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2542 SmallVector<Type*, 4> V;
2543 for (auto *II : LiveValues)
2544 V.push_back(VectorType::get(II->getType(), BundleWidth));
2545 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2546 }
2547
2548 ++PrevInstIt;
2549 }
2550
2551 PrevInst = Inst;
2552 }
2553
2554 return Cost;
2555}
2556
2557int BoUpSLP::getTreeCost() {
2558 int Cost = 0;
2559 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
2560 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
2561
2562 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2563
2564 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
2565 TreeEntry &TE = VectorizableTree[I];
2566
2567 // We create duplicate tree entries for gather sequences that have multiple
2568 // uses. However, we should not compute the cost of duplicate sequences.
2569 // For example, if we have a build vector (i.e., insertelement sequence)
2570 // that is used by more than one vector instruction, we only need to
2571 // compute the cost of the insertelement instructions once. The redundent
2572 // instructions will be eliminated by CSE.
2573 //
2574 // We should consider not creating duplicate tree entries for gather
2575 // sequences, and instead add additional edges to the tree representing
2576 // their uses. Since such an approach results in fewer total entries,
2577 // existing heuristics based on tree size may yeild different results.
2578 //
2579 if (TE.NeedToGather &&
2580 std::any_of(std::next(VectorizableTree.begin(), I + 1),
2581 VectorizableTree.end(), [TE](TreeEntry &Entry) {
2582 return Entry.NeedToGather && Entry.isSame(TE.Scalars);
2583 }))
2584 continue;
2585
2586 int C = getEntryCost(&TE);
2587 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
2588 << *TE.Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
2589 Cost += C;
2590 }
2591
2592 SmallSet<Value *, 16> ExtractCostCalculated;
2593 int ExtractCost = 0;
2594 for (ExternalUser &EU : ExternalUses) {
2595 // We only add extract cost once for the same scalar.
2596 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2597 continue;
2598
2599 // Uses by ephemeral values are free (because the ephemeral value will be
2600 // removed prior to code generation, and so the extraction will be
2601 // removed as well).
2602 if (EphValues.count(EU.User))
2603 continue;
2604
2605 // If we plan to rewrite the tree in a smaller type, we will need to sign
2606 // extend the extracted value back to the original type. Here, we account
2607 // for the extract and the added cost of the sign extend if needed.
2608 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2609 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2610 if (MinBWs.count(ScalarRoot)) {
2611 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2612 auto Extend =
2613 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2614 VecTy = VectorType::get(MinTy, BundleWidth);
2615 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2616 VecTy, EU.Lane);
2617 } else {
2618 ExtractCost +=
2619 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2620 }
2621 }
2622
2623 int SpillCost = getSpillCost();
2624 Cost += SpillCost + ExtractCost;
2625
2626 std::string Str;
2627 {
2628 raw_string_ostream OS(Str);
2629 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2630 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2631 << "SLP: Total Cost = " << Cost << ".\n";
2632 }
2633 DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
2634
2635 if (ViewSLPTree)
2636 ViewGraph(this, "SLP" + F->getName(), false, Str);
2637
2638 return Cost;
2639}
2640
2641int BoUpSLP::getGatherCost(Type *Ty,
2642 const DenseSet<unsigned> &ShuffledIndices) {
2643 int Cost = 0;
2644 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2645 if (!ShuffledIndices.count(i))
2646 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2647 if (!ShuffledIndices.empty())
2648 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2649 return Cost;
2650}
2651
2652int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2653 // Find the type of the operands in VL.
2654 Type *ScalarTy = VL[0]->getType();
2655 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2656 ScalarTy = SI->getValueOperand()->getType();
2657 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2658 // Find the cost of inserting/extracting values from the vector.
2659 // Check if the same elements are inserted several times and count them as
2660 // shuffle candidates.
2661 DenseSet<unsigned> ShuffledElements;
2662 DenseSet<Value *> UniqueElements;
2663 // Iterate in reverse order to consider insert elements with the high cost.
2664 for (unsigned I = VL.size(); I > 0; --I) {
2665 unsigned Idx = I - 1;
2666 if (!UniqueElements.insert(VL[Idx]).second)
2667 ShuffledElements.insert(Idx);
2668 }
2669 return getGatherCost(VecTy, ShuffledElements);
2670}
2671
2672// Reorder commutative operations in alternate shuffle if the resulting vectors
2673// are consecutive loads. This would allow us to vectorize the tree.
2674// If we have something like-
2675// load a[0] - load b[0]
2676// load b[1] + load a[1]
2677// load a[2] - load b[2]
2678// load a[3] + load b[3]
2679// Reordering the second load b[1] load a[1] would allow us to vectorize this
2680// code.
2681void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2682 SmallVectorImpl<Value *> &Left,
2683 SmallVectorImpl<Value *> &Right) {
2684 // Push left and right operands of binary operation into Left and Right
2685 unsigned AltOpcode = getAltOpcode(Opcode);
2686 (void)AltOpcode;
2687 for (Value *V : VL) {
2688 auto *I = cast<Instruction>(V);
2689 assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&(static_cast <bool> (sameOpcodeOrAlt(Opcode, AltOpcode,
I->getOpcode()) && "Incorrect instruction in vector"
) ? void (0) : __assert_fail ("sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2690, __extension__ __PRETTY_FUNCTION__))
2690 "Incorrect instruction in vector")(static_cast <bool> (sameOpcodeOrAlt(Opcode, AltOpcode,
I->getOpcode()) && "Incorrect instruction in vector"
) ? void (0) : __assert_fail ("sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2690, __extension__ __PRETTY_FUNCTION__))
;
2691 Left.push_back(I->getOperand(0));
2692 Right.push_back(I->getOperand(1));
2693 }
2694
2695 // Reorder if we have a commutative operation and consecutive access
2696 // are on either side of the alternate instructions.
2697 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2698 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2699 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2700 Instruction *VL1 = cast<Instruction>(VL[j]);
2701 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2702 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2703 std::swap(Left[j], Right[j]);
2704 continue;
2705 } else if (VL2->isCommutative() &&
2706 isConsecutiveAccess(L, L1, *DL, *SE)) {
2707 std::swap(Left[j + 1], Right[j + 1]);
2708 continue;
2709 }
2710 // else unchanged
2711 }
2712 }
2713 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2714 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2715 Instruction *VL1 = cast<Instruction>(VL[j]);
2716 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2717 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2718 std::swap(Left[j], Right[j]);
2719 continue;
2720 } else if (VL2->isCommutative() &&
2721 isConsecutiveAccess(L, L1, *DL, *SE)) {
2722 std::swap(Left[j + 1], Right[j + 1]);
2723 continue;
2724 }
2725 // else unchanged
2726 }
2727 }
2728 }
2729}
2730
2731// Return true if I should be commuted before adding it's left and right
2732// operands to the arrays Left and Right.
2733//
2734// The vectorizer is trying to either have all elements one side being
2735// instruction with the same opcode to enable further vectorization, or having
2736// a splat to lower the vectorizing cost.
2737static bool shouldReorderOperands(
2738 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2739 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2740 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2741 VLeft = I.getOperand(0);
2742 VRight = I.getOperand(1);
2743 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2744 if (SplatRight) {
2745 if (VRight == Right[i - 1])
2746 // Preserve SplatRight
2747 return false;
2748 if (VLeft == Right[i - 1]) {
2749 // Commuting would preserve SplatRight, but we don't want to break
2750 // SplatLeft either, i.e. preserve the original order if possible.
2751 // (FIXME: why do we care?)
2752 if (SplatLeft && VLeft == Left[i - 1])
2753 return false;
2754 return true;
2755 }
2756 }
2757 // Symmetrically handle Right side.
2758 if (SplatLeft) {
2759 if (VLeft == Left[i - 1])
2760 // Preserve SplatLeft
2761 return false;
2762 if (VRight == Left[i - 1])
2763 return true;
2764 }
2765
2766 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2767 Instruction *IRight = dyn_cast<Instruction>(VRight);
2768
2769 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2770 // it and not the right, in this case we want to commute.
2771 if (AllSameOpcodeRight) {
2772 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2773 if (IRight && RightPrevOpcode == IRight->getOpcode())
2774 // Do not commute, a match on the right preserves AllSameOpcodeRight
2775 return false;
2776 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2777 // We have a match and may want to commute, but first check if there is
2778 // not also a match on the existing operands on the Left to preserve
2779 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2780 // (FIXME: why do we care?)
2781 if (AllSameOpcodeLeft && ILeft &&
2782 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2783 return false;
2784 return true;
2785 }
2786 }
2787 // Symmetrically handle Left side.
2788 if (AllSameOpcodeLeft) {
2789 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2790 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2791 return false;
2792 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2793 return true;
2794 }
2795 return false;
2796}
2797
2798void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2799 ArrayRef<Value *> VL,
2800 SmallVectorImpl<Value *> &Left,
2801 SmallVectorImpl<Value *> &Right) {
2802 if (!VL.empty()) {
2803 // Peel the first iteration out of the loop since there's nothing
2804 // interesting to do anyway and it simplifies the checks in the loop.
2805 auto *I = cast<Instruction>(VL[0]);
2806 Value *VLeft = I->getOperand(0);
2807 Value *VRight = I->getOperand(1);
2808 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2809 // Favor having instruction to the right. FIXME: why?
2810 std::swap(VLeft, VRight);
2811 Left.push_back(VLeft);
2812 Right.push_back(VRight);
2813 }
2814
2815 // Keep track if we have instructions with all the same opcode on one side.
2816 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2817 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2818 // Keep track if we have one side with all the same value (broadcast).
2819 bool SplatLeft = true;
2820 bool SplatRight = true;
2821
2822 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2823 Instruction *I = cast<Instruction>(VL[i]);
2824 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2826, __extension__ __PRETTY_FUNCTION__))
2825 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2826, __extension__ __PRETTY_FUNCTION__))
2826 "Can only process commutative instruction")(static_cast <bool> (((I->getOpcode() == Opcode &&
I->isCommutative()) || (I->getOpcode() != Opcode &&
Instruction::isCommutative(Opcode))) && "Can only process commutative instruction"
) ? void (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2826, __extension__ __PRETTY_FUNCTION__))
;
2827 // Commute to favor either a splat or maximizing having the same opcodes on
2828 // one side.
2829 Value *VLeft;
2830 Value *VRight;
2831 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2832 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2833 VRight)) {
2834 Left.push_back(VRight);
2835 Right.push_back(VLeft);
2836 } else {
2837 Left.push_back(VLeft);
2838 Right.push_back(VRight);
2839 }
2840 // Update Splat* and AllSameOpcode* after the insertion.
2841 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2842 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2843 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2844 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2845 cast<Instruction>(Left[i])->getOpcode());
2846 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2847 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2848 cast<Instruction>(Right[i])->getOpcode());
2849 }
2850
2851 // If one operand end up being broadcast, return this operand order.
2852 if (SplatRight || SplatLeft)
2853 return;
2854
2855 // Finally check if we can get longer vectorizable chain by reordering
2856 // without breaking the good operand order detected above.
2857 // E.g. If we have something like-
2858 // load a[0] load b[0]
2859 // load b[1] load a[1]
2860 // load a[2] load b[2]
2861 // load a[3] load b[3]
2862 // Reordering the second load b[1] load a[1] would allow us to vectorize
2863 // this code and we still retain AllSameOpcode property.
2864 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2865 // such as-
2866 // add a[0],c[0] load b[0]
2867 // add a[1],c[2] load b[1]
2868 // b[2] load b[2]
2869 // add a[3],c[3] load b[3]
2870 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2871 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2872 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2873 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2874 std::swap(Left[j + 1], Right[j + 1]);
2875 continue;
2876 }
2877 }
2878 }
2879 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2880 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2881 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2882 std::swap(Left[j + 1], Right[j + 1]);
2883 continue;
2884 }
2885 }
2886 }
2887 // else unchanged
2888 }
2889}
2890
2891void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2892 // Get the basic block this bundle is in. All instructions in the bundle
2893 // should be in this block.
2894 auto *Front = cast<Instruction>(OpValue);
2895 auto *BB = Front->getParent();
2896 const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2897 const unsigned AltOpcode = getAltOpcode(Opcode);
2898 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2902, __extension__ __PRETTY_FUNCTION__))
2899 return !sameOpcodeOrAlt(Opcode, AltOpcode,(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2902, __extension__ __PRETTY_FUNCTION__))
2900 cast<Instruction>(V)->getOpcode()) ||(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2902, __extension__ __PRETTY_FUNCTION__))
2901 cast<Instruction>(V)->getParent() == BB;(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2902, __extension__ __PRETTY_FUNCTION__))
2902 }))(static_cast <bool> (llvm::all_of(make_range(VL.begin()
, VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt
(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode(
)) || cast<Instruction>(V)->getParent() == BB; })) ?
void (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { return !sameOpcodeOrAlt(Opcode, AltOpcode, cast<Instruction>(V)->getOpcode()) || cast<Instruction>(V)->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2902, __extension__ __PRETTY_FUNCTION__))
;
2903
2904 // The last instruction in the bundle in program order.
2905 Instruction *LastInst = nullptr;
2906
2907 // Find the last instruction. The common case should be that BB has been
2908 // scheduled, and the last instruction is VL.back(). So we start with
2909 // VL.back() and iterate over schedule data until we reach the end of the
2910 // bundle. The end of the bundle is marked by null ScheduleData.
2911 if (BlocksSchedules.count(BB)) {
2912 auto *Bundle =
2913 BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2914 if (Bundle && Bundle->isPartOfBundle())
2915 for (; Bundle; Bundle = Bundle->NextInBundle)
2916 if (Bundle->OpValue == Bundle->Inst)
2917 LastInst = Bundle->Inst;
2918 }
2919
2920 // LastInst can still be null at this point if there's either not an entry
2921 // for BB in BlocksSchedules or there's no ScheduleData available for
2922 // VL.back(). This can be the case if buildTree_rec aborts for various
2923 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2924 // size is reached, etc.). ScheduleData is initialized in the scheduling
2925 // "dry-run".
2926 //
2927 // If this happens, we can still find the last instruction by brute force. We
2928 // iterate forwards from Front (inclusive) until we either see all
2929 // instructions in the bundle or reach the end of the block. If Front is the
2930 // last instruction in program order, LastInst will be set to Front, and we
2931 // will visit all the remaining instructions in the block.
2932 //
2933 // One of the reasons we exit early from buildTree_rec is to place an upper
2934 // bound on compile-time. Thus, taking an additional compile-time hit here is
2935 // not ideal. However, this should be exceedingly rare since it requires that
2936 // we both exit early from buildTree_rec and that the bundle be out-of-order
2937 // (causing us to iterate all the way to the end of the block).
2938 if (!LastInst) {
2939 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2940 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2941 if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2942 LastInst = &I;
2943 if (Bundle.empty())
2944 break;
2945 }
2946 }
2947
2948 // Set the insertion point after the last instruction in the bundle. Set the
2949 // debug location to Front.
2950 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2951 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2952}
2953
2954Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2955 Value *Vec = UndefValue::get(Ty);
2956 // Generate the 'InsertElement' instruction.
2957 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2958 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2959 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2960 GatherSeq.insert(Insrt);
2961 CSEBlocks.insert(Insrt->getParent());
2962
2963 // Add to our 'need-to-extract' list.
2964 if (TreeEntry *E = getTreeEntry(VL[i])) {
2965 // Find which lane we need to extract.
2966 int FoundLane = -1;
2967 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2968 // Is this the lane of the scalar that we are looking for ?
2969 if (E->Scalars[Lane] == VL[i]) {
2970 FoundLane = Lane;
2971 break;
2972 }
2973 }
2974 assert(FoundLane >= 0 && "Could not find the correct lane")(static_cast <bool> (FoundLane >= 0 && "Could not find the correct lane"
) ? void (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2974, __extension__ __PRETTY_FUNCTION__))
;
2975 if (!E->ReuseShuffleIndices.empty()) {
2976 FoundLane =
2977 std::distance(E->ReuseShuffleIndices.begin(),
2978 llvm::find(E->ReuseShuffleIndices, FoundLane));
2979 }
2980 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2981 }
2982 }
2983 }
2984
2985 return Vec;
2986}
2987
2988Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2989 InstructionsState S = getSameOpcode(VL);
2990 if (S.Opcode) {
2991 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2992 if (E->isSame(VL)) {
2993 Value *V = vectorizeTree(E);
2994 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
2995 // We need to get the vectorized value but without shuffle.
2996 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
2997 V = SV->getOperand(0);
2998 } else {
2999 // Reshuffle to get only unique values.
3000 SmallVector<unsigned, 4> UniqueIdxs;
3001 SmallSet<unsigned, 4> UsedIdxs;
3002 for(unsigned Idx : E->ReuseShuffleIndices)
3003 if (UsedIdxs.insert(Idx).second)
3004 UniqueIdxs.emplace_back(Idx);
3005 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3006 UniqueIdxs);
3007 }
3008 }
3009 return V;
3010 }
3011 }
3012 }
3013
3014 Type *ScalarTy = S.OpValue->getType();
3015 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
3016 ScalarTy = SI->getValueOperand()->getType();
3017
3018 // Check that every instruction appears once in this bundle.
3019 SmallVector<unsigned, 4> ReuseShuffleIndicies;
3020 SmallVector<Value *, 4> UniqueValues;
3021 if (VL.size() > 2) {
3022 DenseMap<Value *, unsigned> UniquePositions;
3023 for (Value *V : VL) {
3024 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
3025 ReuseShuffleIndicies.emplace_back(Res.first->second);
3026 if (Res.second || isa<Constant>(V))
3027 UniqueValues.emplace_back(V);
3028 }
3029 // Do not shuffle single element or if number of unique values is not power
3030 // of 2.
3031 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
3032 !llvm::isPowerOf2_32(UniqueValues.size()))
3033 ReuseShuffleIndicies.clear();
3034 else
3035 VL = UniqueValues;
3036 }
3037 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3038
3039 Value *V = Gather(VL, VecTy);
3040 if (!ReuseShuffleIndicies.empty()) {
3041 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3042 ReuseShuffleIndicies, "shuffle");
3043 if (auto *I = dyn_cast<Instruction>(V)) {
3044 GatherSeq.insert(I);
3045 CSEBlocks.insert(I->getParent());
3046 }
3047 }
3048 return V;
3049}
3050
3051static void inversePermutation(ArrayRef<unsigned> Indices,
3052 SmallVectorImpl<unsigned> &Mask) {
3053 Mask.clear();
3054 const unsigned E = Indices.size();
3055 Mask.resize(E);
3056 for (unsigned I = 0; I < E; ++I)
3057 Mask[Indices[I]] = I;
3058}
3059
3060Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
3061 IRBuilder<>::InsertPointGuard Guard(Builder);
3062
3063 if (E->VectorizedValue) {
3064 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
3065 return E->VectorizedValue;
3066 }
3067
3068 InstructionsState S = getSameOpcode(E->Scalars);
3069 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
3070 Type *ScalarTy = VL0->getType();
3071 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3072 ScalarTy = SI->getValueOperand()->getType();
3073 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
3074
3075 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3076
3077 if (E->NeedToGather) {
3078 setInsertPointAfterBundle(E->Scalars, VL0);
3079 auto *V = Gather(E->Scalars, VecTy);
3080 if (NeedToShuffleReuses) {
3081 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3082 E->ReuseShuffleIndices, "shuffle");
3083 if (auto *I = dyn_cast<Instruction>(V)) {
3084 GatherSeq.insert(I);
3085 CSEBlocks.insert(I->getParent());
3086 }
3087 }
3088 E->VectorizedValue = V;
3089 return V;
3090 }
3091
3092 unsigned ShuffleOrOp = S.IsAltShuffle ?
3093 (unsigned) Instruction::ShuffleVector : S.Opcode;
3094 switch (ShuffleOrOp) {
3095 case Instruction::PHI: {
3096 PHINode *PH = dyn_cast<PHINode>(VL0);
3097 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
3098 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3099 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
3100 Value *V = NewPhi;
3101 if (NeedToShuffleReuses) {
3102 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3103 E->ReuseShuffleIndices, "shuffle");
3104 }
3105 E->VectorizedValue = V;
3106
3107 // PHINodes may have multiple entries from the same block. We want to
3108 // visit every block once.
3109 SmallSet<BasicBlock*, 4> VisitedBBs;
3110
3111 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
3112 ValueList Operands;
3113 BasicBlock *IBB = PH->getIncomingBlock(i);
3114
3115 if (!VisitedBBs.insert(IBB).second) {
3116 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
3117 continue;
3118 }
3119
3120 // Prepare the operand vector.
3121 for (Value *V : E->Scalars)
3122 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
3123
3124 Builder.SetInsertPoint(IBB->getTerminator());
3125 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3126 Value *Vec = vectorizeTree(Operands);
3127 NewPhi->addIncoming(Vec, IBB);
3128 }
3129
3130 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3131, __extension__ __PRETTY_FUNCTION__))
3131 "Invalid number of incoming values")(static_cast <bool> (NewPhi->getNumIncomingValues() ==
PH->getNumIncomingValues() && "Invalid number of incoming values"
) ? void (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3131, __extension__ __PRETTY_FUNCTION__))
;
3132 return V;
3133 }
3134
3135 case Instruction::ExtractElement: {
3136 if (!E->NeedToGather) {
3137 Value *V = VL0->getOperand(0);
3138 if (!E->ReorderIndices.empty()) {
3139 OrdersType Mask;
3140 inversePermutation(E->ReorderIndices, Mask);
3141 Builder.SetInsertPoint(VL0);
3142 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
3143 "reorder_shuffle");
3144 }
3145 if (NeedToShuffleReuses) {
3146 // TODO: Merge this shuffle with the ReorderShuffleMask.
3147 if (!E->ReorderIndices.empty())
3148 Builder.SetInsertPoint(VL0);
3149 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3150 E->ReuseShuffleIndices, "shuffle");
3151 }
3152 E->VectorizedValue = V;
3153 return V;
3154 }
3155 setInsertPointAfterBundle(E->Scalars, VL0);
3156 auto *V = Gather(E->Scalars, VecTy);
3157 if (NeedToShuffleReuses) {
3158 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3159 E->ReuseShuffleIndices, "shuffle");
3160 if (auto *I = dyn_cast<Instruction>(V)) {
3161 GatherSeq.insert(I);
3162 CSEBlocks.insert(I->getParent());
3163 }
3164 }
3165 E->VectorizedValue = V;
3166 return V;
3167 }
3168 case Instruction::ExtractValue: {
3169 if (!E->NeedToGather) {
3170 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3171 Builder.SetInsertPoint(LI);
3172 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3173 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3174 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
3175 Value *NewV = propagateMetadata(V, E->Scalars);
3176 if (!E->ReorderIndices.empty()) {
3177 OrdersType Mask;
3178 inversePermutation(E->ReorderIndices, Mask);
3179 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
3180 "reorder_shuffle");
3181 }
3182 if (NeedToShuffleReuses) {
3183 // TODO: Merge this shuffle with the ReorderShuffleMask.
3184 NewV = Builder.CreateShuffleVector(
3185 NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3186 }
3187 E->VectorizedValue = NewV;
3188 return NewV;
3189 }
3190 setInsertPointAfterBundle(E->Scalars, VL0);
3191 auto *V = Gather(E->Scalars, VecTy);
3192 if (NeedToShuffleReuses) {
3193 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3194 E->ReuseShuffleIndices, "shuffle");
3195 if (auto *I = dyn_cast<Instruction>(V)) {
3196 GatherSeq.insert(I);
3197 CSEBlocks.insert(I->getParent());
3198 }
3199 }
3200 E->VectorizedValue = V;
3201 return V;
3202 }
3203 case Instruction::ZExt:
3204 case Instruction::SExt:
3205 case Instruction::FPToUI:
3206 case Instruction::FPToSI:
3207 case Instruction::FPExt:
3208 case Instruction::PtrToInt:
3209 case Instruction::IntToPtr:
3210 case Instruction::SIToFP:
3211 case Instruction::UIToFP:
3212 case Instruction::Trunc:
3213 case Instruction::FPTrunc:
3214 case Instruction::BitCast: {
3215 ValueList INVL;
3216 for (Value *V : E->Scalars)
3217 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3218
3219 setInsertPointAfterBundle(E->Scalars, VL0);
3220
3221 Value *InVec = vectorizeTree(INVL);
3222
3223 if (E->VectorizedValue) {
3224 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3225 return E->VectorizedValue;
3226 }
3227
3228 CastInst *CI = dyn_cast<CastInst>(VL0);
3229 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3230 if (NeedToShuffleReuses) {
3231 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3232 E->ReuseShuffleIndices, "shuffle");
3233 }
3234 E->VectorizedValue = V;
3235 ++NumVectorInstructions;
3236 return V;
3237 }
3238 case Instruction::FCmp:
3239 case Instruction::ICmp: {
3240 ValueList LHSV, RHSV;
3241 for (Value *V : E->Scalars) {
3242 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3243 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3244 }
3245
3246 setInsertPointAfterBundle(E->Scalars, VL0);
3247
3248 Value *L = vectorizeTree(LHSV);
3249 Value *R = vectorizeTree(RHSV);
3250
3251 if (E->VectorizedValue) {
3252 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3253 return E->VectorizedValue;
3254 }
3255
3256 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3257 Value *V;
3258 if (S.Opcode == Instruction::FCmp)
3259 V = Builder.CreateFCmp(P0, L, R);
3260 else
3261 V = Builder.CreateICmp(P0, L, R);
3262
3263 propagateIRFlags(V, E->Scalars, VL0);
3264 if (NeedToShuffleReuses) {
3265 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3266 E->ReuseShuffleIndices, "shuffle");
3267 }
3268 E->VectorizedValue = V;
3269 ++NumVectorInstructions;
3270 return V;
3271 }
3272 case Instruction::Select: {
3273 ValueList TrueVec, FalseVec, CondVec;
3274 for (Value *V : E->Scalars) {
3275 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3276 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3277 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3278 }
3279
3280 setInsertPointAfterBundle(E->Scalars, VL0);
3281
3282 Value *Cond = vectorizeTree(CondVec);
3283 Value *True = vectorizeTree(TrueVec);
3284 Value *False = vectorizeTree(FalseVec);
3285
3286 if (E->VectorizedValue) {
3287 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3288 return E->VectorizedValue;
3289 }
3290
3291 Value *V = Builder.CreateSelect(Cond, True, False);
3292 if (NeedToShuffleReuses) {
3293 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3294 E->ReuseShuffleIndices, "shuffle");
3295 }
3296 E->VectorizedValue = V;
3297 ++NumVectorInstructions;
3298 return V;
3299 }
3300 case Instruction::Add:
3301 case Instruction::FAdd:
3302 case Instruction::Sub:
3303 case Instruction::FSub:
3304 case Instruction::Mul:
3305 case Instruction::FMul:
3306 case Instruction::UDiv:
3307 case Instruction::SDiv:
3308 case Instruction::FDiv:
3309 case Instruction::URem:
3310 case Instruction::SRem:
3311 case Instruction::FRem:
3312 case Instruction::Shl:
3313 case Instruction::LShr:
3314 case Instruction::AShr:
3315 case Instruction::And:
3316 case Instruction::Or:
3317 case Instruction::Xor: {
3318 ValueList LHSVL, RHSVL;
3319 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3320 reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
3321 RHSVL);
3322 else
3323 for (Value *V : E->Scalars) {
3324 auto *I = cast<Instruction>(V);
3325 LHSVL.push_back(I->getOperand(0));
3326 RHSVL.push_back(I->getOperand(1));
3327 }
3328
3329 setInsertPointAfterBundle(E->Scalars, VL0);
3330
3331 Value *LHS = vectorizeTree(LHSVL);
3332 Value *RHS = vectorizeTree(RHSVL);
3333
3334 if (E->VectorizedValue) {
3335 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3336 return E->VectorizedValue;
3337 }
3338
3339 Value *V = Builder.CreateBinOp(
3340 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3341 propagateIRFlags(V, E->Scalars, VL0);
3342 if (auto *I = dyn_cast<Instruction>(V))
3343 V = propagateMetadata(I, E->Scalars);
3344
3345 if (NeedToShuffleReuses) {
3346 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3347 E->ReuseShuffleIndices, "shuffle");
3348 }
3349 E->VectorizedValue = V;
3350 ++NumVectorInstructions;
3351
3352 return V;
3353 }
3354 case Instruction::Load: {
3355 // Loads are inserted at the head of the tree because we don't want to
3356 // sink them all the way down past store instructions.
3357 bool IsReorder = !E->ReorderIndices.empty();
3358 if (IsReorder)
3359 VL0 = cast<Instruction>(E->Scalars[E->ReorderIndices.front()]);
3360 setInsertPointAfterBundle(E->Scalars, VL0);
3361
3362 LoadInst *LI = cast<LoadInst>(VL0);
3363 Type *ScalarLoadTy = LI->getType();
3364 unsigned AS = LI->getPointerAddressSpace();
3365
3366 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3367 VecTy->getPointerTo(AS));
3368
3369 // The pointer operand uses an in-tree scalar so we add the new BitCast to
3370 // ExternalUses list to make sure that an extract will be generated in the
3371 // future.
3372 Value *PO = LI->getPointerOperand();
3373 if (getTreeEntry(PO))
3374 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3375
3376 unsigned Alignment = LI->getAlignment();
3377 LI = Builder.CreateLoad(VecPtr);
3378 if (!Alignment) {
3379 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3380 }
3381 LI->setAlignment(Alignment);
3382 Value *V = propagateMetadata(LI, E->Scalars);
3383 if (IsReorder) {
3384 OrdersType Mask;
3385 inversePermutation(E->ReorderIndices, Mask);
3386 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3387 Mask, "reorder_shuffle");
3388 }
3389 if (NeedToShuffleReuses) {
3390 // TODO: Merge this shuffle with the ReorderShuffleMask.
3391 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3392 E->ReuseShuffleIndices, "shuffle");
3393 }
3394 E->VectorizedValue = V;
3395 ++NumVectorInstructions;
3396 return V;
3397 }
3398 case Instruction::Store: {
3399 StoreInst *SI = cast<StoreInst>(VL0);
3400 unsigned Alignment = SI->getAlignment();
3401 unsigned AS = SI->getPointerAddressSpace();
3402
3403 ValueList ScalarStoreValues;
3404 for (Value *V : E->Scalars)
3405 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3406
3407 setInsertPointAfterBundle(E->Scalars, VL0);
3408
3409 Value *VecValue = vectorizeTree(ScalarStoreValues);
3410 Value *ScalarPtr = SI->getPointerOperand();
3411 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3412 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3413
3414 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3415 // ExternalUses to make sure that an extract will be generated in the
3416 // future.
3417 if (getTreeEntry(ScalarPtr))
3418 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3419
3420 if (!Alignment)
3421 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3422
3423 S->setAlignment(Alignment);
3424 Value *V = propagateMetadata(S, E->Scalars);
3425 if (NeedToShuffleReuses) {
3426 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3427 E->ReuseShuffleIndices, "shuffle");
3428 }
3429 E->VectorizedValue = V;
3430 ++NumVectorInstructions;
3431 return V;
3432 }
3433 case Instruction::GetElementPtr: {
3434 setInsertPointAfterBundle(E->Scalars, VL0);
3435
3436 ValueList Op0VL;
3437 for (Value *V : E->Scalars)
3438 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3439
3440 Value *Op0 = vectorizeTree(Op0VL);
3441
3442 std::vector<Value *> OpVecs;
3443 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3444 ++j) {
3445 ValueList OpVL;
3446 for (Value *V : E->Scalars)
3447 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3448
3449 Value *OpVec = vectorizeTree(OpVL);
3450 OpVecs.push_back(OpVec);
3451 }
3452
3453 Value *V = Builder.CreateGEP(
3454 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3455 if (Instruction *I = dyn_cast<Instruction>(V))
3456 V = propagateMetadata(I, E->Scalars);
3457
3458 if (NeedToShuffleReuses) {
3459 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3460 E->ReuseShuffleIndices, "shuffle");
3461 }
3462 E->VectorizedValue = V;
3463 ++NumVectorInstructions;
3464
3465 return V;
3466 }
3467 case Instruction::Call: {
3468 CallInst *CI = cast<CallInst>(VL0);
3469 setInsertPointAfterBundle(E->Scalars, VL0);
3470 Function *FI;
3471 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3472 Value *ScalarArg = nullptr;
3473 if (CI && (FI = CI->getCalledFunction())) {
3474 IID = FI->getIntrinsicID();
3475 }
3476 std::vector<Value *> OpVecs;
3477 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3478 ValueList OpVL;
3479 // ctlz,cttz and powi are special intrinsics whose second argument is
3480 // a scalar. This argument should not be vectorized.
3481 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3482 CallInst *CEI = cast<CallInst>(VL0);
3483 ScalarArg = CEI->getArgOperand(j);
3484 OpVecs.push_back(CEI->getArgOperand(j));
3485 continue;
3486 }
3487 for (Value *V : E->Scalars) {
3488 CallInst *CEI = cast<CallInst>(V);
3489 OpVL.push_back(CEI->getArgOperand(j));
3490 }
3491
3492 Value *OpVec = vectorizeTree(OpVL);
3493 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
3494 OpVecs.push_back(OpVec);
3495 }
3496
3497 Module *M = F->getParent();
3498 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3499 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3500 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3501 SmallVector<OperandBundleDef, 1> OpBundles;
3502 CI->getOperandBundlesAsDefs(OpBundles);
3503 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3504
3505 // The scalar argument uses an in-tree scalar so we add the new vectorized
3506 // call to ExternalUses list to make sure that an extract will be
3507 // generated in the future.
3508 if (ScalarArg && getTreeEntry(ScalarArg))
3509 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3510
3511 propagateIRFlags(V, E->Scalars, VL0);
3512 if (NeedToShuffleReuses) {
3513 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3514 E->ReuseShuffleIndices, "shuffle");
3515 }
3516 E->VectorizedValue = V;
3517 ++NumVectorInstructions;
3518 return V;
3519 }
3520 case Instruction::ShuffleVector: {
3521 ValueList LHSVL, RHSVL;
3522 assert(Instruction::isBinaryOp(S.Opcode) &&(static_cast <bool> (Instruction::isBinaryOp(S.Opcode) &&
"Invalid Shuffle Vector Operand") ? void (0) : __assert_fail
("Instruction::isBinaryOp(S.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3523, __extension__ __PRETTY_FUNCTION__))
3523 "Invalid Shuffle Vector Operand")(static_cast <bool> (Instruction::isBinaryOp(S.Opcode) &&
"Invalid Shuffle Vector Operand") ? void (0) : __assert_fail
("Instruction::isBinaryOp(S.Opcode) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3523, __extension__ __PRETTY_FUNCTION__))
;
3524 reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3525 setInsertPointAfterBundle(E->Scalars, VL0);
3526
3527 Value *LHS = vectorizeTree(LHSVL);
3528 Value *RHS = vectorizeTree(RHSVL);
3529
3530 if (E->VectorizedValue) {
3531 DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3532 return E->VectorizedValue;
3533 }
3534
3535 // Create a vector of LHS op1 RHS
3536 Value *V0 = Builder.CreateBinOp(
3537 static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3538
3539 unsigned AltOpcode = getAltOpcode(S.Opcode);
3540 // Create a vector of LHS op2 RHS
3541 Value *V1 = Builder.CreateBinOp(
3542 static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3543
3544 // Create shuffle to take alternate operations from the vector.
3545 // Also, gather up odd and even scalar ops to propagate IR flags to
3546 // each vector operation.
3547 ValueList OddScalars, EvenScalars;
3548 unsigned e = E->Scalars.size();
3549 SmallVector<Constant *, 8> Mask(e);
3550 for (unsigned i = 0; i < e; ++i) {
3551 if (isOdd(i)) {
3552 Mask[i] = Builder.getInt32(e + i);
3553 OddScalars.push_back(E->Scalars[i]);
3554 } else {
3555 Mask[i] = Builder.getInt32(i);
3556 EvenScalars.push_back(E->Scalars[i]);
3557 }
3558 }
3559
3560 Value *ShuffleMask = ConstantVector::get(Mask);
3561 propagateIRFlags(V0, EvenScalars);
3562 propagateIRFlags(V1, OddScalars);
3563
3564 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3565 if (Instruction *I = dyn_cast<Instruction>(V))
3566 V = propagateMetadata(I, E->Scalars);
3567 if (NeedToShuffleReuses) {
3568 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3569 E->ReuseShuffleIndices, "shuffle");
3570 }
3571 E->VectorizedValue = V;
3572 ++NumVectorInstructions;
3573
3574 return V;
3575 }
3576 default:
3577 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3577)
;
3578 }
3579 return nullptr;
3580}
3581
3582Value *BoUpSLP::vectorizeTree() {
3583 ExtraValueToDebugLocsMap ExternallyUsedValues;
3584 return vectorizeTree(ExternallyUsedValues);
3585}
3586
3587Value *
3588BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3589 // All blocks must be scheduled before any instructions are inserted.
3590 for (auto &BSIter : BlocksSchedules) {
3591 scheduleBlock(BSIter.second.get());
3592 }
3593
3594 Builder.SetInsertPoint(&F->getEntryBlock().front());
3595 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3596
3597 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3598 // vectorized root. InstCombine will then rewrite the entire expression. We
3599 // sign extend the extracted values below.
3600 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3601 if (MinBWs.count(ScalarRoot)) {
3602 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3603 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3604 auto BundleWidth = VectorizableTree[0].Scalars.size();
3605 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3606 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3607 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3608 VectorizableTree[0].VectorizedValue = Trunc;
3609 }
3610
3611 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
3612
3613 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3614 // specified by ScalarType.
3615 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3616 if (!MinBWs.count(ScalarRoot))
3617 return Ex;
3618 if (MinBWs[ScalarRoot].second)
3619 return Builder.CreateSExt(Ex, ScalarType);
3620 return Builder.CreateZExt(Ex, ScalarType);
3621 };
3622
3623 // Extract all of the elements with the external uses.
3624 for (const auto &ExternalUse : ExternalUses) {
3625 Value *Scalar = ExternalUse.Scalar;
3626 llvm::User *User = ExternalUse.User;
3627
3628 // Skip users that we already RAUW. This happens when one instruction
3629 // has multiple uses of the same value.
3630 if (User && !is_contained(Scalar->users(), User))
3631 continue;
3632 TreeEntry *E = getTreeEntry(Scalar);
3633 assert(E && "Invalid scalar")(static_cast <bool> (E && "Invalid scalar") ? void
(0) : __assert_fail ("E && \"Invalid scalar\"", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3633, __extension__ __PRETTY_FUNCTION__))
;
3634 assert(!E->NeedToGather && "Extracting from a gather list")(static_cast <bool> (!E->NeedToGather && "Extracting from a gather list"
) ? void (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3634, __extension__ __PRETTY_FUNCTION__))
;
3635
3636 Value *Vec = E->VectorizedValue;
3637 assert(Vec && "Can't find vectorizable value")(static_cast <bool> (Vec && "Can't find vectorizable value"
) ? void (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3637, __extension__ __PRETTY_FUNCTION__))
;
3638
3639 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3640 // If User == nullptr, the Scalar is used as extra arg. Generate
3641 // ExtractElement instruction and update the record for this scalar in
3642 // ExternallyUsedValues.
3643 if (!User) {
3644 assert(ExternallyUsedValues.count(Scalar) &&(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3646, __extension__ __PRETTY_FUNCTION__))
3645 "Scalar with nullptr as an external user must be registered in "(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3646, __extension__ __PRETTY_FUNCTION__))
3646 "ExternallyUsedValues map")(static_cast <bool> (ExternallyUsedValues.count(Scalar)
&& "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? void (0) : __assert_fail ("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3646, __extension__ __PRETTY_FUNCTION__))
;
3647 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3648 Builder.SetInsertPoint(VecI->getParent(),
3649 std::next(VecI->getIterator()));
3650 } else {
3651 Builder.SetInsertPoint(&F->getEntryBlock().front());
3652 }
3653 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3654 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3655 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3656 auto &Locs = ExternallyUsedValues[Scalar];
3657 ExternallyUsedValues.insert({Ex, Locs});
3658 ExternallyUsedValues.erase(Scalar);
3659 continue;
3660 }
3661
3662 // Generate extracts for out-of-tree users.
3663 // Find the insertion point for the extractelement lane.
3664 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3665 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3666 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3667 if (PH->getIncomingValue(i) == Scalar) {
3668 TerminatorInst *IncomingTerminator =
3669 PH->getIncomingBlock(i)->getTerminator();
3670 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3671 Builder.SetInsertPoint(VecI->getParent(),
3672 std::next(VecI->getIterator()));
3673 } else {
3674 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3675 }
3676 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3677 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3678 CSEBlocks.insert(PH->getIncomingBlock(i));
3679 PH->setOperand(i, Ex);
3680 }
3681 }
3682 } else {
3683 Builder.SetInsertPoint(cast<Instruction>(User));
3684 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3685 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3686 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3687 User->replaceUsesOfWith(Scalar, Ex);
3688 }
3689 } else {
3690 Builder.SetInsertPoint(&F->getEntryBlock().front());
3691 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3692 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3693 CSEBlocks.insert(&F->getEntryBlock());
3694 User->replaceUsesOfWith(Scalar, Ex);
3695 }
3696
3697 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
3698 }
3699
3700 // For each vectorized value:
3701 for (TreeEntry &EIdx : VectorizableTree) {
3702 TreeEntry *Entry = &EIdx;
3703
3704 // No need to handle users of gathered values.
3705 if (Entry->NeedToGather)
3706 continue;
3707
3708 assert(Entry->VectorizedValue && "Can't find vectorizable value")(static_cast <bool> (Entry->VectorizedValue &&
"Can't find vectorizable value") ? void (0) : __assert_fail (
"Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3708, __extension__ __PRETTY_FUNCTION__))
;
3709
3710 // For each lane:
3711 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3712 Value *Scalar = Entry->Scalars[Lane];
3713
3714 Type *Ty = Scalar->getType();
3715 if (!Ty->isVoidTy()) {
3716#ifndef NDEBUG
3717 for (User *U : Scalar->users()) {
3718 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
3719
3720 // It is legal to replace users in the ignorelist by undef.
3721 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&(static_cast <bool> ((getTreeEntry(U) || is_contained(UserIgnoreList
, U)) && "Replacing out-of-tree value with undef") ? void
(0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3722, __extension__ __PRETTY_FUNCTION__))
3722 "Replacing out-of-tree value with undef")(static_cast <bool> ((getTreeEntry(U) || is_contained(UserIgnoreList
, U)) && "Replacing out-of-tree value with undef") ? void
(0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3722, __extension__ __PRETTY_FUNCTION__))
;
3723 }
3724#endif
3725 Value *Undef = UndefValue::get(Ty);
3726 Scalar->replaceAllUsesWith(Undef);
3727 }
3728 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
3729 eraseInstruction(cast<Instruction>(Scalar));
3730 }
3731 }
3732
3733 Builder.ClearInsertionPoint();
3734
3735 return VectorizableTree[0].VectorizedValue;
3736}
3737
3738void BoUpSLP::optimizeGatherSequence() {
3739 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
3740 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
3741 // LICM InsertElementInst sequences.
3742 for (Instruction *I : GatherSeq) {
3743 if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3744 continue;
3745
3746 // Check if this block is inside a loop.
3747 Loop *L = LI->getLoopFor(I->getParent());
3748 if (!L)
3749 continue;
3750
3751 // Check if it has a preheader.
3752 BasicBlock *PreHeader = L->getLoopPreheader();
3753 if (!PreHeader)
3754 continue;
3755
3756 // If the vector or the element that we insert into it are
3757 // instructions that are defined in this basic block then we can't
3758 // hoist this instruction.
3759 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3760 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3761 if (Op0 && L->contains(Op0))
3762 continue;
3763 if (Op1 && L->contains(Op1))
3764 continue;
3765
3766 // We can hoist this instruction. Move it to the pre-header.
3767 I->moveBefore(PreHeader->getTerminator());
3768 }
3769
3770 // Make a list of all reachable blocks in our CSE queue.
3771 SmallVector<const DomTreeNode *, 8> CSEWorkList;
3772 CSEWorkList.reserve(CSEBlocks.size());
3773 for (BasicBlock *BB : CSEBlocks)
3774 if (DomTreeNode *N = DT->getNode(BB)) {
3775 assert(DT->isReachableFromEntry(N))(static_cast <bool> (DT->isReachableFromEntry(N)) ? void
(0) : __assert_fail ("DT->isReachableFromEntry(N)", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3775, __extension__ __PRETTY_FUNCTION__))
;
3776 CSEWorkList.push_back(N);
3777 }
3778
3779 // Sort blocks by domination. This ensures we visit a block after all blocks
3780 // dominating it are visited.
3781 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3782 [this](const DomTreeNode *A, const DomTreeNode *B) {
3783 return DT->properlyDominates(A, B);
3784 });
3785
3786 // Perform O(N^2) search over the gather sequences and merge identical
3787 // instructions. TODO: We can further optimize this scan if we split the
3788 // instructions into different buckets based on the insert lane.
3789 SmallVector<Instruction *, 16> Visited;
3790 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3791 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(static_cast <bool> ((I == CSEWorkList.begin() || !DT->
dominates(*I, *std::prev(I))) && "Worklist not sorted properly!"
) ? void (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3792, __extension__ __PRETTY_FUNCTION__))
3792 "Worklist not sorted properly!")(static_cast <bool> ((I == CSEWorkList.begin() || !DT->
dominates(*I, *std::prev(I))) && "Worklist not sorted properly!"
) ? void (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3792, __extension__ __PRETTY_FUNCTION__))
;
3793 BasicBlock *BB = (*I)->getBlock();
3794 // For all instructions in blocks containing gather sequences:
3795 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3796 Instruction *In = &*it++;
3797 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3798 continue;
3799
3800 // Check if we can replace this instruction with any of the
3801 // visited instructions.
3802 for (Instruction *v : Visited) {
3803 if (In->isIdenticalTo(v) &&
3804 DT->dominates(v->getParent(), In->getParent())) {
3805 In->replaceAllUsesWith(v);
3806 eraseInstruction(In);
3807 In = nullptr;
3808 break;
3809 }
3810 }
3811 if (In) {
3812 assert(!is_contained(Visited, In))(static_cast <bool> (!is_contained(Visited, In)) ? void
(0) : __assert_fail ("!is_contained(Visited, In)", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3812, __extension__ __PRETTY_FUNCTION__))
;
3813 Visited.push_back(In);
3814 }
3815 }
3816 }
3817 CSEBlocks.clear();
3818 GatherSeq.clear();
3819}
3820
3821// Groups the instructions to a bundle (which is then a single scheduling entity)
3822// and schedules instructions until the bundle gets ready.
3823bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3824 BoUpSLP *SLP, Value *OpValue) {
3825 if (isa<PHINode>(OpValue))
22
Taking false branch
3826 return true;
3827
3828 // Initialize the instruction bundle.
3829 Instruction *OldScheduleEnd = ScheduleEnd;
3830 ScheduleData *PrevInBundle = nullptr;
3831 ScheduleData *Bundle = nullptr;
23
'Bundle' initialized to a null pointer value
3832 bool ReSchedule = false;
3833 DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *OpValue
<< "\n"; } } while (false)
;
3834
3835 // Make sure that the scheduling region contains all
3836 // instructions of the bundle.
3837 for (Value *V : VL) {
24
Assuming '__begin1' is equal to '__end1'
3838 if (!extendSchedulingRegion(V, OpValue))
3839 return false;
3840 }
3841
3842 for (Value *V : VL) {
25
Assuming '__begin1' is equal to '__end1'
3843 ScheduleData *BundleMember = getScheduleData(V);
3844 assert(BundleMember &&(static_cast <bool> (BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? void (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3845, __extension__ __PRETTY_FUNCTION__))
3845 "no ScheduleData for bundle member (maybe not in same basic block)")(static_cast <bool> (BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? void (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3845, __extension__ __PRETTY_FUNCTION__))
;
3846 if (BundleMember->IsScheduled) {
3847 // A bundle member was scheduled as single instruction before and now
3848 // needs to be scheduled as part of the bundle. We just get rid of the
3849 // existing schedule.
3850 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3851 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
3852 ReSchedule = true;
3853 }
3854 assert(BundleMember->isSchedulingEntity() &&(static_cast <bool> (BundleMember->isSchedulingEntity
() && "bundle member already part of other bundle") ?
void (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3855, __extension__ __PRETTY_FUNCTION__))
3855 "bundle member already part of other bundle")(static_cast <bool> (BundleMember->isSchedulingEntity
() && "bundle member already part of other bundle") ?
void (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3855, __extension__ __PRETTY_FUNCTION__))
;
3856 if (PrevInBundle) {
3857 PrevInBundle->NextInBundle = BundleMember;
3858 } else {
3859 Bundle = BundleMember;
3860 }
3861 BundleMember->UnscheduledDepsInBundle = 0;
3862 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3863
3864 // Group the instructions to a bundle.
3865 BundleMember->FirstInBundle = Bundle;
3866 PrevInBundle = BundleMember;
3867 }
3868 if (ScheduleEnd != OldScheduleEnd) {
26
Taking false branch
3869 // The scheduling region got new instructions at the lower end (or it is a
3870 // new region for the first bundle). This makes it necessary to
3871 // recalculate all dependencies.
3872 // It is seldom that this needs to be done a second time after adding the
3873 // initial bundle to the region.
3874 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3875 doForAllOpcodes(I, [](ScheduleData *SD) {
3876 SD->clearDependencies();
3877 });
3878 }
3879 ReSchedule = true;
3880 }
3881 if (ReSchedule) {
27
Taking false branch
3882 resetSchedule();
3883 initialFillReadyList(ReadyInsts);
3884 }
3885
3886 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
3887 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
3888
3889 calculateDependencies(Bundle, true, SLP);
3890
3891 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3892 // means that there are no cyclic dependencies and we can schedule it.
3893 // Note that's important that we don't "schedule" the bundle yet (see
3894 // cancelScheduling).
3895 while (!Bundle->isReady() && !ReadyInsts.empty()) {
28
Called C++ object pointer is null
3896
3897 ScheduleData *pickedSD = ReadyInsts.back();
3898 ReadyInsts.pop_back();
3899
3900 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3901 schedule(pickedSD, ReadyInsts);
3902 }
3903 }
3904 if (!Bundle->isReady()) {
3905 cancelScheduling(VL, OpValue);
3906 return false;
3907 }
3908 return true;
3909}
3910
3911void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3912 Value *OpValue) {
3913 if (isa<PHINode>(OpValue))
3914 return;
3915
3916 ScheduleData *Bundle = getScheduleData(OpValue);
3917 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3918 assert(!Bundle->IsScheduled &&(static_cast <bool> (!Bundle->IsScheduled &&
"Can't cancel bundle which is already scheduled") ? void (0)
: __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3919, __extension__ __PRETTY_FUNCTION__))
3919 "Can't cancel bundle which is already scheduled")(static_cast <bool> (!Bundle->IsScheduled &&
"Can't cancel bundle which is already scheduled") ? void (0)
: __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3919, __extension__ __PRETTY_FUNCTION__))
;
3920 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&(static_cast <bool> (Bundle->isSchedulingEntity() &&
Bundle->isPartOfBundle() && "tried to unbundle something which is not a bundle"
) ? void (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3921, __extension__ __PRETTY_FUNCTION__))
3921 "tried to unbundle something which is not a bundle")(static_cast <bool> (Bundle->isSchedulingEntity() &&
Bundle->isPartOfBundle() && "tried to unbundle something which is not a bundle"
) ? void (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3921, __extension__ __PRETTY_FUNCTION__))
;
3922
3923 // Un-bundle: make single instructions out of the bundle.
3924 ScheduleData *BundleMember = Bundle;
3925 while (BundleMember) {
3926 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")(static_cast <bool> (BundleMember->FirstInBundle == Bundle
&& "corrupt bundle links") ? void (0) : __assert_fail
("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3926, __extension__ __PRETTY_FUNCTION__))
;
3927 BundleMember->FirstInBundle = BundleMember;
3928 ScheduleData *Next = BundleMember->NextInBundle;
3929 BundleMember->NextInBundle = nullptr;
3930 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3931 if (BundleMember->UnscheduledDepsInBundle == 0) {
3932 ReadyInsts.insert(BundleMember);
3933 }
3934 BundleMember = Next;
3935 }
3936}
3937
3938BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3939 // Allocate a new ScheduleData for the instruction.
3940 if (ChunkPos >= ChunkSize) {
3941 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3942 ChunkPos = 0;
3943 }
3944 return &(ScheduleDataChunks.back()[ChunkPos++]);
3945}
3946
3947bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3948 Value *OpValue) {
3949 if (getScheduleData(V, isOneOf(OpValue, V)))
3950 return true;
3951 Instruction *I = dyn_cast<Instruction>(V);
3952 assert(I && "bundle member must be an instruction")(static_cast <bool> (I && "bundle member must be an instruction"
) ? void (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3952, __extension__ __PRETTY_FUNCTION__))
;
3953 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")(static_cast <bool> (!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? void (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3953, __extension__ __PRETTY_FUNCTION__))
;
3954 auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3955 ScheduleData *ISD = getScheduleData(I);
3956 if (!ISD)
3957 return false;
3958 assert(isInSchedulingRegion(ISD) &&(static_cast <bool> (isInSchedulingRegion(ISD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3959, __extension__ __PRETTY_FUNCTION__))
3959 "ScheduleData not in scheduling region")(static_cast <bool> (isInSchedulingRegion(ISD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3959, __extension__ __PRETTY_FUNCTION__))
;
3960 ScheduleData *SD = allocateScheduleDataChunks();
3961 SD->Inst = I;
3962 SD->init(SchedulingRegionID, OpValue);
3963 ExtraScheduleDataMap[I][OpValue] = SD;
3964 return true;
3965 };
3966 if (CheckSheduleForI(I))
3967 return true;
3968 if (!ScheduleStart) {
3969 // It's the first instruction in the new region.
3970 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3971 ScheduleStart = I;
3972 ScheduleEnd = I->getNextNode();
3973 if (isOneOf(OpValue, I) != I)
3974 CheckSheduleForI(I);
3975 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")(static_cast <bool> (ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? void (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3975, __extension__ __PRETTY_FUNCTION__))
;
3976 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3977 return true;
3978 }
3979 // Search up and down at the same time, because we don't know if the new
3980 // instruction is above or below the existing scheduling region.
3981 BasicBlock::reverse_iterator UpIter =
3982 ++ScheduleStart->getIterator().getReverse();
3983 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3984 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3985 BasicBlock::iterator LowerEnd = BB->end();
3986 while (true) {
3987 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3988 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3989 return false;
3990 }
3991
3992 if (UpIter != UpperEnd) {
3993 if (&*UpIter == I) {
3994 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3995 ScheduleStart = I;
3996 if (isOneOf(OpValue, I) != I)
3997 CheckSheduleForI(I);
3998 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3999 return true;
4000 }
4001 UpIter++;
4002 }
4003 if (DownIter != LowerEnd) {
4004 if (&*DownIter == I) {
4005 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
4006 nullptr);
4007 ScheduleEnd = I->getNextNode();
4008 if (isOneOf(OpValue, I) != I)
4009 CheckSheduleForI(I);
4010 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")(static_cast <bool> (ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? void (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4010, __extension__ __PRETTY_FUNCTION__))
;
4011 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
4012 return true;
4013 }
4014 DownIter++;
4015 }
4016 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(static_cast <bool> ((UpIter != UpperEnd || DownIter !=
LowerEnd) && "instruction not found in block") ? void
(0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4017, __extension__ __PRETTY_FUNCTION__))
4017 "instruction not found in block")(static_cast <bool> ((UpIter != UpperEnd || DownIter !=
LowerEnd) && "instruction not found in block") ? void
(0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4017, __extension__ __PRETTY_FUNCTION__))
;
4018 }
4019 return true;
4020}
4021
4022void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
4023 Instruction *ToI,
4024 ScheduleData *PrevLoadStore,
4025 ScheduleData *NextLoadStore) {
4026 ScheduleData *CurrentLoadStore = PrevLoadStore;
4027 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
4028 ScheduleData *SD = ScheduleDataMap[I];
4029 if (!SD) {
4030 SD = allocateScheduleDataChunks();
4031 ScheduleDataMap[I] = SD;
4032 SD->Inst = I;
4033 }
4034 assert(!isInSchedulingRegion(SD) &&(static_cast <bool> (!isInSchedulingRegion(SD) &&
"new ScheduleData already in scheduling region") ? void (0) :
__assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4035, __extension__ __PRETTY_FUNCTION__))
4035 "new ScheduleData already in scheduling region")(static_cast <bool> (!isInSchedulingRegion(SD) &&
"new ScheduleData already in scheduling region") ? void (0) :
__assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4035, __extension__ __PRETTY_FUNCTION__))
;
4036 SD->init(SchedulingRegionID, I);
4037
4038 if (I->mayReadOrWriteMemory() &&
4039 (!isa<IntrinsicInst>(I) ||
4040 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
4041 // Update the linked list of memory accessing instructions.
4042 if (CurrentLoadStore) {
4043 CurrentLoadStore->NextLoadStore = SD;
4044 } else {
4045 FirstLoadStoreInRegion = SD;
4046 }
4047 CurrentLoadStore = SD;
4048 }
4049 }
4050 if (NextLoadStore) {
4051 if (CurrentLoadStore)
4052 CurrentLoadStore->NextLoadStore = NextLoadStore;
4053 } else {
4054 LastLoadStoreInRegion = CurrentLoadStore;
4055 }
4056}
4057
4058void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
4059 bool InsertInReadyList,
4060 BoUpSLP *SLP) {
4061 assert(SD->isSchedulingEntity())(static_cast <bool> (SD->isSchedulingEntity()) ? void
(0) : __assert_fail ("SD->isSchedulingEntity()", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4061, __extension__ __PRETTY_FUNCTION__))
;
4062
4063 SmallVector<ScheduleData *, 10> WorkList;
4064 WorkList.push_back(SD);
4065
4066 while (!WorkList.empty()) {
4067 ScheduleData *SD = WorkList.back();
4068 WorkList.pop_back();
4069
4070 ScheduleData *BundleMember = SD;
4071 while (BundleMember) {
4072 assert(isInSchedulingRegion(BundleMember))(static_cast <bool> (isInSchedulingRegion(BundleMember)
) ? void (0) : __assert_fail ("isInSchedulingRegion(BundleMember)"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4072, __extension__ __PRETTY_FUNCTION__))
;
4073 if (!BundleMember->hasValidDependencies()) {
4074
4075 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
4076 BundleMember->Dependencies = 0;
4077 BundleMember->resetUnscheduledDeps();
4078
4079 // Handle def-use chain dependencies.
4080 if (BundleMember->OpValue != BundleMember->Inst) {
4081 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
4082 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4083 BundleMember->Dependencies++;
4084 ScheduleData *DestBundle = UseSD->FirstInBundle;
4085 if (!DestBundle->IsScheduled)
4086 BundleMember->incrementUnscheduledDeps(1);
4087 if (!DestBundle->hasValidDependencies())
4088 WorkList.push_back(DestBundle);
4089 }
4090 } else {
4091 for (User *U : BundleMember->Inst->users()) {
4092 if (isa<Instruction>(U)) {
4093 ScheduleData *UseSD = getScheduleData(U);
4094 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4095 BundleMember->Dependencies++;
4096 ScheduleData *DestBundle = UseSD->FirstInBundle;
4097 if (!DestBundle->IsScheduled)
4098 BundleMember->incrementUnscheduledDeps(1);
4099 if (!DestBundle->hasValidDependencies())
4100 WorkList.push_back(DestBundle);
4101 }
4102 } else {
4103 // I'm not sure if this can ever happen. But we need to be safe.
4104 // This lets the instruction/bundle never be scheduled and
4105 // eventually disable vectorization.
4106 BundleMember->Dependencies++;
4107 BundleMember->incrementUnscheduledDeps(1);
4108 }
4109 }
4110 }
4111
4112 // Handle the memory dependencies.
4113 ScheduleData *DepDest = BundleMember->NextLoadStore;
4114 if (DepDest) {
4115 Instruction *SrcInst = BundleMember->Inst;
4116 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
4117 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
4118 unsigned numAliased = 0;
4119 unsigned DistToSrc = 1;
4120
4121 while (DepDest) {
4122 assert(isInSchedulingRegion(DepDest))(static_cast <bool> (isInSchedulingRegion(DepDest)) ? void
(0) : __assert_fail ("isInSchedulingRegion(DepDest)", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4122, __extension__ __PRETTY_FUNCTION__))
;
4123
4124 // We have two limits to reduce the complexity:
4125 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
4126 // SLP->isAliased (which is the expensive part in this loop).
4127 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
4128 // the whole loop (even if the loop is fast, it's quadratic).
4129 // It's important for the loop break condition (see below) to
4130 // check this limit even between two read-only instructions.
4131 if (DistToSrc >= MaxMemDepDistance ||
4132 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
4133 (numAliased >= AliasedCheckLimit ||
4134 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
4135
4136 // We increment the counter only if the locations are aliased
4137 // (instead of counting all alias checks). This gives a better
4138 // balance between reduced runtime and accurate dependencies.
4139 numAliased++;
4140
4141 DepDest->MemoryDependencies.push_back(BundleMember);
4142 BundleMember->Dependencies++;
4143 ScheduleData *DestBundle = DepDest->FirstInBundle;
4144 if (!DestBundle->IsScheduled) {
4145 BundleMember->incrementUnscheduledDeps(1);
4146 }
4147 if (!DestBundle->hasValidDependencies()) {
4148 WorkList.push_back(DestBundle);
4149 }
4150 }
4151 DepDest = DepDest->NextLoadStore;
4152
4153 // Example, explaining the loop break condition: Let's assume our
4154 // starting instruction is i0 and MaxMemDepDistance = 3.
4155 //
4156 // +--------v--v--v
4157 // i0,i1,i2,i3,i4,i5,i6,i7,i8
4158 // +--------^--^--^
4159 //
4160 // MaxMemDepDistance let us stop alias-checking at i3 and we add
4161 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
4162 // Previously we already added dependencies from i3 to i6,i7,i8
4163 // (because of MaxMemDepDistance). As we added a dependency from
4164 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4165 // and we can abort this loop at i6.
4166 if (DistToSrc >= 2 * MaxMemDepDistance)
4167 break;
4168 DistToSrc++;
4169 }
4170 }
4171 }
4172 BundleMember = BundleMember->NextInBundle;
4173 }
4174 if (InsertInReadyList && SD->isReady()) {
4175 ReadyInsts.push_back(SD);
4176 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
4177 }
4178 }
4179}
4180
4181void BoUpSLP::BlockScheduling::resetSchedule() {
4182 assert(ScheduleStart &&(static_cast <bool> (ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? void (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4183, __extension__ __PRETTY_FUNCTION__))
4183 "tried to reset schedule on block which has not been scheduled")(static_cast <bool> (ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? void (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4183, __extension__ __PRETTY_FUNCTION__))
;
4184 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4185 doForAllOpcodes(I, [&](ScheduleData *SD) {
4186 assert(isInSchedulingRegion(SD) &&(static_cast <bool> (isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4187, __extension__ __PRETTY_FUNCTION__))
4187 "ScheduleData not in scheduling region")(static_cast <bool> (isInSchedulingRegion(SD) &&
"ScheduleData not in scheduling region") ? void (0) : __assert_fail
("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4187, __extension__ __PRETTY_FUNCTION__))
;
4188 SD->IsScheduled = false;
4189 SD->resetUnscheduledDeps();
4190 });
4191 }
4192 ReadyInsts.clear();
4193}
4194
4195void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4196 if (!BS->ScheduleStart)
4197 return;
4198
4199 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
4200
4201 BS->resetSchedule();
4202
4203 // For the real scheduling we use a more sophisticated ready-list: it is
4204 // sorted by the original instruction location. This lets the final schedule
4205 // be as close as possible to the original instruction order.
4206 struct ScheduleDataCompare {
4207 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4208 return SD2->SchedulingPriority < SD1->SchedulingPriority;
4209 }
4210 };
4211 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4212
4213 // Ensure that all dependency data is updated and fill the ready-list with
4214 // initial instructions.
4215 int Idx = 0;
4216 int NumToSchedule = 0;
4217 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4218 I = I->getNextNode()) {
4219 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4220 assert(SD->isPartOfBundle() ==(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4222, __extension__ __PRETTY_FUNCTION__))
4221 (getTreeEntry(SD->Inst) != nullptr) &&(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4222, __extension__ __PRETTY_FUNCTION__))
4222 "scheduler and vectorizer bundle mismatch")(static_cast <bool> (SD->isPartOfBundle() == (getTreeEntry
(SD->Inst) != nullptr) && "scheduler and vectorizer bundle mismatch"
) ? void (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4222, __extension__ __PRETTY_FUNCTION__))
;
4223 SD->FirstInBundle->SchedulingPriority = Idx++;
4224 if (SD->isSchedulingEntity()) {
4225 BS->calculateDependencies(SD, false, this);
4226 NumToSchedule++;
4227 }
4228 });
4229 }
4230 BS->initialFillReadyList(ReadyInsts);
4231
4232 Instruction *LastScheduledInst = BS->ScheduleEnd;
4233
4234 // Do the "real" scheduling.
4235 while (!ReadyInsts.empty()) {
4236 ScheduleData *picked = *ReadyInsts.begin();
4237 ReadyInsts.erase(ReadyInsts.begin());
4238
4239 // Move the scheduled instruction(s) to their dedicated places, if not
4240 // there yet.
4241 ScheduleData *BundleMember = picked;
4242 while (BundleMember) {
4243 Instruction *pickedInst = BundleMember->Inst;
4244 if (LastScheduledInst->getNextNode() != pickedInst) {
4245 BS->BB->getInstList().remove(pickedInst);
4246 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4247 pickedInst);
4248 }
4249 LastScheduledInst = pickedInst;
4250 BundleMember = BundleMember->NextInBundle;
4251 }
4252
4253 BS->schedule(picked, ReadyInsts);
4254 NumToSchedule--;
4255 }
4256 assert(NumToSchedule == 0 && "could not schedule all instructions")(static_cast <bool> (NumToSchedule == 0 && "could not schedule all instructions"
) ? void (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4256, __extension__ __PRETTY_FUNCTION__))
;
4257
4258 // Avoid duplicate scheduling of the block.
4259 BS->ScheduleStart = nullptr;
4260}
4261
4262unsigned BoUpSLP::getVectorElementSize(Value *V) {
4263 // If V is a store, just return the width of the stored value without
4264 // traversing the expression tree. This is the common case.
4265 if (auto *Store = dyn_cast<StoreInst>(V))
4266 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4267
4268 // If V is not a store, we can traverse the expression tree to find loads
4269 // that feed it. The type of the loaded value may indicate a more suitable
4270 // width than V's type. We want to base the vector element size on the width
4271 // of memory operations where possible.
4272 SmallVector<Instruction *, 16> Worklist;
4273 SmallPtrSet<Instruction *, 16> Visited;
4274 if (auto *I = dyn_cast<Instruction>(V))
4275 Worklist.push_back(I);
4276
4277 // Traverse the expression tree in bottom-up order looking for loads. If we
4278 // encounter an instruciton we don't yet handle, we give up.
4279 auto MaxWidth = 0u;
4280 auto FoundUnknownInst = false;
4281 while (!Worklist.empty() && !FoundUnknownInst) {
4282 auto *I = Worklist.pop_back_val();
4283 Visited.insert(I);
4284
4285 // We should only be looking at scalar instructions here. If the current
4286 // instruction has a vector type, give up.
4287 auto *Ty = I->getType();
4288 if (isa<VectorType>(Ty))
4289 FoundUnknownInst = true;
4290
4291 // If the current instruction is a load, update MaxWidth to reflect the
4292 // width of the loaded value.
4293 else if (isa<LoadInst>(I))
4294 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4295
4296 // Otherwise, we need to visit the operands of the instruction. We only
4297 // handle the interesting cases from buildTree here. If an operand is an
4298 // instruction we haven't yet visited, we add it to the worklist.
4299 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4300 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4301 for (Use &U : I->operands())
4302 if (auto *J = dyn_cast<Instruction>(U.get()))
4303 if (!Visited.count(J))
4304 Worklist.push_back(J);
4305 }
4306
4307 // If we don't yet handle the instruction, give up.
4308 else
4309 FoundUnknownInst = true;
4310 }
4311
4312 // If we didn't encounter a memory access in the expression tree, or if we
4313 // gave up for some reason, just return the width of V.
4314 if (!MaxWidth || FoundUnknownInst)
4315 return DL->getTypeSizeInBits(V->getType());
4316
4317 // Otherwise, return the maximum width we found.
4318 return MaxWidth;
4319}
4320
4321// Determine if a value V in a vectorizable expression Expr can be demoted to a
4322// smaller type with a truncation. We collect the values that will be demoted
4323// in ToDemote and additional roots that require investigating in Roots.
4324static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
4325 SmallVectorImpl<Value *> &ToDemote,
4326 SmallVectorImpl<Value *> &Roots) {
4327 // We can always demote constants.
4328 if (isa<Constant>(V)) {
4329 ToDemote.push_back(V);
4330 return true;
4331 }
4332
4333 // If the value is not an instruction in the expression with only one use, it
4334 // cannot be demoted.
4335 auto *I = dyn_cast<Instruction>(V);
4336 if (!I || !I->hasOneUse() || !Expr.count(I))
4337 return false;
4338
4339 switch (I->getOpcode()) {
4340
4341 // We can always demote truncations and extensions. Since truncations can
4342 // seed additional demotion, we save the truncated value.
4343 case Instruction::Trunc:
4344 Roots.push_back(I->getOperand(0));
4345 break;
4346 case Instruction::ZExt:
4347 case Instruction::SExt:
4348 break;
4349
4350 // We can demote certain binary operations if we can demote both of their
4351 // operands.
4352 case Instruction::Add:
4353 case Instruction::Sub:
4354 case Instruction::Mul:
4355 case Instruction::And:
4356 case Instruction::Or:
4357 case Instruction::Xor:
4358 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4359 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4360 return false;
4361 break;
4362
4363 // We can demote selects if we can demote their true and false values.
4364 case Instruction::Select: {
4365 SelectInst *SI = cast<SelectInst>(I);
4366 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4367 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4368 return false;
4369 break;
4370 }
4371
4372 // We can demote phis if we can demote all their incoming operands. Note that
4373 // we don't need to worry about cycles since we ensure single use above.
4374 case Instruction::PHI: {
4375 PHINode *PN = cast<PHINode>(I);
4376 for (Value *IncValue : PN->incoming_values())
4377 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4378 return false;
4379 break;
4380 }
4381
4382 // Otherwise, conservatively give up.
4383 default:
4384 return false;
4385 }
4386
4387 // Record the value that we can demote.
4388 ToDemote.push_back(V);
4389 return true;
4390}
4391
4392void BoUpSLP::computeMinimumValueSizes() {
4393 // If there are no external uses, the expression tree must be rooted by a
4394 // store. We can't demote in-memory values, so there is nothing to do here.
4395 if (ExternalUses.empty())
4396 return;
4397
4398 // We only attempt to truncate integer expressions.
4399 auto &TreeRoot = VectorizableTree[0].Scalars;
4400 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4401 if (!TreeRootIT)
4402 return;
4403
4404 // If the expression is not rooted by a store, these roots should have
4405 // external uses. We will rely on InstCombine to rewrite the expression in
4406 // the narrower type. However, InstCombine only rewrites single-use values.
4407 // This means that if a tree entry other than a root is used externally, it
4408 // must have multiple uses and InstCombine will not rewrite it. The code
4409 // below ensures that only the roots are used externally.
4410 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4411 for (auto &EU : ExternalUses)
4412 if (!Expr.erase(EU.Scalar))
4413 return;
4414 if (!Expr.empty())
4415 return;
4416
4417 // Collect the scalar values of the vectorizable expression. We will use this
4418 // context to determine which values can be demoted. If we see a truncation,
4419 // we mark it as seeding another demotion.
4420 for (auto &Entry : VectorizableTree)
4421 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4422
4423 // Ensure the roots of the vectorizable tree don't form a cycle. They must
4424 // have a single external user that is not in the vectorizable tree.
4425 for (auto *Root : TreeRoot)
4426 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4427 return;
4428
4429 // Conservatively determine if we can actually truncate the roots of the
4430 // expression. Collect the values that can be demoted in ToDemote and
4431 // additional roots that require investigating in Roots.
4432 SmallVector<Value *, 32> ToDemote;
4433 SmallVector<Value *, 4> Roots;
4434 for (auto *Root : TreeRoot)
4435 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4436 return;
4437
4438 // The maximum bit width required to represent all the values that can be
4439 // demoted without loss of precision. It would be safe to truncate the roots
4440 // of the expression to this width.
4441 auto MaxBitWidth = 8u;
4442
4443 // We first check if all the bits of the roots are demanded. If they're not,
4444 // we can truncate the roots to this narrower type.
4445 for (auto *Root : TreeRoot) {
4446 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4447 MaxBitWidth = std::max<unsigned>(
4448 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4449 }
4450
4451 // True if the roots can be zero-extended back to their original type, rather
4452 // than sign-extended. We know that if the leading bits are not demanded, we
4453 // can safely zero-extend. So we initialize IsKnownPositive to True.
4454 bool IsKnownPositive = true;
4455
4456 // If all the bits of the roots are demanded, we can try a little harder to
4457 // compute a narrower type. This can happen, for example, if the roots are
4458 // getelementptr indices. InstCombine promotes these indices to the pointer
4459 // width. Thus, all their bits are technically demanded even though the
4460 // address computation might be vectorized in a smaller type.
4461 //
4462 // We start by looking at each entry that can be demoted. We compute the
4463 // maximum bit width required to store the scalar by using ValueTracking to
4464 // compute the number of high-order bits we can truncate.
4465 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
4466 llvm::all_of(TreeRoot, [](Value *R) {
4467 assert(R->hasOneUse() && "Root should have only one use!")(static_cast <bool> (R->hasOneUse() && "Root should have only one use!"
) ? void (0) : __assert_fail ("R->hasOneUse() && \"Root should have only one use!\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4467, __extension__ __PRETTY_FUNCTION__))
;
4468 return isa<GetElementPtrInst>(R->user_back());
4469 })) {
4470 MaxBitWidth = 8u;
4471
4472 // Determine if the sign bit of all the roots is known to be zero. If not,
4473 // IsKnownPositive is set to False.
4474 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4475 KnownBits Known = computeKnownBits(R, *DL);
4476 return Known.isNonNegative();
4477 });
4478
4479 // Determine the maximum number of bits required to store the scalar
4480 // values.
4481 for (auto *Scalar : ToDemote) {
4482 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4483 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4484 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4485 }
4486
4487 // If we can't prove that the sign bit is zero, we must add one to the
4488 // maximum bit width to account for the unknown sign bit. This preserves
4489 // the existing sign bit so we can safely sign-extend the root back to the
4490 // original type. Otherwise, if we know the sign bit is zero, we will
4491 // zero-extend the root instead.
4492 //
4493 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4494 // one to the maximum bit width will yield a larger-than-necessary
4495 // type. In general, we need to add an extra bit only if we can't
4496 // prove that the upper bit of the original type is equal to the
4497 // upper bit of the proposed smaller type. If these two bits are the
4498 // same (either zero or one) we know that sign-extending from the
4499 // smaller type will result in the same value. Here, since we can't
4500 // yet prove this, we are just making the proposed smaller type
4501 // larger to ensure correctness.
4502 if (!IsKnownPositive)
4503 ++MaxBitWidth;
4504 }
4505
4506 // Round MaxBitWidth up to the next power-of-two.
4507 if (!isPowerOf2_64(MaxBitWidth))
4508 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4509
4510 // If the maximum bit width we compute is less than the with of the roots'
4511 // type, we can proceed with the narrowing. Otherwise, do nothing.
4512 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4513 return;
4514
4515 // If we can truncate the root, we must collect additional values that might
4516 // be demoted as a result. That is, those seeded by truncations we will
4517 // modify.
4518 while (!Roots.empty())
4519 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4520
4521 // Finally, map the values we can demote to the maximum bit with we computed.
4522 for (auto *Scalar : ToDemote)
4523 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4524}
4525
4526namespace {
4527
4528/// The SLPVectorizer Pass.
4529struct SLPVectorizer : public FunctionPass {
4530 SLPVectorizerPass Impl;
4531
4532 /// Pass identification, replacement for typeid
4533 static char ID;
4534
4535 explicit SLPVectorizer() : FunctionPass(ID) {
4536 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4537 }
4538
4539 bool doInitialization(Module &M) override {
4540 return false;
4541 }
4542
4543 bool runOnFunction(Function &F) override {
4544 if (skipFunction(F))
4545 return false;
4546
4547 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4548 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4549 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4550 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4551 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4552 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4553 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4554 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4555 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4556 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4557
4558 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4559 }
4560
4561 void getAnalysisUsage(AnalysisUsage &AU) const override {
4562 FunctionPass::getAnalysisUsage(AU);
4563 AU.addRequired<AssumptionCacheTracker>();
4564 AU.addRequired<ScalarEvolutionWrapperPass>();
4565 AU.addRequired<AAResultsWrapperPass>();
4566 AU.addRequired<TargetTransformInfoWrapperPass>();
4567 AU.addRequired<LoopInfoWrapperPass>();
4568 AU.addRequired<DominatorTreeWrapperPass>();
4569 AU.addRequired<DemandedBitsWrapperPass>();
4570 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4571 AU.addPreserved<LoopInfoWrapperPass>();
4572 AU.addPreserved<DominatorTreeWrapperPass>();
4573 AU.addPreserved<AAResultsWrapperPass>();
4574 AU.addPreserved<GlobalsAAWrapperPass>();
4575 AU.setPreservesCFG();
4576 }
4577};
4578
4579} // end anonymous namespace
4580
4581PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4582 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4583 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4584 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4585 auto *AA = &AM.getResult<AAManager>(F);
4586 auto *LI = &AM.getResult<LoopAnalysis>(F);
4587 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4588 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4589 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4590 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4591
4592 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4593 if (!Changed)
4594 return PreservedAnalyses::all();
4595
4596 PreservedAnalyses PA;
4597 PA.preserveSet<CFGAnalyses>();
4598 PA.preserve<AAManager>();
4599 PA.preserve<GlobalsAA>();
4600 return PA;
4601}
4602
4603bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4604 TargetTransformInfo *TTI_,
4605 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4606 LoopInfo *LI_, DominatorTree *DT_,
4607 AssumptionCache *AC_, DemandedBits *DB_,
4608 OptimizationRemarkEmitter *ORE_) {
4609 SE = SE_;
4610 TTI = TTI_;
4611 TLI = TLI_;
4612 AA = AA_;
4613 LI = LI_;
4614 DT = DT_;
4615 AC = AC_;
4616 DB = DB_;
4617 DL = &F.getParent()->getDataLayout();
4618
4619 Stores.clear();
4620 GEPs.clear();
4621 bool Changed = false;
4622
4623 // If the target claims to have no vector registers don't attempt
4624 // vectorization.
4625 if (!TTI->getNumberOfRegisters(true))
4626 return false;
4627
4628 // Don't vectorize when the attribute NoImplicitFloat is used.
4629 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4630 return false;
4631
4632 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
4633
4634 // Use the bottom up slp vectorizer to construct chains that start with
4635 // store instructions.
4636 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4637
4638 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4639 // delete instructions.
4640
4641 // Scan the blocks in the function in post order.
4642 for (auto BB : post_order(&F.getEntryBlock())) {
4643 collectSeedInstructions(BB);
4644
4645 // Vectorize trees that end at stores.
4646 if (!Stores.empty()) {
4647 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
4648 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
4649 Changed |= vectorizeStoreChains(R);
4650 }
4651
4652 // Vectorize trees that end at reductions.
4653 Changed |= vectorizeChainsInBlock(BB, R);
4654
4655 // Vectorize the index computations of getelementptr instructions. This
4656 // is primarily intended to catch gather-like idioms ending at
4657 // non-consecutive loads.
4658 if (!GEPs.empty()) {
4659 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
4660 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
4661 Changed |= vectorizeGEPIndices(BB, R);
4662 }
4663 }
4664
4665 if (Changed) {
4666 R.optimizeGatherSequence();
4667 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
4668 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
4669 }
4670 return Changed;
4671}
4672
4673/// \brief Check that the Values in the slice in VL array are still existent in
4674/// the WeakTrackingVH array.
4675/// Vectorization of part of the VL array may cause later values in the VL array
4676/// to become invalid. We track when this has happened in the WeakTrackingVH
4677/// array.
4678static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4679 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4680 unsigned SliceSize) {
4681 VL = VL.slice(SliceBegin, SliceSize);
4682 VH = VH.slice(SliceBegin, SliceSize);
4683 return !std::equal(VL.begin(), VL.end(), VH.begin());
4684}
4685
4686bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4687 unsigned VecRegSize) {
4688 const unsigned ChainLen = Chain.size();
4689 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
4690 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
4691 const unsigned Sz = R.getVectorElementSize(Chain[0]);
4692 const unsigned VF = VecRegSize / Sz;
4693
4694 if (!isPowerOf2_32(Sz) || VF < 2)
4695 return false;
4696
4697 // Keep track of values that were deleted by vectorizing in the loop below.
4698 const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4699
4700 bool Changed = false;
4701 // Look for profitable vectorizable trees at all offsets, starting at zero.
4702 for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4703
4704 // Check that a previous iteration of this loop did not delete the Value.
4705 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4706 continue;
4707
4708 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
4709 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
4710 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4711
4712 R.buildTree(Operands);
4713 if (R.isTreeTinyAndNotFullyVectorizable())
4714 continue;
4715
4716 R.computeMinimumValueSizes();
4717
4718 int Cost = R.getTreeCost();
4719
4720 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
4721 if (Cost < -SLPCostThreshold) {
4722 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
4723
4724 using namespace ore;
4725
4726 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "StoresVectorized",
4727 cast<StoreInst>(Chain[i]))
4728 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4729 << " and with tree size "
4730 << NV("TreeSize", R.getTreeSize()));
4731
4732 R.vectorizeTree();
4733
4734 // Move to the next bundle.
4735 i += VF - 1;
4736 Changed = true;
4737 }
4738 }
4739
4740 return Changed;
4741}
4742
4743bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4744 BoUpSLP &R) {
4745 SetVector<StoreInst *> Heads;
4746 SmallDenseSet<StoreInst *> Tails;
4747 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4748
4749 // We may run into multiple chains that merge into a single chain. We mark the
4750 // stores that we vectorized so that we don't visit the same store twice.
4751 BoUpSLP::ValueSet VectorizedStores;
4752 bool Changed = false;
4753
4754 // Do a quadratic search on all of the given stores in reverse order and find
4755 // all of the pairs of stores that follow each other.
4756 SmallVector<unsigned, 16> IndexQueue;
4757 unsigned E = Stores.size();
4758 IndexQueue.resize(E - 1);
4759 for (unsigned I = E; I > 0; --I) {
4760 unsigned Idx = I - 1;
4761 // If a store has multiple consecutive store candidates, search Stores
4762 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4763 // This is because usually pairing with immediate succeeding or preceding
4764 // candidate create the best chance to find slp vectorization opportunity.
4765 unsigned Offset = 1;
4766 unsigned Cnt = 0;
4767 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4768 if (Idx >= Offset) {
4769 IndexQueue[Cnt] = Idx - Offset;
4770 ++Cnt;
4771 }
4772 if (Idx + Offset < E) {
4773 IndexQueue[Cnt] = Idx + Offset;
4774 ++Cnt;
4775 }
4776 }
4777
4778 for (auto K : IndexQueue) {
4779 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4780 Tails.insert(Stores[Idx]);
4781 Heads.insert(Stores[K]);
4782 ConsecutiveChain[Stores[K]] = Stores[Idx];
4783 break;
4784 }
4785 }
4786 }
4787
4788 // For stores that start but don't end a link in the chain:
4789 for (auto *SI : llvm::reverse(Heads)) {
4790 if (Tails.count(SI))
4791 continue;
4792
4793 // We found a store instr that starts a chain. Now follow the chain and try
4794 // to vectorize it.
4795 BoUpSLP::ValueList Operands;
4796 StoreInst *I = SI;
4797 // Collect the chain into a list.
4798 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4799 Operands.push_back(I);
4800 // Move to the next value in the chain.
4801 I = ConsecutiveChain[I];
4802 }
4803
4804 // FIXME: Is division-by-2 the correct step? Should we assert that the
4805 // register size is a power-of-2?
4806 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4807 Size /= 2) {
4808 if (vectorizeStoreChain(Operands, R, Size)) {
4809 // Mark the vectorized stores so that we don't vectorize them again.
4810 VectorizedStores.insert(Operands.begin(), Operands.end());
4811 Changed = true;
4812 break;
4813 }
4814 }
4815 }
4816
4817 return Changed;
4818}
4819
4820void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4821 // Initialize the collections. We will make a single pass over the block.
4822 Stores.clear();
4823 GEPs.clear();
4824
4825 // Visit the store and getelementptr instructions in BB and organize them in
4826 // Stores and GEPs according to the underlying objects of their pointer
4827 // operands.
4828 for (Instruction &I : *BB) {
4829 // Ignore store instructions that are volatile or have a pointer operand
4830 // that doesn't point to a scalar type.
4831 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4832 if (!SI->isSimple())
4833 continue;
4834 if (!isValidElementType(SI->getValueOperand()->getType()))
4835 continue;
4836 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4837 }
4838
4839 // Ignore getelementptr instructions that have more than one index, a
4840 // constant index, or a pointer operand that doesn't point to a scalar
4841 // type.
4842 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4843 auto Idx = GEP->idx_begin()->get();
4844 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4845 continue;
4846 if (!isValidElementType(Idx->getType()))
4847 continue;
4848 if (GEP->getType()->isVectorTy())
4849 continue;
4850 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4851 }
4852 }
4853}
4854
4855bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4856 if (!A || !B)
4857 return false;
4858 Value *VL[] = { A, B };
4859 return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4860}
4861
4862bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4863 int UserCost, bool AllowReorder) {
4864 if (VL.size() < 2)
4865 return false;
4866
4867 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4868 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
4869
4870 // Check that all of the parts are scalar instructions of the same type.
4871 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4872 if (!I0)
4873 return false;
4874
4875 unsigned Opcode0 = I0->getOpcode();
4876
4877 unsigned Sz = R.getVectorElementSize(I0);
4878 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4879 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4880 if (MaxVF < 2) {
4881 R.getORE()->emit([&]() {
4882 return OptimizationRemarkMissed(
4883 SV_NAME"slp-vectorizer", "SmallVF", I0)
4884 << "Cannot SLP vectorize list: vectorization factor "
4885 << "less than 2 is not supported";
4886 });
4887 return false;
4888 }
4889
4890 for (Value *V : VL) {
4891 Type *Ty = V->getType();
4892 if (!isValidElementType(Ty)) {
4893 // NOTE: the following will give user internal llvm type name, which may not be useful
4894 R.getORE()->emit([&]() {
4895 std::string type_str;
4896 llvm::raw_string_ostream rso(type_str);
4897 Ty->print(rso);
4898 return OptimizationRemarkMissed(
4899 SV_NAME"slp-vectorizer", "UnsupportedType", I0)
4900 << "Cannot SLP vectorize list: type "
4901 << rso.str() + " is unsupported by vectorizer";
4902 });
4903 return false;
4904 }
4905 Instruction *Inst = dyn_cast<Instruction>(V);
4906
4907 if (!Inst)
4908 return false;
4909 if (Inst->getOpcode() != Opcode0) {
4910 R.getORE()->emit([&]() {
4911 return OptimizationRemarkMissed(
4912 SV_NAME"slp-vectorizer", "InequableTypes", I0)
4913 << "Cannot SLP vectorize list: not all of the "
4914 << "parts of scalar instructions are of the same type: "
4915 << ore::NV("Instruction1Opcode", I0) << " and "
4916 << ore::NV("Instruction2Opcode", Inst);
4917 });
4918 return false;
4919 }
4920 }
4921
4922 bool Changed = false;
4923 bool CandidateFound = false;
4924 int MinCost = SLPCostThreshold;
4925
4926 // Keep track of values that were deleted by vectorizing in the loop below.
4927 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4928
4929 unsigned NextInst = 0, MaxInst = VL.size();
4930 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4931 VF /= 2) {
4932 // No actual vectorization should happen, if number of parts is the same as
4933 // provided vectorization factor (i.e. the scalar type is used for vector
4934 // code during codegen).
4935 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4936 if (TTI->getNumberOfParts(VecTy) == VF)
4937 continue;
4938 for (unsigned I = NextInst; I < MaxInst; ++I) {
4939 unsigned OpsWidth = 0;
4940
4941 if (I + VF > MaxInst)
4942 OpsWidth = MaxInst - I;
4943 else
4944 OpsWidth = VF;
4945
4946 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4947 break;
4948
4949 // Check that a previous iteration of this loop did not delete the Value.
4950 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4951 continue;
4952
4953 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4954 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
4955 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4956
4957 R.buildTree(Ops);
4958 Optional<ArrayRef<unsigned>> Order = R.bestOrder();
4959 // TODO: check if we can allow reordering for more cases.
4960 if (AllowReorder && Order) {
4961 // TODO: reorder tree nodes without tree rebuilding.
4962 // Conceptually, there is nothing actually preventing us from trying to
4963 // reorder a larger list. In fact, we do exactly this when vectorizing
4964 // reductions. However, at this point, we only expect to get here when
4965 // there are exactly two operations.
4966 assert(Ops.size() == 2)(static_cast <bool> (Ops.size() == 2) ? void (0) : __assert_fail
("Ops.size() == 2", "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4966, __extension__ __PRETTY_FUNCTION__))
;
4967 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4968 R.buildTree(ReorderedOps, None);
4969 }
4970 if (R.isTreeTinyAndNotFullyVectorizable())
4971 continue;
4972
4973 R.computeMinimumValueSizes();
4974 int Cost = R.getTreeCost() - UserCost;
4975 CandidateFound = true;
4976 MinCost = std::min(MinCost, Cost);
4977
4978 if (Cost < -SLPCostThreshold) {
4979 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
4980 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedList",
4981 cast<Instruction>(Ops[0]))
4982 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4983 << " and with tree size "
4984 << ore::NV("TreeSize", R.getTreeSize()));
4985
4986 R.vectorizeTree();
4987 // Move to the next bundle.
4988 I += VF - 1;
4989 NextInst = I + 1;
4990 Changed = true;
4991 }
4992 }
4993 }
4994
4995 if (!Changed && CandidateFound) {
4996 R.getORE()->emit([&]() {
4997 return OptimizationRemarkMissed(
4998 SV_NAME"slp-vectorizer", "NotBeneficial", I0)
4999 << "List vectorization was possible but not beneficial with cost "
5000 << ore::NV("Cost", MinCost) << " >= "
5001 << ore::NV("Treshold", -SLPCostThreshold);
5002 });
5003 } else if (!Changed) {
5004 R.getORE()->emit([&]() {
5005 return OptimizationRemarkMissed(
5006 SV_NAME"slp-vectorizer", "NotPossible", I0)
5007 << "Cannot SLP vectorize list: vectorization was impossible"
5008 << " with available vectorization factors";
5009 });
5010 }
5011 return Changed;
5012}
5013
5014bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
5015 if (!I)
5016 return false;
5017
5018 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
5019 return false;
5020
5021 Value *P = I->getParent();
5022
5023 // Vectorize in current basic block only.
5024 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
5025 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
5026 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
5027 return false;
5028
5029 // Try to vectorize V.
5030 if (tryToVectorizePair(Op0, Op1, R))
5031 return true;
5032
5033 auto *A = dyn_cast<BinaryOperator>(Op0);
5034 auto *B = dyn_cast<BinaryOperator>(Op1);
5035 // Try to skip B.
5036 if (B && B->hasOneUse()) {
5037 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
5038 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
5039 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
5040 return true;
5041 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
5042 return true;
5043 }
5044
5045 // Try to skip A.
5046 if (A && A->hasOneUse()) {
5047 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
5048 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
5049 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
5050 return true;
5051 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
5052 return true;
5053 }
5054 return false;
5055}
5056
5057/// \brief Generate a shuffle mask to be used in a reduction tree.
5058///
5059/// \param VecLen The length of the vector to be reduced.
5060/// \param NumEltsToRdx The number of elements that should be reduced in the
5061/// vector.
5062/// \param IsPairwise Whether the reduction is a pairwise or splitting
5063/// reduction. A pairwise reduction will generate a mask of
5064/// <0,2,...> or <1,3,..> while a splitting reduction will generate
5065/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
5066/// \param IsLeft True will generate a mask of even elements, odd otherwise.
5067static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
5068 bool IsPairwise, bool IsLeft,
5069 IRBuilder<> &Builder) {
5070 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(static_cast <bool> ((IsPairwise || !IsLeft) &&
"Don't support a <0,1,undef,...> mask") ? void (0) : __assert_fail
("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5070, __extension__ __PRETTY_FUNCTION__))
;
5071
5072 SmallVector<Constant *, 32> ShuffleMask(
5073 VecLen, UndefValue::get(Builder.getInt32Ty()));
5074
5075 if (IsPairwise)
5076 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
5077 for (unsigned i = 0; i != NumEltsToRdx; ++i)
5078 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
5079 else
5080 // Move the upper half of the vector to the lower half.
5081 for (unsigned i = 0; i != NumEltsToRdx; ++i)
5082 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
5083
5084 return ConstantVector::get(ShuffleMask);
5085}
5086
5087namespace {
5088
5089/// Model horizontal reductions.
5090///
5091/// A horizontal reduction is a tree of reduction operations (currently add and
5092/// fadd) that has operations that can be put into a vector as its leaf.
5093/// For example, this tree:
5094///
5095/// mul mul mul mul
5096/// \ / \ /
5097/// + +
5098/// \ /
5099/// +
5100/// This tree has "mul" as its reduced values and "+" as its reduction
5101/// operations. A reduction might be feeding into a store or a binary operation
5102/// feeding a phi.
5103/// ...
5104/// \ /
5105/// +
5106/// |
5107/// phi +=
5108///
5109/// Or:
5110/// ...
5111/// \ /
5112/// +
5113/// |
5114/// *p =
5115///
5116class HorizontalReduction {
5117 using ReductionOpsType = SmallVector<Value *, 16>;
5118 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
5119 ReductionOpsListType ReductionOps;
5120 SmallVector<Value *, 32> ReducedVals;
5121 // Use map vector to make stable output.
5122 MapVector<Instruction *, Value *> ExtraArgs;
5123
5124 /// Kind of the reduction data.
5125 enum ReductionKind {
5126 RK_None, /// Not a reduction.
5127 RK_Arithmetic, /// Binary reduction data.
5128 RK_Min, /// Minimum reduction data.
5129 RK_UMin, /// Unsigned minimum reduction data.
5130 RK_Max, /// Maximum reduction data.
5131 RK_UMax, /// Unsigned maximum reduction data.
5132 };
5133
5134 /// Contains info about operation, like its opcode, left and right operands.
5135 class OperationData {
5136 /// Opcode of the instruction.
5137 unsigned Opcode = 0;
5138
5139 /// Left operand of the reduction operation.
5140 Value *LHS = nullptr;
5141
5142 /// Right operand of the reduction operation.
5143 Value *RHS = nullptr;
5144
5145 /// Kind of the reduction operation.
5146 ReductionKind Kind = RK_None;
5147
5148 /// True if float point min/max reduction has no NaNs.
5149 bool NoNaN = false;
5150
5151 /// Checks if the reduction operation can be vectorized.
5152 bool isVectorizable() const {
5153 return LHS && RHS &&
5154 // We currently only support adds && min/max reductions.
5155 ((Kind == RK_Arithmetic &&
5156 (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
5157 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5158 (Kind == RK_Min || Kind == RK_Max)) ||
5159 (Opcode == Instruction::ICmp &&
5160 (Kind == RK_UMin || Kind == RK_UMax)));
5161 }
5162
5163 /// Creates reduction operation with the current opcode.
5164 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5165 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5166, __extension__ __PRETTY_FUNCTION__))
5166 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5166, __extension__ __PRETTY_FUNCTION__))
;
5167 Value *Cmp;
5168 switch (Kind) {
5169 case RK_Arithmetic:
5170 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5171 Name);
5172 case RK_Min:
5173 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5174 : Builder.CreateFCmpOLT(LHS, RHS);
5175 break;
5176 case RK_Max:
5177 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5178 : Builder.CreateFCmpOGT(LHS, RHS);
5179 break;
5180 case RK_UMin:
5181 assert(Opcode == Instruction::ICmp && "Expected integer types.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Expected integer types.") ? void (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5181, __extension__ __PRETTY_FUNCTION__))
;
5182 Cmp = Builder.CreateICmpULT(LHS, RHS);
5183 break;
5184 case RK_UMax:
5185 assert(Opcode == Instruction::ICmp && "Expected integer types.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Expected integer types.") ? void (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5185, __extension__ __PRETTY_FUNCTION__))
;
5186 Cmp = Builder.CreateICmpUGT(LHS, RHS);
5187 break;
5188 case RK_None:
5189 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5189)
;
5190 }
5191 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5192 }
5193
5194 public:
5195 explicit OperationData() = default;
5196
5197 /// Construction for reduced values. They are identified by opcode only and
5198 /// don't have associated LHS/RHS values.
5199 explicit OperationData(Value *V) {
5200 if (auto *I = dyn_cast<Instruction>(V))
5201 Opcode = I->getOpcode();
5202 }
5203
5204 /// Constructor for reduction operations with opcode and its left and
5205 /// right operands.
5206 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5207 bool NoNaN = false)
5208 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5209 assert(Kind != RK_None && "One of the reduction operations is expected.")(static_cast <bool> (Kind != RK_None && "One of the reduction operations is expected."
) ? void (0) : __assert_fail ("Kind != RK_None && \"One of the reduction operations is expected.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5209, __extension__ __PRETTY_FUNCTION__))
;
5210 }
5211
5212 explicit operator bool() const { return Opcode; }
5213
5214 /// Get the index of the first operand.
5215 unsigned getFirstOperandIndex() const {
5216 assert(!!*this && "The opcode is not set.")(static_cast <bool> (!!*this && "The opcode is not set."
) ? void (0) : __assert_fail ("!!*this && \"The opcode is not set.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5216, __extension__ __PRETTY_FUNCTION__))
;
5217 switch (Kind) {
5218 case RK_Min:
5219 case RK_UMin:
5220 case RK_Max:
5221 case RK_UMax:
5222 return 1;
5223 case RK_Arithmetic:
5224 case RK_None:
5225 break;
5226 }
5227 return 0;
5228 }
5229
5230 /// Total number of operands in the reduction operation.
5231 unsigned getNumberOfOperands() const {
5232 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5233, __extension__ __PRETTY_FUNCTION__))
5233 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5233, __extension__ __PRETTY_FUNCTION__))
;
5234 switch (Kind) {
5235 case RK_Arithmetic:
5236 return 2;
5237 case RK_Min:
5238 case RK_UMin:
5239 case RK_Max:
5240 case RK_UMax:
5241 return 3;
5242 case RK_None:
5243 break;
5244 }
5245 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5245)
;
5246 }
5247
5248 /// Checks if the operation has the same parent as \p P.
5249 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5250 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5251, __extension__ __PRETTY_FUNCTION__))
5251 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5251, __extension__ __PRETTY_FUNCTION__))
;
5252 if (!IsRedOp)
5253 return I->getParent() == P;
5254 switch (Kind) {
5255 case RK_Arithmetic:
5256 // Arithmetic reduction operation must be used once only.
5257 return I->getParent() == P;
5258 case RK_Min:
5259 case RK_UMin:
5260 case RK_Max:
5261 case RK_UMax: {
5262 // SelectInst must be used twice while the condition op must have single
5263 // use only.
5264 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5265 return I->getParent() == P && Cmp && Cmp->getParent() == P;
5266 }
5267 case RK_None:
5268 break;
5269 }
5270 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5270)
;
5271 }
5272 /// Expected number of uses for reduction operations/reduced values.
5273 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5274 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5275, __extension__ __PRETTY_FUNCTION__))
5275 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5275, __extension__ __PRETTY_FUNCTION__))
;
5276 switch (Kind) {
5277 case RK_Arithmetic:
5278 return I->hasOneUse();
5279 case RK_Min:
5280 case RK_UMin:
5281 case RK_Max:
5282 case RK_UMax:
5283 return I->hasNUses(2) &&
5284 (!IsReductionOp ||
5285 cast<SelectInst>(I)->getCondition()->hasOneUse());
5286 case RK_None:
5287 break;
5288 }
5289 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5289)
;
5290 }
5291
5292 /// Initializes the list of reduction operations.
5293 void initReductionOps(ReductionOpsListType &ReductionOps) {
5294 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5295, __extension__ __PRETTY_FUNCTION__))
5295 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5295, __extension__ __PRETTY_FUNCTION__))
;
5296 switch (Kind) {
5297 case RK_Arithmetic:
5298 ReductionOps.assign(1, ReductionOpsType());
5299 break;
5300 case RK_Min:
5301 case RK_UMin:
5302 case RK_Max:
5303 case RK_UMax:
5304 ReductionOps.assign(2, ReductionOpsType());
5305 break;
5306 case RK_None:
5307 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5307)
;
5308 }
5309 }
5310 /// Add all reduction operations for the reduction instruction \p I.
5311 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5312 assert(Kind != RK_None && !!*this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5313, __extension__ __PRETTY_FUNCTION__))
5313 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && !!*this
&& LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5313, __extension__ __PRETTY_FUNCTION__))
;
5314 switch (Kind) {
5315 case RK_Arithmetic:
5316 ReductionOps[0].emplace_back(I);
5317 break;
5318 case RK_Min:
5319 case RK_UMin:
5320 case RK_Max:
5321 case RK_UMax:
5322 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5323 ReductionOps[1].emplace_back(I);
5324 break;
5325 case RK_None:
5326 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5326)
;
5327 }
5328 }
5329
5330 /// Checks if instruction is associative and can be vectorized.
5331 bool isAssociative(Instruction *I) const {
5332 assert(Kind != RK_None && *this && LHS && RHS &&(static_cast <bool> (Kind != RK_None && *this &&
LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5333, __extension__ __PRETTY_FUNCTION__))
5333 "Expected reduction operation.")(static_cast <bool> (Kind != RK_None && *this &&
LHS && RHS && "Expected reduction operation."
) ? void (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5333, __extension__ __PRETTY_FUNCTION__))
;
5334 switch (Kind) {
5335 case RK_Arithmetic:
5336 return I->isAssociative();
5337 case RK_Min:
5338 case RK_Max:
5339 return Opcode == Instruction::ICmp ||
5340 cast<Instruction>(I->getOperand(0))->isFast();
5341 case RK_UMin:
5342 case RK_UMax:
5343 assert(Opcode == Instruction::ICmp &&(static_cast <bool> (Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.") ? void (0) : __assert_fail
("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5344, __extension__ __PRETTY_FUNCTION__))
5344 "Only integer compare operation is expected.")(static_cast <bool> (Opcode == Instruction::ICmp &&
"Only integer compare operation is expected.") ? void (0) : __assert_fail
("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5344, __extension__ __PRETTY_FUNCTION__))
;
5345 return true;
5346 case RK_None:
5347 break;
5348 }
5349 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5349)
;
5350 }
5351
5352 /// Checks if the reduction operation can be vectorized.
5353 bool isVectorizable(Instruction *I) const {
5354 return isVectorizable() && isAssociative(I);
5355 }
5356
5357 /// Checks if two operation data are both a reduction op or both a reduced
5358 /// value.
5359 bool operator==(const OperationData &OD) {
5360 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&(static_cast <bool> (((Kind != OD.Kind) || ((!LHS == !OD
.LHS) && (!RHS == !OD.RHS))) && "One of the comparing operations is incorrect."
) ? void (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5361, __extension__ __PRETTY_FUNCTION__))
5361 "One of the comparing operations is incorrect.")(static_cast <bool> (((Kind != OD.Kind) || ((!LHS == !OD
.LHS) && (!RHS == !OD.RHS))) && "One of the comparing operations is incorrect."
) ? void (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5361, __extension__ __PRETTY_FUNCTION__))
;
5362 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5363 }
5364 bool operator!=(const OperationData &OD) { return !(*this == OD); }
5365 void clear() {
5366 Opcode = 0;
5367 LHS = nullptr;
5368 RHS = nullptr;
5369 Kind = RK_None;
5370 NoNaN = false;
5371 }
5372
5373 /// Get the opcode of the reduction operation.
5374 unsigned getOpcode() const {
5375 assert(isVectorizable() && "Expected vectorizable operation.")(static_cast <bool> (isVectorizable() && "Expected vectorizable operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected vectorizable operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5375, __extension__ __PRETTY_FUNCTION__))
;
5376 return Opcode;
5377 }
5378
5379 /// Get kind of reduction data.
5380 ReductionKind getKind() const { return Kind; }
5381 Value *getLHS() const { return LHS; }
5382 Value *getRHS() const { return RHS; }
5383 Type *getConditionType() const {
5384 switch (Kind) {
5385 case RK_Arithmetic:
5386 return nullptr;
5387 case RK_Min:
5388 case RK_Max:
5389 case RK_UMin:
5390 case RK_UMax:
5391 return CmpInst::makeCmpResultType(LHS->getType());
5392 case RK_None:
5393 break;
5394 }
5395 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5395)
;
5396 }
5397
5398 /// Creates reduction operation with the current opcode with the IR flags
5399 /// from \p ReductionOps.
5400 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5401 const ReductionOpsListType &ReductionOps) const {
5402 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5403, __extension__ __PRETTY_FUNCTION__))
5403 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5403, __extension__ __PRETTY_FUNCTION__))
;
5404 auto *Op = createOp(Builder, Name);
5405 switch (Kind) {
5406 case RK_Arithmetic:
5407 propagateIRFlags(Op, ReductionOps[0]);
5408 return Op;
5409 case RK_Min:
5410 case RK_Max:
5411 case RK_UMin:
5412 case RK_UMax:
5413 if (auto *SI = dyn_cast<SelectInst>(Op))
5414 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5415 propagateIRFlags(Op, ReductionOps[1]);
5416 return Op;
5417 case RK_None:
5418 break;
5419 }
5420 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5420)
;
5421 }
5422 /// Creates reduction operation with the current opcode with the IR flags
5423 /// from \p I.
5424 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5425 Instruction *I) const {
5426 assert(isVectorizable() &&(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5427, __extension__ __PRETTY_FUNCTION__))
5427 "Expected add|fadd or min/max reduction operation.")(static_cast <bool> (isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? void (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5427, __extension__ __PRETTY_FUNCTION__))
;
5428 auto *Op = createOp(Builder, Name);
5429 switch (Kind) {
5430 case RK_Arithmetic:
5431 propagateIRFlags(Op, I);
5432 return Op;
5433 case RK_Min:
5434 case RK_Max:
5435 case RK_UMin:
5436 case RK_UMax:
5437 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5438 propagateIRFlags(SI->getCondition(),
5439 cast<SelectInst>(I)->getCondition());
5440 }
5441 propagateIRFlags(Op, I);
5442 return Op;
5443 case RK_None:
5444 break;
5445 }
5446 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5446)
;
5447 }
5448
5449 TargetTransformInfo::ReductionFlags getFlags() const {
5450 TargetTransformInfo::ReductionFlags Flags;
5451 Flags.NoNaN = NoNaN;
5452 switch (Kind) {
5453 case RK_Arithmetic:
5454 break;
5455 case RK_Min:
5456 Flags.IsSigned = Opcode == Instruction::ICmp;
5457 Flags.IsMaxOp = false;
5458 break;
5459 case RK_Max:
5460 Flags.IsSigned = Opcode == Instruction::ICmp;
5461 Flags.IsMaxOp = true;
5462 break;
5463 case RK_UMin:
5464 Flags.IsSigned = false;
5465 Flags.IsMaxOp = false;
5466 break;
5467 case RK_UMax:
5468 Flags.IsSigned = false;
5469 Flags.IsMaxOp = true;
5470 break;
5471 case RK_None:
5472 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5472)
;
5473 }
5474 return Flags;
5475 }
5476 };
5477
5478 Instruction *ReductionRoot = nullptr;
5479
5480 /// The operation data of the reduction operation.
5481 OperationData ReductionData;
5482
5483 /// The operation data of the values we perform a reduction on.
5484 OperationData ReducedValueData;
5485
5486 /// Should we model this reduction as a pairwise reduction tree or a tree that
5487 /// splits the vector in halves and adds those halves.
5488 bool IsPairwiseReduction = false;
5489
5490 /// Checks if the ParentStackElem.first should be marked as a reduction
5491 /// operation with an extra argument or as extra argument itself.
5492 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5493 Value *ExtraArg) {
5494 if (ExtraArgs.count(ParentStackElem.first)) {
5495 ExtraArgs[ParentStackElem.first] = nullptr;
5496 // We ran into something like:
5497 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5498 // The whole ParentStackElem.first should be considered as an extra value
5499 // in this case.
5500 // Do not perform analysis of remaining operands of ParentStackElem.first
5501 // instruction, this whole instruction is an extra argument.
5502 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5503 } else {
5504 // We ran into something like:
5505 // ParentStackElem.first += ... + ExtraArg + ...
5506 ExtraArgs[ParentStackElem.first] = ExtraArg;
5507 }
5508 }
5509
5510 static OperationData getOperationData(Value *V) {
5511 if (!V)
5512 return OperationData();
5513
5514 Value *LHS;
5515 Value *RHS;
5516 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5517 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5518 RK_Arithmetic);
5519 }
5520 if (auto *Select = dyn_cast<SelectInst>(V)) {
5521 // Look for a min/max pattern.
5522 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5523 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5524 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5525 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5526 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5527 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5528 return OperationData(
5529 Instruction::FCmp, LHS, RHS, RK_Min,
5530 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5531 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5532 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5533 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5534 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5535 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5536 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5537 return OperationData(
5538 Instruction::FCmp, LHS, RHS, RK_Max,
5539 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5540 }
5541 }
5542 return OperationData(V);
5543 }
5544
5545public:
5546 HorizontalReduction() = default;
5547
5548 /// \brief Try to find a reduction tree.
5549 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5550 assert((!Phi || is_contained(Phi->operands(), B)) &&(static_cast <bool> ((!Phi || is_contained(Phi->operands
(), B)) && "Thi phi needs to use the binary operator"
) ? void (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5551, __extension__ __PRETTY_FUNCTION__))
5551 "Thi phi needs to use the binary operator")(static_cast <bool> ((!Phi || is_contained(Phi->operands
(), B)) && "Thi phi needs to use the binary operator"
) ? void (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-7~svn329677/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5551, __extension__ __PRETTY_FUNCTION__))
;
5552
5553 ReductionData = getOperationData(B);
5554
5555 // We could have a initial reductions that is not an add.
5556 // r *= v1 + v2 + v3 + v4
5557 // In such a case start looking for a tree rooted in the first '+'.
5558 if (Phi) {
5559 if (ReductionData.getLHS() == Phi) {
5560 Phi = nullptr;
5561 B = dyn_cast<Instruction>(ReductionData.getRHS());
5562 ReductionData = getOperationData(B);
5563 } else if (ReductionData.getRHS() == Phi) {
5564 Phi = nullptr;
5565 B = dyn_cast<Instruction>(ReductionData.getLHS());
5566 ReductionData = getOperationData(B);
5567 }
5568 }
5569
5570 if (!ReductionData.isVectorizable(B))
5571 return false;
5572
5573 Type *Ty = B->getType();
5574 if (!isValidElementType(Ty))
5575 return false;
5576
5577 ReducedValueData.clear();
5578 ReductionRoot = B;
5579
5580 // Post order traverse the reduction tree starting at B. We only handle true
5581 // trees containing only binary operators.
5582 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5583 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5584 ReductionData.initReductionOps(ReductionOps);
5585 while (!Stack.empty()) {
5586 Instruction *TreeN = Stack.back().first;
5587 unsigned EdgeToVist = Stack.back().second++;
5588 OperationData OpData = getOperationData(TreeN);
5589 bool IsReducedValue = OpData != ReductionData;
5590
5591 // Postorder vist.
5592 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5593 if (IsReducedValue)
5594 ReducedVals.push_back(TreeN);
5595 else {
5596 auto I = ExtraArgs.find(TreeN);
5597 if (I != ExtraArgs.end() && !I->second) {
5598 // Check if TreeN is an extra argument of its parent operation.
5599 if (Stack.size() <= 1) {
5600 // TreeN can't be an extra argument as it is a root reduction
5601 // operation.
5602 return false;
5603 }
5604 // Yes, TreeN is an extra argument, do not add it to a list of
5605 // reduction operations.
5606 // Stack[Stack.size() - 2] always points to the parent operation.
5607 markExtraArg(Stack[Stack.size() - 2], TreeN);
5608 ExtraArgs.erase(TreeN);
5609 } else
5610 ReductionData.addReductionOps(TreeN, ReductionOps);
5611 }
5612 // Retract.
5613 Stack.pop_back();
5614 continue;
5615 }
5616
5617 // Visit left or right.
5618 Value *NextV = TreeN->getOperand(EdgeToVist);
5619 if (NextV != Phi) {
5620 auto *I = dyn_cast<Instruction>(NextV);
5621 OpData = getOperationData(I);
5622 // Continue analysis if the next operand is a reduction operation or
5623 // (possibly) a reduced value. If the reduced value opcode is not set,
5624 // the first met operation != reduction operation is considered as the
5625 // reduced value class.
5626 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5627 OpData == ReductionData)) {
5628