Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 3885, column 11
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SLPVectorizer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-config-compatibility-mode=true -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn348900/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-8~svn348900/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn348900/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn348900/build-llvm/lib/Transforms/Vectorize -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-12-042652-12204-1 -x c++ /build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp -faddrsig
1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11// stores that can be put together into vector-stores. Next, it attempts to
12// construct vectorizable tree using the use-def chains. If a profitable tree
13// was found, the SLP vectorizer performs vectorization on the tree.
14//
15// The pass is inspired by the work described in the paper:
16// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/DenseSet.h"
24#include "llvm/ADT/MapVector.h"
25#include "llvm/ADT/None.h"
26#include "llvm/ADT/Optional.h"
27#include "llvm/ADT/PostOrderIterator.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SetVector.h"
30#include "llvm/ADT/SmallPtrSet.h"
31#include "llvm/ADT/SmallSet.h"
32#include "llvm/ADT/SmallVector.h"
33#include "llvm/ADT/Statistic.h"
34#include "llvm/ADT/iterator.h"
35#include "llvm/ADT/iterator_range.h"
36#include "llvm/Analysis/AliasAnalysis.h"
37#include "llvm/Analysis/CodeMetrics.h"
38#include "llvm/Analysis/DemandedBits.h"
39#include "llvm/Analysis/GlobalsModRef.h"
40#include "llvm/Analysis/LoopAccessAnalysis.h"
41#include "llvm/Analysis/LoopInfo.h"
42#include "llvm/Analysis/MemoryLocation.h"
43#include "llvm/Analysis/OptimizationRemarkEmitter.h"
44#include "llvm/Analysis/ScalarEvolution.h"
45#include "llvm/Analysis/ScalarEvolutionExpressions.h"
46#include "llvm/Analysis/TargetLibraryInfo.h"
47#include "llvm/Analysis/TargetTransformInfo.h"
48#include "llvm/Analysis/ValueTracking.h"
49#include "llvm/Analysis/VectorUtils.h"
50#include "llvm/IR/Attributes.h"
51#include "llvm/IR/BasicBlock.h"
52#include "llvm/IR/Constant.h"
53#include "llvm/IR/Constants.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/IRBuilder.h"
60#include "llvm/IR/InstrTypes.h"
61#include "llvm/IR/Instruction.h"
62#include "llvm/IR/Instructions.h"
63#include "llvm/IR/IntrinsicInst.h"
64#include "llvm/IR/Intrinsics.h"
65#include "llvm/IR/Module.h"
66#include "llvm/IR/NoFolder.h"
67#include "llvm/IR/Operator.h"
68#include "llvm/IR/PassManager.h"
69#include "llvm/IR/PatternMatch.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
75#include "llvm/IR/Verifier.h"
76#include "llvm/Pass.h"
77#include "llvm/Support/Casting.h"
78#include "llvm/Support/CommandLine.h"
79#include "llvm/Support/Compiler.h"
80#include "llvm/Support/DOTGraphTraits.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/Support/ErrorHandling.h"
83#include "llvm/Support/GraphWriter.h"
84#include "llvm/Support/KnownBits.h"
85#include "llvm/Support/MathExtras.h"
86#include "llvm/Support/raw_ostream.h"
87#include "llvm/Transforms/Utils/LoopUtils.h"
88#include "llvm/Transforms/Vectorize.h"
89#include <algorithm>
90#include <cassert>
91#include <cstdint>
92#include <iterator>
93#include <memory>
94#include <set>
95#include <string>
96#include <tuple>
97#include <utility>
98#include <vector>
99
100using namespace llvm;
101using namespace llvm::PatternMatch;
102using namespace slpvectorizer;
103
104#define SV_NAME"slp-vectorizer" "slp-vectorizer"
105#define DEBUG_TYPE"SLP" "SLP"
106
107STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, {false}}
;
108
109static cl::opt<int>
110 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111 cl::desc("Only vectorize if you gain more than this "
112 "number "));
113
114static cl::opt<bool>
115ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116 cl::desc("Attempt to vectorize horizontal reductions"));
117
118static cl::opt<bool> ShouldStartVectorizeHorAtStore(
119 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120 cl::desc(
121 "Attempt to vectorize horizontal reductions feeding into a store"));
122
123static cl::opt<int>
124MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125 cl::desc("Attempt to vectorize for this register size in bits"));
126
127/// Limits the size of scheduling regions in a block.
128/// It avoid long compile times for _very_ large blocks where vector
129/// instructions are spread over a wide range.
130/// This limit is way higher than needed by real-world functions.
131static cl::opt<int>
132ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133 cl::desc("Limit the size of the SLP scheduling region per block"));
134
135static cl::opt<int> MinVectorRegSizeOption(
136 "slp-min-reg-size", cl::init(128), cl::Hidden,
137 cl::desc("Attempt to vectorize for this register size in bits"));
138
139static cl::opt<unsigned> RecursionMaxDepth(
140 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141 cl::desc("Limit the recursion depth when building a vectorizable tree"));
142
143static cl::opt<unsigned> MinTreeSize(
144 "slp-min-tree-size", cl::init(3), cl::Hidden,
145 cl::desc("Only vectorize small trees if they are fully vectorizable"));
146
147static cl::opt<bool>
148 ViewSLPTree("view-slp-tree", cl::Hidden,
149 cl::desc("Display the SLP trees with Graphviz"));
150
151// Limit the number of alias checks. The limit is chosen so that
152// it has no negative effect on the llvm benchmarks.
153static const unsigned AliasedCheckLimit = 10;
154
155// Another limit for the alias checks: The maximum distance between load/store
156// instructions where alias checks are done.
157// This limit is useful for very large basic blocks.
158static const unsigned MaxMemDepDistance = 160;
159
160/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161/// regions to be handled.
162static const int MinScheduleRegionSize = 16;
163
164/// Predicate for the element types that the SLP vectorizer supports.
165///
166/// The most important thing to filter here are types which are invalid in LLVM
167/// vectors. We also filter target specific types which have absolutely no
168/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169/// avoids spending time checking the cost model and realizing that they will
170/// be inevitably scalarized.
171static bool isValidElementType(Type *Ty) {
172 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173 !Ty->isPPC_FP128Ty();
174}
175
176/// \returns true if all of the instructions in \p VL are in the same block or
177/// false otherwise.
178static bool allSameBlock(ArrayRef<Value *> VL) {
179 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180 if (!I0)
181 return false;
182 BasicBlock *BB = I0->getParent();
183 for (int i = 1, e = VL.size(); i < e; i++) {
184 Instruction *I = dyn_cast<Instruction>(VL[i]);
185 if (!I)
186 return false;
187
188 if (BB != I->getParent())
189 return false;
190 }
191 return true;
192}
193
194/// \returns True if all of the values in \p VL are constants.
195static bool allConstant(ArrayRef<Value *> VL) {
196 for (Value *i : VL)
197 if (!isa<Constant>(i))
198 return false;
199 return true;
200}
201
202/// \returns True if all of the values in \p VL are identical.
203static bool isSplat(ArrayRef<Value *> VL) {
204 for (unsigned i = 1, e = VL.size(); i < e; ++i)
205 if (VL[i] != VL[0])
206 return false;
207 return true;
208}
209
210/// Checks if the vector of instructions can be represented as a shuffle, like:
211/// %x0 = extractelement <4 x i8> %x, i32 0
212/// %x3 = extractelement <4 x i8> %x, i32 3
213/// %y1 = extractelement <4 x i8> %y, i32 1
214/// %y2 = extractelement <4 x i8> %y, i32 2
215/// %x0x0 = mul i8 %x0, %x0
216/// %x3x3 = mul i8 %x3, %x3
217/// %y1y1 = mul i8 %y1, %y1
218/// %y2y2 = mul i8 %y2, %y2
219/// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220/// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221/// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222/// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223/// ret <4 x i8> %ins4
224/// can be transformed into:
225/// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226/// i32 6>
227/// %2 = mul <4 x i8> %1, %1
228/// ret <4 x i8> %2
229/// We convert this initially to something like:
230/// %x0 = extractelement <4 x i8> %x, i32 0
231/// %x3 = extractelement <4 x i8> %x, i32 3
232/// %y1 = extractelement <4 x i8> %y, i32 1
233/// %y2 = extractelement <4 x i8> %y, i32 2
234/// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235/// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236/// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237/// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238/// %5 = mul <4 x i8> %4, %4
239/// %6 = extractelement <4 x i8> %5, i32 0
240/// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241/// %7 = extractelement <4 x i8> %5, i32 1
242/// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243/// %8 = extractelement <4 x i8> %5, i32 2
244/// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245/// %9 = extractelement <4 x i8> %5, i32 3
246/// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247/// ret <4 x i8> %ins4
248/// InstCombiner transforms this into a shuffle and vector mul
249/// TODO: Can we split off and reuse the shuffle mask detection from
250/// TargetTransformInfo::getInstructionThroughput?
251static Optional<TargetTransformInfo::ShuffleKind>
252isShuffle(ArrayRef<Value *> VL) {
253 auto *EI0 = cast<ExtractElementInst>(VL[0]);
254 unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
255 Value *Vec1 = nullptr;
256 Value *Vec2 = nullptr;
257 enum ShuffleMode { Unknown, Select, Permute };
258 ShuffleMode CommonShuffleMode = Unknown;
259 for (unsigned I = 0, E = VL.size(); I < E; ++I) {
260 auto *EI = cast<ExtractElementInst>(VL[I]);
261 auto *Vec = EI->getVectorOperand();
262 // All vector operands must have the same number of vector elements.
263 if (Vec->getType()->getVectorNumElements() != Size)
264 return None;
265 auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
266 if (!Idx)
267 return None;
268 // Undefined behavior if Idx is negative or >= Size.
269 if (Idx->getValue().uge(Size))
270 continue;
271 unsigned IntIdx = Idx->getValue().getZExtValue();
272 // We can extractelement from undef vector.
273 if (isa<UndefValue>(Vec))
274 continue;
275 // For correct shuffling we have to have at most 2 different vector operands
276 // in all extractelement instructions.
277 if (!Vec1 || Vec1 == Vec)
278 Vec1 = Vec;
279 else if (!Vec2 || Vec2 == Vec)
280 Vec2 = Vec;
281 else
282 return None;
283 if (CommonShuffleMode == Permute)
284 continue;
285 // If the extract index is not the same as the operation number, it is a
286 // permutation.
287 if (IntIdx != I) {
288 CommonShuffleMode = Permute;
289 continue;
290 }
291 CommonShuffleMode = Select;
292 }
293 // If we're not crossing lanes in different vectors, consider it as blending.
294 if (CommonShuffleMode == Select && Vec2)
295 return TargetTransformInfo::SK_Select;
296 // If Vec2 was never used, we have a permutation of a single vector, otherwise
297 // we have permutation of 2 vectors.
298 return Vec2 ? TargetTransformInfo::SK_PermuteTwoSrc
299 : TargetTransformInfo::SK_PermuteSingleSrc;
300}
301
302namespace {
303
304/// Main data required for vectorization of instructions.
305struct InstructionsState {
306 /// The very first instruction in the list with the main opcode.
307 Value *OpValue = nullptr;
308
309 /// The main/alternate instruction.
310 Instruction *MainOp = nullptr;
311 Instruction *AltOp = nullptr;
312
313 /// The main/alternate opcodes for the list of instructions.
314 unsigned getOpcode() const {
315 return MainOp ? MainOp->getOpcode() : 0;
316 }
317
318 unsigned getAltOpcode() const {
319 return AltOp ? AltOp->getOpcode() : 0;
320 }
321
322 /// Some of the instructions in the list have alternate opcodes.
323 bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
324
325 bool isOpcodeOrAlt(Instruction *I) const {
326 unsigned CheckedOpcode = I->getOpcode();
327 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
328 }
329
330 InstructionsState() = delete;
331 InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
332 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
333};
334
335} // end anonymous namespace
336
337/// Chooses the correct key for scheduling data. If \p Op has the same (or
338/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
339/// OpValue.
340static Value *isOneOf(const InstructionsState &S, Value *Op) {
341 auto *I = dyn_cast<Instruction>(Op);
342 if (I && S.isOpcodeOrAlt(I))
343 return Op;
344 return S.OpValue;
345}
346
347/// \returns analysis of the Instructions in \p VL described in
348/// InstructionsState, the Opcode that we suppose the whole list
349/// could be vectorized even if its structure is diverse.
350static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
351 unsigned BaseIndex = 0) {
352 // Make sure these are all Instructions.
353 if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
354 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
355
356 bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
357 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
358 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
359 unsigned AltOpcode = Opcode;
360 unsigned AltIndex = BaseIndex;
361
362 // Check for one alternate opcode from another BinaryOperator.
363 // TODO - generalize to support all operators (types, calls etc.).
364 for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
365 unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
366 if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
367 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
368 continue;
369 if (Opcode == AltOpcode) {
370 AltOpcode = InstOpcode;
371 AltIndex = Cnt;
372 continue;
373 }
374 } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
375 Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
376 Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
377 if (Ty0 == Ty1) {
378 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
379 continue;
380 if (Opcode == AltOpcode) {
381 AltOpcode = InstOpcode;
382 AltIndex = Cnt;
383 continue;
384 }
385 }
386 } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
387 continue;
388 return InstructionsState(VL[BaseIndex], nullptr, nullptr);
389 }
390
391 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
392 cast<Instruction>(VL[AltIndex]));
393}
394
395/// \returns true if all of the values in \p VL have the same type or false
396/// otherwise.
397static bool allSameType(ArrayRef<Value *> VL) {
398 Type *Ty = VL[0]->getType();
399 for (int i = 1, e = VL.size(); i < e; i++)
400 if (VL[i]->getType() != Ty)
401 return false;
402
403 return true;
404}
405
406/// \returns True if Extract{Value,Element} instruction extracts element Idx.
407static Optional<unsigned> getExtractIndex(Instruction *E) {
408 unsigned Opcode = E->getOpcode();
409 assert((Opcode == Instruction::ExtractElement ||(((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? static_cast<void> (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 411, __PRETTY_FUNCTION__))
410 Opcode == Instruction::ExtractValue) &&(((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? static_cast<void> (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 411, __PRETTY_FUNCTION__))
411 "Expected extractelement or extractvalue instruction.")(((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) && "Expected extractelement or extractvalue instruction."
) ? static_cast<void> (0) : __assert_fail ("(Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue) && \"Expected extractelement or extractvalue instruction.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 411, __PRETTY_FUNCTION__))
;
412 if (Opcode == Instruction::ExtractElement) {
413 auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
414 if (!CI)
415 return None;
416 return CI->getZExtValue();
417 }
418 ExtractValueInst *EI = cast<ExtractValueInst>(E);
419 if (EI->getNumIndices() != 1)
420 return None;
421 return *EI->idx_begin();
422}
423
424/// \returns True if in-tree use also needs extract. This refers to
425/// possible scalar operand in vectorized instruction.
426static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
427 TargetLibraryInfo *TLI) {
428 unsigned Opcode = UserInst->getOpcode();
429 switch (Opcode) {
430 case Instruction::Load: {
431 LoadInst *LI = cast<LoadInst>(UserInst);
432 return (LI->getPointerOperand() == Scalar);
433 }
434 case Instruction::Store: {
435 StoreInst *SI = cast<StoreInst>(UserInst);
436 return (SI->getPointerOperand() == Scalar);
437 }
438 case Instruction::Call: {
439 CallInst *CI = cast<CallInst>(UserInst);
440 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
441 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
442 return (CI->getArgOperand(1) == Scalar);
443 }
444 LLVM_FALLTHROUGH[[clang::fallthrough]];
445 }
446 default:
447 return false;
448 }
449}
450
451/// \returns the AA location that is being access by the instruction.
452static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
453 if (StoreInst *SI = dyn_cast<StoreInst>(I))
454 return MemoryLocation::get(SI);
455 if (LoadInst *LI = dyn_cast<LoadInst>(I))
456 return MemoryLocation::get(LI);
457 return MemoryLocation();
458}
459
460/// \returns True if the instruction is not a volatile or atomic load/store.
461static bool isSimple(Instruction *I) {
462 if (LoadInst *LI = dyn_cast<LoadInst>(I))
463 return LI->isSimple();
464 if (StoreInst *SI = dyn_cast<StoreInst>(I))
465 return SI->isSimple();
466 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
467 return !MI->isVolatile();
468 return true;
469}
470
471namespace llvm {
472
473namespace slpvectorizer {
474
475/// Bottom Up SLP Vectorizer.
476class BoUpSLP {
477public:
478 using ValueList = SmallVector<Value *, 8>;
479 using InstrList = SmallVector<Instruction *, 16>;
480 using ValueSet = SmallPtrSet<Value *, 16>;
481 using StoreList = SmallVector<StoreInst *, 8>;
482 using ExtraValueToDebugLocsMap =
483 MapVector<Value *, SmallVector<Instruction *, 2>>;
484
485 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
486 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
487 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
488 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
489 : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
490 DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
491 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
492 // Use the vector register size specified by the target unless overridden
493 // by a command-line option.
494 // TODO: It would be better to limit the vectorization factor based on
495 // data type rather than just register size. For example, x86 AVX has
496 // 256-bit registers, but it does not support integer operations
497 // at that width (that requires AVX2).
498 if (MaxVectorRegSizeOption.getNumOccurrences())
499 MaxVecRegSize = MaxVectorRegSizeOption;
500 else
501 MaxVecRegSize = TTI->getRegisterBitWidth(true);
502
503 if (MinVectorRegSizeOption.getNumOccurrences())
504 MinVecRegSize = MinVectorRegSizeOption;
505 else
506 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
507 }
508
509 /// Vectorize the tree that starts with the elements in \p VL.
510 /// Returns the vectorized root.
511 Value *vectorizeTree();
512
513 /// Vectorize the tree but with the list of externally used values \p
514 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
515 /// generated extractvalue instructions.
516 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
517
518 /// \returns the cost incurred by unwanted spills and fills, caused by
519 /// holding live values over call sites.
520 int getSpillCost();
521
522 /// \returns the vectorization cost of the subtree that starts at \p VL.
523 /// A negative number means that this is profitable.
524 int getTreeCost();
525
526 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
527 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
528 void buildTree(ArrayRef<Value *> Roots,
529 ArrayRef<Value *> UserIgnoreLst = None);
530
531 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
532 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
533 /// into account (anf updating it, if required) list of externally used
534 /// values stored in \p ExternallyUsedValues.
535 void buildTree(ArrayRef<Value *> Roots,
536 ExtraValueToDebugLocsMap &ExternallyUsedValues,
537 ArrayRef<Value *> UserIgnoreLst = None);
538
539 /// Clear the internal data structures that are created by 'buildTree'.
540 void deleteTree() {
541 VectorizableTree.clear();
542 ScalarToTreeEntry.clear();
543 MustGather.clear();
544 ExternalUses.clear();
545 NumOpsWantToKeepOrder.clear();
546 NumOpsWantToKeepOriginalOrder = 0;
547 for (auto &Iter : BlocksSchedules) {
548 BlockScheduling *BS = Iter.second.get();
549 BS->clear();
550 }
551 MinBWs.clear();
552 }
553
554 unsigned getTreeSize() const { return VectorizableTree.size(); }
555
556 /// Perform LICM and CSE on the newly generated gather sequences.
557 void optimizeGatherSequence();
558
559 /// \returns The best order of instructions for vectorization.
560 Optional<ArrayRef<unsigned>> bestOrder() const {
561 auto I = std::max_element(
562 NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
563 [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
564 const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
565 return D1.second < D2.second;
566 });
567 if (I == NumOpsWantToKeepOrder.end() ||
568 I->getSecond() <= NumOpsWantToKeepOriginalOrder)
569 return None;
570
571 return makeArrayRef(I->getFirst());
572 }
573
574 /// \return The vector element size in bits to use when vectorizing the
575 /// expression tree ending at \p V. If V is a store, the size is the width of
576 /// the stored value. Otherwise, the size is the width of the largest loaded
577 /// value reaching V. This method is used by the vectorizer to calculate
578 /// vectorization factors.
579 unsigned getVectorElementSize(Value *V);
580
581 /// Compute the minimum type sizes required to represent the entries in a
582 /// vectorizable tree.
583 void computeMinimumValueSizes();
584
585 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
586 unsigned getMaxVecRegSize() const {
587 return MaxVecRegSize;
588 }
589
590 // \returns minimum vector register size as set by cl::opt.
591 unsigned getMinVecRegSize() const {
592 return MinVecRegSize;
593 }
594
595 /// Check if ArrayType or StructType is isomorphic to some VectorType.
596 ///
597 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
598 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
599
600 /// \returns True if the VectorizableTree is both tiny and not fully
601 /// vectorizable. We do not vectorize such trees.
602 bool isTreeTinyAndNotFullyVectorizable();
603
604 OptimizationRemarkEmitter *getORE() { return ORE; }
605
606private:
607 struct TreeEntry;
608
609 /// Checks if all users of \p I are the part of the vectorization tree.
610 bool areAllUsersVectorized(Instruction *I) const;
611
612 /// \returns the cost of the vectorizable entry.
613 int getEntryCost(TreeEntry *E);
614
615 /// This is the recursive part of buildTree.
616 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
617
618 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
619 /// be vectorized to use the original vector (or aggregate "bitcast" to a
620 /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
621 /// returns false, setting \p CurrentOrder to either an empty vector or a
622 /// non-identity permutation that allows to reuse extract instructions.
623 bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
624 SmallVectorImpl<unsigned> &CurrentOrder) const;
625
626 /// Vectorize a single entry in the tree.
627 Value *vectorizeTree(TreeEntry *E);
628
629 /// Vectorize a single entry in the tree, starting in \p VL.
630 Value *vectorizeTree(ArrayRef<Value *> VL);
631
632 /// \returns the scalarization cost for this type. Scalarization in this
633 /// context means the creation of vectors from a group of scalars.
634 int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
635
636 /// \returns the scalarization cost for this list of values. Assuming that
637 /// this subtree gets vectorized, we may need to extract the values from the
638 /// roots. This method calculates the cost of extracting the values.
639 int getGatherCost(ArrayRef<Value *> VL);
640
641 /// Set the Builder insert point to one after the last instruction in
642 /// the bundle
643 void setInsertPointAfterBundle(ArrayRef<Value *> VL,
644 const InstructionsState &S);
645
646 /// \returns a vector from a collection of scalars in \p VL.
647 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
648
649 /// \returns whether the VectorizableTree is fully vectorizable and will
650 /// be beneficial even the tree height is tiny.
651 bool isFullyVectorizableTinyTree();
652
653 /// \reorder commutative operands in alt shuffle if they result in
654 /// vectorized code.
655 void reorderAltShuffleOperands(const InstructionsState &S,
656 ArrayRef<Value *> VL,
657 SmallVectorImpl<Value *> &Left,
658 SmallVectorImpl<Value *> &Right);
659
660 /// \reorder commutative operands to get better probability of
661 /// generating vectorized code.
662 void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
663 SmallVectorImpl<Value *> &Left,
664 SmallVectorImpl<Value *> &Right);
665 struct TreeEntry {
666 TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
667
668 /// \returns true if the scalars in VL are equal to this entry.
669 bool isSame(ArrayRef<Value *> VL) const {
670 if (VL.size() == Scalars.size())
671 return std::equal(VL.begin(), VL.end(), Scalars.begin());
672 return VL.size() == ReuseShuffleIndices.size() &&
673 std::equal(
674 VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
675 [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
676 }
677
678 /// A vector of scalars.
679 ValueList Scalars;
680
681 /// The Scalars are vectorized into this value. It is initialized to Null.
682 Value *VectorizedValue = nullptr;
683
684 /// Do we need to gather this sequence ?
685 bool NeedToGather = false;
686
687 /// Does this sequence require some shuffling?
688 SmallVector<unsigned, 4> ReuseShuffleIndices;
689
690 /// Does this entry require reordering?
691 ArrayRef<unsigned> ReorderIndices;
692
693 /// Points back to the VectorizableTree.
694 ///
695 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
696 /// to be a pointer and needs to be able to initialize the child iterator.
697 /// Thus we need a reference back to the container to translate the indices
698 /// to entries.
699 std::vector<TreeEntry> &Container;
700
701 /// The TreeEntry index containing the user of this entry. We can actually
702 /// have multiple users so the data structure is not truly a tree.
703 SmallVector<int, 1> UserTreeIndices;
704 };
705
706 /// Create a new VectorizableTree entry.
707 void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
708 ArrayRef<unsigned> ReuseShuffleIndices = None,
709 ArrayRef<unsigned> ReorderIndices = None) {
710 VectorizableTree.emplace_back(VectorizableTree);
711 int idx = VectorizableTree.size() - 1;
712 TreeEntry *Last = &VectorizableTree[idx];
713 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
714 Last->NeedToGather = !Vectorized;
715 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
716 ReuseShuffleIndices.end());
717 Last->ReorderIndices = ReorderIndices;
718 if (Vectorized) {
719 for (int i = 0, e = VL.size(); i != e; ++i) {
720 assert(!getTreeEntry(VL[i]) && "Scalar already in tree!")((!getTreeEntry(VL[i]) && "Scalar already in tree!") ?
static_cast<void> (0) : __assert_fail ("!getTreeEntry(VL[i]) && \"Scalar already in tree!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 720, __PRETTY_FUNCTION__))
;
721 ScalarToTreeEntry[VL[i]] = idx;
722 }
723 } else {
724 MustGather.insert(VL.begin(), VL.end());
725 }
726
727 if (UserTreeIdx >= 0)
728 Last->UserTreeIndices.push_back(UserTreeIdx);
729 UserTreeIdx = idx;
730 }
731
732 /// -- Vectorization State --
733 /// Holds all of the tree entries.
734 std::vector<TreeEntry> VectorizableTree;
735
736 TreeEntry *getTreeEntry(Value *V) {
737 auto I = ScalarToTreeEntry.find(V);
738 if (I != ScalarToTreeEntry.end())
739 return &VectorizableTree[I->second];
740 return nullptr;
741 }
742
743 /// Maps a specific scalar to its tree entry.
744 SmallDenseMap<Value*, int> ScalarToTreeEntry;
745
746 /// A list of scalars that we found that we need to keep as scalars.
747 ValueSet MustGather;
748
749 /// This POD struct describes one external user in the vectorized tree.
750 struct ExternalUser {
751 ExternalUser(Value *S, llvm::User *U, int L)
752 : Scalar(S), User(U), Lane(L) {}
753
754 // Which scalar in our function.
755 Value *Scalar;
756
757 // Which user that uses the scalar.
758 llvm::User *User;
759
760 // Which lane does the scalar belong to.
761 int Lane;
762 };
763 using UserList = SmallVector<ExternalUser, 16>;
764
765 /// Checks if two instructions may access the same memory.
766 ///
767 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
768 /// is invariant in the calling loop.
769 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
770 Instruction *Inst2) {
771 // First check if the result is already in the cache.
772 AliasCacheKey key = std::make_pair(Inst1, Inst2);
773 Optional<bool> &result = AliasCache[key];
774 if (result.hasValue()) {
775 return result.getValue();
776 }
777 MemoryLocation Loc2 = getLocation(Inst2, AA);
778 bool aliased = true;
779 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
780 // Do the alias check.
781 aliased = AA->alias(Loc1, Loc2);
782 }
783 // Store the result in the cache.
784 result = aliased;
785 return aliased;
786 }
787
788 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
789
790 /// Cache for alias results.
791 /// TODO: consider moving this to the AliasAnalysis itself.
792 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
793
794 /// Removes an instruction from its block and eventually deletes it.
795 /// It's like Instruction::eraseFromParent() except that the actual deletion
796 /// is delayed until BoUpSLP is destructed.
797 /// This is required to ensure that there are no incorrect collisions in the
798 /// AliasCache, which can happen if a new instruction is allocated at the
799 /// same address as a previously deleted instruction.
800 void eraseInstruction(Instruction *I) {
801 I->removeFromParent();
802 I->dropAllReferences();
803 DeletedInstructions.emplace_back(I);
804 }
805
806 /// Temporary store for deleted instructions. Instructions will be deleted
807 /// eventually when the BoUpSLP is destructed.
808 SmallVector<unique_value, 8> DeletedInstructions;
809
810 /// A list of values that need to extracted out of the tree.
811 /// This list holds pairs of (Internal Scalar : External User). External User
812 /// can be nullptr, it means that this Internal Scalar will be used later,
813 /// after vectorization.
814 UserList ExternalUses;
815
816 /// Values used only by @llvm.assume calls.
817 SmallPtrSet<const Value *, 32> EphValues;
818
819 /// Holds all of the instructions that we gathered.
820 SetVector<Instruction *> GatherSeq;
821
822 /// A list of blocks that we are going to CSE.
823 SetVector<BasicBlock *> CSEBlocks;
824
825 /// Contains all scheduling relevant data for an instruction.
826 /// A ScheduleData either represents a single instruction or a member of an
827 /// instruction bundle (= a group of instructions which is combined into a
828 /// vector instruction).
829 struct ScheduleData {
830 // The initial value for the dependency counters. It means that the
831 // dependencies are not calculated yet.
832 enum { InvalidDeps = -1 };
833
834 ScheduleData() = default;
835
836 void init(int BlockSchedulingRegionID, Value *OpVal) {
837 FirstInBundle = this;
838 NextInBundle = nullptr;
839 NextLoadStore = nullptr;
840 IsScheduled = false;
841 SchedulingRegionID = BlockSchedulingRegionID;
842 UnscheduledDepsInBundle = UnscheduledDeps;
843 clearDependencies();
844 OpValue = OpVal;
845 }
846
847 /// Returns true if the dependency information has been calculated.
848 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
849
850 /// Returns true for single instructions and for bundle representatives
851 /// (= the head of a bundle).
852 bool isSchedulingEntity() const { return FirstInBundle == this; }
853
854 /// Returns true if it represents an instruction bundle and not only a
855 /// single instruction.
856 bool isPartOfBundle() const {
857 return NextInBundle != nullptr || FirstInBundle != this;
858 }
859
860 /// Returns true if it is ready for scheduling, i.e. it has no more
861 /// unscheduled depending instructions/bundles.
862 bool isReady() const {
863 assert(isSchedulingEntity() &&((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 864, __PRETTY_FUNCTION__))
864 "can't consider non-scheduling entity for ready list")((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 864, __PRETTY_FUNCTION__))
;
865 return UnscheduledDepsInBundle == 0 && !IsScheduled;
866 }
867
868 /// Modifies the number of unscheduled dependencies, also updating it for
869 /// the whole bundle.
870 int incrementUnscheduledDeps(int Incr) {
871 UnscheduledDeps += Incr;
872 return FirstInBundle->UnscheduledDepsInBundle += Incr;
873 }
874
875 /// Sets the number of unscheduled dependencies to the number of
876 /// dependencies.
877 void resetUnscheduledDeps() {
878 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
879 }
880
881 /// Clears all dependency information.
882 void clearDependencies() {
883 Dependencies = InvalidDeps;
884 resetUnscheduledDeps();
885 MemoryDependencies.clear();
886 }
887
888 void dump(raw_ostream &os) const {
889 if (!isSchedulingEntity()) {
890 os << "/ " << *Inst;
891 } else if (NextInBundle) {
892 os << '[' << *Inst;
893 ScheduleData *SD = NextInBundle;
894 while (SD) {
895 os << ';' << *SD->Inst;
896 SD = SD->NextInBundle;
897 }
898 os << ']';
899 } else {
900 os << *Inst;
901 }
902 }
903
904 Instruction *Inst = nullptr;
905
906 /// Points to the head in an instruction bundle (and always to this for
907 /// single instructions).
908 ScheduleData *FirstInBundle = nullptr;
909
910 /// Single linked list of all instructions in a bundle. Null if it is a
911 /// single instruction.
912 ScheduleData *NextInBundle = nullptr;
913
914 /// Single linked list of all memory instructions (e.g. load, store, call)
915 /// in the block - until the end of the scheduling region.
916 ScheduleData *NextLoadStore = nullptr;
917
918 /// The dependent memory instructions.
919 /// This list is derived on demand in calculateDependencies().
920 SmallVector<ScheduleData *, 4> MemoryDependencies;
921
922 /// This ScheduleData is in the current scheduling region if this matches
923 /// the current SchedulingRegionID of BlockScheduling.
924 int SchedulingRegionID = 0;
925
926 /// Used for getting a "good" final ordering of instructions.
927 int SchedulingPriority = 0;
928
929 /// The number of dependencies. Constitutes of the number of users of the
930 /// instruction plus the number of dependent memory instructions (if any).
931 /// This value is calculated on demand.
932 /// If InvalidDeps, the number of dependencies is not calculated yet.
933 int Dependencies = InvalidDeps;
934
935 /// The number of dependencies minus the number of dependencies of scheduled
936 /// instructions. As soon as this is zero, the instruction/bundle gets ready
937 /// for scheduling.
938 /// Note that this is negative as long as Dependencies is not calculated.
939 int UnscheduledDeps = InvalidDeps;
940
941 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
942 /// single instructions.
943 int UnscheduledDepsInBundle = InvalidDeps;
944
945 /// True if this instruction is scheduled (or considered as scheduled in the
946 /// dry-run).
947 bool IsScheduled = false;
948
949 /// Opcode of the current instruction in the schedule data.
950 Value *OpValue = nullptr;
951 };
952
953#ifndef NDEBUG
954 friend inline raw_ostream &operator<<(raw_ostream &os,
955 const BoUpSLP::ScheduleData &SD) {
956 SD.dump(os);
957 return os;
958 }
959#endif
960
961 friend struct GraphTraits<BoUpSLP *>;
962 friend struct DOTGraphTraits<BoUpSLP *>;
963
964 /// Contains all scheduling data for a basic block.
965 struct BlockScheduling {
966 BlockScheduling(BasicBlock *BB)
967 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
968
969 void clear() {
970 ReadyInsts.clear();
971 ScheduleStart = nullptr;
972 ScheduleEnd = nullptr;
973 FirstLoadStoreInRegion = nullptr;
974 LastLoadStoreInRegion = nullptr;
975
976 // Reduce the maximum schedule region size by the size of the
977 // previous scheduling run.
978 ScheduleRegionSizeLimit -= ScheduleRegionSize;
979 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
980 ScheduleRegionSizeLimit = MinScheduleRegionSize;
981 ScheduleRegionSize = 0;
982
983 // Make a new scheduling region, i.e. all existing ScheduleData is not
984 // in the new region yet.
985 ++SchedulingRegionID;
986 }
987
988 ScheduleData *getScheduleData(Value *V) {
989 ScheduleData *SD = ScheduleDataMap[V];
990 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
991 return SD;
992 return nullptr;
993 }
994
995 ScheduleData *getScheduleData(Value *V, Value *Key) {
996 if (V == Key)
997 return getScheduleData(V);
998 auto I = ExtraScheduleDataMap.find(V);
999 if (I != ExtraScheduleDataMap.end()) {
1000 ScheduleData *SD = I->second[Key];
1001 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1002 return SD;
1003 }
1004 return nullptr;
1005 }
1006
1007 bool isInSchedulingRegion(ScheduleData *SD) {
1008 return SD->SchedulingRegionID == SchedulingRegionID;
1009 }
1010
1011 /// Marks an instruction as scheduled and puts all dependent ready
1012 /// instructions into the ready-list.
1013 template <typename ReadyListType>
1014 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1015 SD->IsScheduled = true;
1016 LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
1017
1018 ScheduleData *BundleMember = SD;
1019 while (BundleMember) {
1020 if (BundleMember->Inst != BundleMember->OpValue) {
1021 BundleMember = BundleMember->NextInBundle;
1022 continue;
1023 }
1024 // Handle the def-use chain dependencies.
1025 for (Use &U : BundleMember->Inst->operands()) {
1026 auto *I = dyn_cast<Instruction>(U.get());
1027 if (!I)
1028 continue;
1029 doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1030 if (OpDef && OpDef->hasValidDependencies() &&
1031 OpDef->incrementUnscheduledDeps(-1) == 0) {
1032 // There are no more unscheduled dependencies after
1033 // decrementing, so we can put the dependent instruction
1034 // into the ready list.
1035 ScheduleData *DepBundle = OpDef->FirstInBundle;
1036 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1037, __PRETTY_FUNCTION__))
1037 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1037, __PRETTY_FUNCTION__))
;
1038 ReadyList.insert(DepBundle);
1039 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
1040 << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
1041 }
1042 });
1043 }
1044 // Handle the memory dependencies.
1045 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1046 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1047 // There are no more unscheduled dependencies after decrementing,
1048 // so we can put the dependent instruction into the ready list.
1049 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1050 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1051, __PRETTY_FUNCTION__))
1051 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1051, __PRETTY_FUNCTION__))
;
1052 ReadyList.insert(DepBundle);
1053 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
1054 << "SLP: gets ready (mem): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
1055 }
1056 }
1057 BundleMember = BundleMember->NextInBundle;
1058 }
1059 }
1060
1061 void doForAllOpcodes(Value *V,
1062 function_ref<void(ScheduleData *SD)> Action) {
1063 if (ScheduleData *SD = getScheduleData(V))
1064 Action(SD);
1065 auto I = ExtraScheduleDataMap.find(V);
1066 if (I != ExtraScheduleDataMap.end())
1067 for (auto &P : I->second)
1068 if (P.second->SchedulingRegionID == SchedulingRegionID)
1069 Action(P.second);
1070 }
1071
1072 /// Put all instructions into the ReadyList which are ready for scheduling.
1073 template <typename ReadyListType>
1074 void initialFillReadyList(ReadyListType &ReadyList) {
1075 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1076 doForAllOpcodes(I, [&](ScheduleData *SD) {
1077 if (SD->isSchedulingEntity() && SD->isReady()) {
1078 ReadyList.insert(SD);
1079 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
1080 << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
1081 }
1082 });
1083 }
1084 }
1085
1086 /// Checks if a bundle of instructions can be scheduled, i.e. has no
1087 /// cyclic dependencies. This is only a dry-run, no instructions are
1088 /// actually moved at this stage.
1089 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
1090 const InstructionsState &S);
1091
1092 /// Un-bundles a group of instructions.
1093 void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1094
1095 /// Allocates schedule data chunk.
1096 ScheduleData *allocateScheduleDataChunks();
1097
1098 /// Extends the scheduling region so that V is inside the region.
1099 /// \returns true if the region size is within the limit.
1100 bool extendSchedulingRegion(Value *V, const InstructionsState &S);
1101
1102 /// Initialize the ScheduleData structures for new instructions in the
1103 /// scheduling region.
1104 void initScheduleData(Instruction *FromI, Instruction *ToI,
1105 ScheduleData *PrevLoadStore,
1106 ScheduleData *NextLoadStore);
1107
1108 /// Updates the dependency information of a bundle and of all instructions/
1109 /// bundles which depend on the original bundle.
1110 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1111 BoUpSLP *SLP);
1112
1113 /// Sets all instruction in the scheduling region to un-scheduled.
1114 void resetSchedule();
1115
1116 BasicBlock *BB;
1117
1118 /// Simple memory allocation for ScheduleData.
1119 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1120
1121 /// The size of a ScheduleData array in ScheduleDataChunks.
1122 int ChunkSize;
1123
1124 /// The allocator position in the current chunk, which is the last entry
1125 /// of ScheduleDataChunks.
1126 int ChunkPos;
1127
1128 /// Attaches ScheduleData to Instruction.
1129 /// Note that the mapping survives during all vectorization iterations, i.e.
1130 /// ScheduleData structures are recycled.
1131 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1132
1133 /// Attaches ScheduleData to Instruction with the leading key.
1134 DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
1135 ExtraScheduleDataMap;
1136
1137 struct ReadyList : SmallVector<ScheduleData *, 8> {
1138 void insert(ScheduleData *SD) { push_back(SD); }
1139 };
1140
1141 /// The ready-list for scheduling (only used for the dry-run).
1142 ReadyList ReadyInsts;
1143
1144 /// The first instruction of the scheduling region.
1145 Instruction *ScheduleStart = nullptr;
1146
1147 /// The first instruction _after_ the scheduling region.
1148 Instruction *ScheduleEnd = nullptr;
1149
1150 /// The first memory accessing instruction in the scheduling region
1151 /// (can be null).
1152 ScheduleData *FirstLoadStoreInRegion = nullptr;
1153
1154 /// The last memory accessing instruction in the scheduling region
1155 /// (can be null).
1156 ScheduleData *LastLoadStoreInRegion = nullptr;
1157
1158 /// The current size of the scheduling region.
1159 int ScheduleRegionSize = 0;
1160
1161 /// The maximum size allowed for the scheduling region.
1162 int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1163
1164 /// The ID of the scheduling region. For a new vectorization iteration this
1165 /// is incremented which "removes" all ScheduleData from the region.
1166 // Make sure that the initial SchedulingRegionID is greater than the
1167 // initial SchedulingRegionID in ScheduleData (which is 0).
1168 int SchedulingRegionID = 1;
1169 };
1170
1171 /// Attaches the BlockScheduling structures to basic blocks.
1172 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
1173
1174 /// Performs the "real" scheduling. Done before vectorization is actually
1175 /// performed in a basic block.
1176 void scheduleBlock(BlockScheduling *BS);
1177
1178 /// List of users to ignore during scheduling and that don't need extracting.
1179 ArrayRef<Value *> UserIgnoreList;
1180
1181 using OrdersType = SmallVector<unsigned, 4>;
1182 /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
1183 /// sorted SmallVectors of unsigned.
1184 struct OrdersTypeDenseMapInfo {
1185 static OrdersType getEmptyKey() {
1186 OrdersType V;
1187 V.push_back(~1U);
1188 return V;
1189 }
1190
1191 static OrdersType getTombstoneKey() {
1192 OrdersType V;
1193 V.push_back(~2U);
1194 return V;
1195 }
1196
1197 static unsigned getHashValue(const OrdersType &V) {
1198 return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1199 }
1200
1201 static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
1202 return LHS == RHS;
1203 }
1204 };
1205
1206 /// Contains orders of operations along with the number of bundles that have
1207 /// operations in this order. It stores only those orders that require
1208 /// reordering, if reordering is not required it is counted using \a
1209 /// NumOpsWantToKeepOriginalOrder.
1210 DenseMap<OrdersType, unsigned, OrdersTypeDenseMapInfo> NumOpsWantToKeepOrder;
1211 /// Number of bundles that do not require reordering.
1212 unsigned NumOpsWantToKeepOriginalOrder = 0;
1213
1214 // Analysis and block reference.
1215 Function *F;
1216 ScalarEvolution *SE;
1217 TargetTransformInfo *TTI;
1218 TargetLibraryInfo *TLI;
1219 AliasAnalysis *AA;
1220 LoopInfo *LI;
1221 DominatorTree *DT;
1222 AssumptionCache *AC;
1223 DemandedBits *DB;
1224 const DataLayout *DL;
1225 OptimizationRemarkEmitter *ORE;
1226
1227 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1228 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1229
1230 /// Instruction builder to construct the vectorized tree.
1231 IRBuilder<> Builder;
1232
1233 /// A map of scalar integer values to the smallest bit width with which they
1234 /// can legally be represented. The values map to (width, signed) pairs,
1235 /// where "width" indicates the minimum bit width and "signed" is True if the
1236 /// value must be signed-extended, rather than zero-extended, back to its
1237 /// original width.
1238 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
1239};
1240
1241} // end namespace slpvectorizer
1242
1243template <> struct GraphTraits<BoUpSLP *> {
1244 using TreeEntry = BoUpSLP::TreeEntry;
1245
1246 /// NodeRef has to be a pointer per the GraphWriter.
1247 using NodeRef = TreeEntry *;
1248
1249 /// Add the VectorizableTree to the index iterator to be able to return
1250 /// TreeEntry pointers.
1251 struct ChildIteratorType
1252 : public iterator_adaptor_base<ChildIteratorType,
1253 SmallVector<int, 1>::iterator> {
1254 std::vector<TreeEntry> &VectorizableTree;
1255
1256 ChildIteratorType(SmallVector<int, 1>::iterator W,
1257 std::vector<TreeEntry> &VT)
1258 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1259
1260 NodeRef operator*() { return &VectorizableTree[*I]; }
1261 };
1262
1263 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1264
1265 static ChildIteratorType child_begin(NodeRef N) {
1266 return {N->UserTreeIndices.begin(), N->Container};
1267 }
1268
1269 static ChildIteratorType child_end(NodeRef N) {
1270 return {N->UserTreeIndices.end(), N->Container};
1271 }
1272
1273 /// For the node iterator we just need to turn the TreeEntry iterator into a
1274 /// TreeEntry* iterator so that it dereferences to NodeRef.
1275 using nodes_iterator = pointer_iterator<std::vector<TreeEntry>::iterator>;
1276
1277 static nodes_iterator nodes_begin(BoUpSLP *R) {
1278 return nodes_iterator(R->VectorizableTree.begin());
1279 }
1280
1281 static nodes_iterator nodes_end(BoUpSLP *R) {
1282 return nodes_iterator(R->VectorizableTree.end());
1283 }
1284
1285 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1286};
1287
1288template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1289 using TreeEntry = BoUpSLP::TreeEntry;
1290
1291 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1292
1293 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1294 std::string Str;
1295 raw_string_ostream OS(Str);
1296 if (isSplat(Entry->Scalars)) {
1297 OS << "<splat> " << *Entry->Scalars[0];
1298 return Str;
1299 }
1300 for (auto V : Entry->Scalars) {
1301 OS << *V;
1302 if (std::any_of(
1303 R->ExternalUses.begin(), R->ExternalUses.end(),
1304 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1305 OS << " <extract>";
1306 OS << "\n";
1307 }
1308 return Str;
1309 }
1310
1311 static std::string getNodeAttributes(const TreeEntry *Entry,
1312 const BoUpSLP *) {
1313 if (Entry->NeedToGather)
1314 return "color=red";
1315 return "";
1316 }
1317};
1318
1319} // end namespace llvm
1320
1321void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1322 ArrayRef<Value *> UserIgnoreLst) {
1323 ExtraValueToDebugLocsMap ExternallyUsedValues;
1324 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1325}
1326
1327void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1328 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1329 ArrayRef<Value *> UserIgnoreLst) {
1330 deleteTree();
1331 UserIgnoreList = UserIgnoreLst;
1332 if (!allSameType(Roots))
1333 return;
1334 buildTree_rec(Roots, 0, -1);
1335
1336 // Collect the values that we need to extract from the tree.
1337 for (TreeEntry &EIdx : VectorizableTree) {
1338 TreeEntry *Entry = &EIdx;
1339
1340 // No need to handle users of gathered values.
1341 if (Entry->NeedToGather)
1342 continue;
1343
1344 // For each lane:
1345 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1346 Value *Scalar = Entry->Scalars[Lane];
1347 int FoundLane = Lane;
1348 if (!Entry->ReuseShuffleIndices.empty()) {
1349 FoundLane =
1350 std::distance(Entry->ReuseShuffleIndices.begin(),
1351 llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1352 }
1353
1354 // Check if the scalar is externally used as an extra arg.
1355 auto ExtI = ExternallyUsedValues.find(Scalar);
1356 if (ExtI != ExternallyUsedValues.end()) {
1357 LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
1358 << Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
1359 ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1360 }
1361 for (User *U : Scalar->users()) {
1362 LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
1363
1364 Instruction *UserInst = dyn_cast<Instruction>(U);
1365 if (!UserInst)
1366 continue;
1367
1368 // Skip in-tree scalars that become vectors
1369 if (TreeEntry *UseEntry = getTreeEntry(U)) {
1370 Value *UseScalar = UseEntry->Scalars[0];
1371 // Some in-tree scalars will remain as scalar in vectorized
1372 // instructions. If that is the case, the one in Lane 0 will
1373 // be used.
1374 if (UseScalar != U ||
1375 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1376 LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1377 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
1378 assert(!UseEntry->NeedToGather && "Bad state")((!UseEntry->NeedToGather && "Bad state") ? static_cast
<void> (0) : __assert_fail ("!UseEntry->NeedToGather && \"Bad state\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1378, __PRETTY_FUNCTION__))
;
1379 continue;
1380 }
1381 }
1382
1383 // Ignore users in the user ignore list.
1384 if (is_contained(UserIgnoreList, UserInst))
1385 continue;
1386
1387 LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1388 << Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
1389 ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1390 }
1391 }
1392 }
1393}
1394
1395void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1396 int UserTreeIdx) {
1397 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(((allConstant(VL) || allSameType(VL)) && "Invalid types!"
) ? static_cast<void> (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1397, __PRETTY_FUNCTION__))
;
1
Assuming the condition is true
2
'?' condition is true
1398
1399 InstructionsState S = getSameOpcode(VL);
1400 if (Depth == RecursionMaxDepth) {
3
Assuming the condition is false
4
Taking false branch
1401 LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
1402 newTreeEntry(VL, false, UserTreeIdx);
1403 return;
1404 }
1405
1406 // Don't handle vectors.
1407 if (S.OpValue->getType()->isVectorTy()) {
5
Taking false branch
1408 LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
1409 newTreeEntry(VL, false, UserTreeIdx);
1410 return;
1411 }
1412
1413 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
6
Taking false branch
1414 if (SI->getValueOperand()->getType()->isVectorTy()) {
1415 LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1416 newTreeEntry(VL, false, UserTreeIdx);
1417 return;
1418 }
1419
1420 // If all of the operands are identical or constant we have a simple solution.
1421 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
7
Assuming the condition is false
8
Assuming the condition is false
9
Assuming the condition is false
10
Assuming the condition is false
11
Taking false branch
1422 LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1423 newTreeEntry(VL, false, UserTreeIdx);
1424 return;
1425 }
1426
1427 // We now know that this is a vector of instructions of the same type from
1428 // the same block.
1429
1430 // Don't vectorize ephemeral values.
1431 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
12
Assuming 'i' is equal to 'e'
13
Loop condition is false. Execution continues on line 1441
1432 if (EphValues.count(VL[i])) {
1433 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1434 << ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1435 newTreeEntry(VL, false, UserTreeIdx);
1436 return;
1437 }
1438 }
1439
1440 // Check if this is a duplicate of another entry.
1441 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
14
Taking false branch
1442 LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*S.OpValue << ".\n"; } } while (false)
;
1443 if (!E->isSame(VL)) {
1444 LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1445 newTreeEntry(VL, false, UserTreeIdx);
1446 return;
1447 }
1448 // Record the reuse of the tree node. FIXME, currently this is only used to
1449 // properly draw the graph rather than for the actual vectorization.
1450 E->UserTreeIndices.push_back(UserTreeIdx);
1451 LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValuedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
1452 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*S.OpValue << ".\n"; } } while (false)
;
1453 return;
1454 }
1455
1456 // Check that none of the instructions in the bundle are already in the tree.
1457 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
15
Assuming 'i' is equal to 'e'
16
Loop condition is false. Execution continues on line 1471
1458 auto *I = dyn_cast<Instruction>(VL[i]);
1459 if (!I)
1460 continue;
1461 if (getTreeEntry(I)) {
1462 LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1463 << ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1464 newTreeEntry(VL, false, UserTreeIdx);
1465 return;
1466 }
1467 }
1468
1469 // If any of the scalars is marked as a value that needs to stay scalar, then
1470 // we need to gather the scalars.
1471 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
17
Assuming 'i' is equal to 'e'
18
Loop condition is false. Execution continues on line 1481
1472 if (MustGather.count(VL[i])) {
1473 LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1474 newTreeEntry(VL, false, UserTreeIdx);
1475 return;
1476 }
1477 }
1478
1479 // Check that all of the users of the scalars that we want to vectorize are
1480 // schedulable.
1481 auto *VL0 = cast<Instruction>(S.OpValue);
1482 BasicBlock *BB = VL0->getParent();
1483
1484 if (!DT->isReachableFromEntry(BB)) {
19
Assuming the condition is false
20
Taking false branch
1485 // Don't go into unreachable blocks. They may contain instructions with
1486 // dependency cycles which confuse the final scheduling.
1487 LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1488 newTreeEntry(VL, false, UserTreeIdx);
1489 return;
1490 }
1491
1492 // Check that every instruction appears once in this bundle.
1493 SmallVector<unsigned, 4> ReuseShuffleIndicies;
1494 SmallVector<Value *, 4> UniqueValues;
1495 DenseMap<Value *, unsigned> UniquePositions;
1496 for (Value *V : VL) {
21
Assuming '__begin1' is equal to '__end1'
1497 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1498 ReuseShuffleIndicies.emplace_back(Res.first->second);
1499 if (Res.second)
1500 UniqueValues.emplace_back(V);
1501 }
1502 if (UniqueValues.size() == VL.size()) {
22
Assuming the condition is true
23
Taking true branch
1503 ReuseShuffleIndicies.clear();
1504 } else {
1505 LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Shuffle for reused scalars.\n"
; } } while (false)
;
1506 if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1507 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1508 newTreeEntry(VL, false, UserTreeIdx);
1509 return;
1510 }
1511 VL = UniqueValues;
1512 }
1513
1514 auto &BSRef = BlocksSchedules[BB];
1515 if (!BSRef)
24
Taking false branch
1516 BSRef = llvm::make_unique<BlockScheduling>(BB);
1517
1518 BlockScheduling &BS = *BSRef.get();
1519
1520 if (!BS.tryScheduleBundle(VL, this, S)) {
25
Calling 'BlockScheduling::tryScheduleBundle'
1521 LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1522 assert((!BS.getScheduleData(VL0) ||(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1524, __PRETTY_FUNCTION__))
1523 !BS.getScheduleData(VL0)->isPartOfBundle()) &&(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1524, __PRETTY_FUNCTION__))
1524 "tryScheduleBundle should cancelScheduling on failure")(((!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle
()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL0) || !BS.getScheduleData(VL0)->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1524, __PRETTY_FUNCTION__))
;
1525 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1526 return;
1527 }
1528 LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1529
1530 unsigned ShuffleOrOp = S.isAltShuffle() ?
1531 (unsigned) Instruction::ShuffleVector : S.getOpcode();
1532 switch (ShuffleOrOp) {
1533 case Instruction::PHI: {
1534 PHINode *PH = dyn_cast<PHINode>(VL0);
1535
1536 // Check for terminator values (e.g. invoke).
1537 for (unsigned j = 0; j < VL.size(); ++j)
1538 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1539 Instruction *Term = dyn_cast<Instruction>(
1540 cast<PHINode>(VL[j])->getIncomingValueForBlock(
1541 PH->getIncomingBlock(i)));
1542 if (Term && Term->isTerminator()) {
1543 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (terminator use).\n"
; } } while (false)
1544 << "SLP: Need to swizzle PHINodes (terminator use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (terminator use).\n"
; } } while (false)
;
1545 BS.cancelScheduling(VL, VL0);
1546 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1547 return;
1548 }
1549 }
1550
1551 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1552 LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1553
1554 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1555 ValueList Operands;
1556 // Prepare the operand vector.
1557 for (Value *j : VL)
1558 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1559 PH->getIncomingBlock(i)));
1560
1561 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1562 }
1563 return;
1564 }
1565 case Instruction::ExtractValue:
1566 case Instruction::ExtractElement: {
1567 OrdersType CurrentOrder;
1568 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
1569 if (Reuse) {
1570 LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing or shuffling extract sequence.\n"
; } } while (false)
;
1571 ++NumOpsWantToKeepOriginalOrder;
1572 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1573 ReuseShuffleIndicies);
1574 return;
1575 }
1576 if (!CurrentOrder.empty()) {
1577 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1578 dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1579 "with order";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1580 for (unsigned Idx : CurrentOrder)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1581 dbgs() << " " << Idx;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1582 dbgs() << "\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
1583 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
"with order"; for (unsigned Idx : CurrentOrder) dbgs() <<
" " << Idx; dbgs() << "\n"; }; } } while (false)
;
1584 // Insert new order with initial value 0, if it does not exist,
1585 // otherwise return the iterator to the existing one.
1586 auto StoredCurrentOrderAndNum =
1587 NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1588 ++StoredCurrentOrderAndNum->getSecond();
1589 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx, ReuseShuffleIndicies,
1590 StoredCurrentOrderAndNum->getFirst());
1591 return;
1592 }
1593 LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gather extract sequence.\n";
} } while (false)
;
1594 newTreeEntry(VL, /*Vectorized=*/false, UserTreeIdx, ReuseShuffleIndicies);
1595 BS.cancelScheduling(VL, VL0);
1596 return;
1597 }
1598 case Instruction::Load: {
1599 // Check that a vectorized load would load the same memory as a scalar
1600 // load. For example, we don't want to vectorize loads that are smaller
1601 // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1602 // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1603 // from such a struct, we read/write packed bits disagreeing with the
1604 // unvectorized version.
1605 Type *ScalarTy = VL0->getType();
1606
1607 if (DL->getTypeSizeInBits(ScalarTy) !=
1608 DL->getTypeAllocSizeInBits(ScalarTy)) {
1609 BS.cancelScheduling(VL, VL0);
1610 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1611 LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1612 return;
1613 }
1614
1615 // Make sure all loads in the bundle are simple - we can't vectorize
1616 // atomic or volatile loads.
1617 SmallVector<Value *, 4> PointerOps(VL.size());
1618 auto POIter = PointerOps.begin();
1619 for (Value *V : VL) {
1620 auto *L = cast<LoadInst>(V);
1621 if (!L->isSimple()) {
1622 BS.cancelScheduling(VL, VL0);
1623 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1624 LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1625 return;
1626 }
1627 *POIter = L->getPointerOperand();
1628 ++POIter;
1629 }
1630
1631 OrdersType CurrentOrder;
1632 // Check the order of pointer operands.
1633 if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
1634 Value *Ptr0;
1635 Value *PtrN;
1636 if (CurrentOrder.empty()) {
1637 Ptr0 = PointerOps.front();
1638 PtrN = PointerOps.back();
1639 } else {
1640 Ptr0 = PointerOps[CurrentOrder.front()];
1641 PtrN = PointerOps[CurrentOrder.back()];
1642 }
1643 const SCEV *Scev0 = SE->getSCEV(Ptr0);
1644 const SCEV *ScevN = SE->getSCEV(PtrN);
1645 const auto *Diff =
1646 dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
1647 uint64_t Size = DL->getTypeAllocSize(ScalarTy);
1648 // Check that the sorted loads are consecutive.
1649 if (Diff && Diff->getAPInt().getZExtValue() == (VL.size() - 1) * Size) {
1650 if (CurrentOrder.empty()) {
1651 // Original loads are consecutive and does not require reordering.
1652 ++NumOpsWantToKeepOriginalOrder;
1653 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1654 ReuseShuffleIndicies);
1655 LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1656 } else {
1657 // Need to reorder.
1658 auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1659 ++I->getSecond();
1660 newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1661 ReuseShuffleIndicies, I->getFirst());
1662 LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of jumbled loads.\n"
; } } while (false)
;
1663 }
1664 return;
1665 }
1666 }
1667
1668 LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1669 BS.cancelScheduling(VL, VL0);
1670 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1671 return;
1672 }
1673 case Instruction::ZExt:
1674 case Instruction::SExt:
1675 case Instruction::FPToUI:
1676 case Instruction::FPToSI:
1677 case Instruction::FPExt:
1678 case Instruction::PtrToInt:
1679 case Instruction::IntToPtr:
1680 case Instruction::SIToFP:
1681 case Instruction::UIToFP:
1682 case Instruction::Trunc:
1683 case Instruction::FPTrunc:
1684 case Instruction::BitCast: {
1685 Type *SrcTy = VL0->getOperand(0)->getType();
1686 for (unsigned i = 0; i < VL.size(); ++i) {
1687 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1688 if (Ty != SrcTy || !isValidElementType(Ty)) {
1689 BS.cancelScheduling(VL, VL0);
1690 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1691 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
1692 << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1693 return;
1694 }
1695 }
1696 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1697 LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1698
1699 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1700 ValueList Operands;
1701 // Prepare the operand vector.
1702 for (Value *j : VL)
1703 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1704
1705 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1706 }
1707 return;
1708 }
1709 case Instruction::ICmp:
1710 case Instruction::FCmp: {
1711 // Check that all of the compares have the same predicate.
1712 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1713 Type *ComparedTy = VL0->getOperand(0)->getType();
1714 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1715 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1716 if (Cmp->getPredicate() != P0 ||
1717 Cmp->getOperand(0)->getType() != ComparedTy) {
1718 BS.cancelScheduling(VL, VL0);
1719 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1720 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
1721 << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1722 return;
1723 }
1724 }
1725
1726 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1727 LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1728
1729 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1730 ValueList Operands;
1731 // Prepare the operand vector.
1732 for (Value *j : VL)
1733 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1734
1735 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1736 }
1737 return;
1738 }
1739 case Instruction::Select:
1740 case Instruction::Add:
1741 case Instruction::FAdd:
1742 case Instruction::Sub:
1743 case Instruction::FSub:
1744 case Instruction::Mul:
1745 case Instruction::FMul:
1746 case Instruction::UDiv:
1747 case Instruction::SDiv:
1748 case Instruction::FDiv:
1749 case Instruction::URem:
1750 case Instruction::SRem:
1751 case Instruction::FRem:
1752 case Instruction::Shl:
1753 case Instruction::LShr:
1754 case Instruction::AShr:
1755 case Instruction::And:
1756 case Instruction::Or:
1757 case Instruction::Xor:
1758 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1759 LLVM_DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1760
1761 // Sort operands of the instructions so that each side is more likely to
1762 // have the same opcode.
1763 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1764 ValueList Left, Right;
1765 reorderInputsAccordingToOpcode(S.getOpcode(), VL, Left, Right);
1766 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1767 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1768 return;
1769 }
1770
1771 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1772 ValueList Operands;
1773 // Prepare the operand vector.
1774 for (Value *j : VL)
1775 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1776
1777 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1778 }
1779 return;
1780
1781 case Instruction::GetElementPtr: {
1782 // We don't combine GEPs with complicated (nested) indexing.
1783 for (unsigned j = 0; j < VL.size(); ++j) {
1784 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1785 LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1786 BS.cancelScheduling(VL, VL0);
1787 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1788 return;
1789 }
1790 }
1791
1792 // We can't combine several GEPs into one vector if they operate on
1793 // different types.
1794 Type *Ty0 = VL0->getOperand(0)->getType();
1795 for (unsigned j = 0; j < VL.size(); ++j) {
1796 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1797 if (Ty0 != CurTy) {
1798 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
1799 << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1800 BS.cancelScheduling(VL, VL0);
1801 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1802 return;
1803 }
1804 }
1805
1806 // We don't combine GEPs with non-constant indexes.
1807 for (unsigned j = 0; j < VL.size(); ++j) {
1808 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1809 if (!isa<ConstantInt>(Op)) {
1810 LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1811 << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1812 BS.cancelScheduling(VL, VL0);
1813 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1814 return;
1815 }
1816 }
1817
1818 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1819 LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1820 for (unsigned i = 0, e = 2; i < e; ++i) {
1821 ValueList Operands;
1822 // Prepare the operand vector.
1823 for (Value *j : VL)
1824 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1825
1826 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1827 }
1828 return;
1829 }
1830 case Instruction::Store: {
1831 // Check if the stores are consecutive or of we need to swizzle them.
1832 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1833 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1834 BS.cancelScheduling(VL, VL0);
1835 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1836 LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1837 return;
1838 }
1839
1840 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1841 LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1842
1843 ValueList Operands;
1844 for (Value *j : VL)
1845 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1846
1847 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1848 return;
1849 }
1850 case Instruction::Call: {
1851 // Check if the calls are all to the same vectorizable intrinsic.
1852 CallInst *CI = cast<CallInst>(VL0);
1853 // Check if this is an Intrinsic call or something that can be
1854 // represented by an intrinsic call
1855 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1856 if (!isTriviallyVectorizable(ID)) {
1857 BS.cancelScheduling(VL, VL0);
1858 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1859 LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1860 return;
1861 }
1862 Function *Int = CI->getCalledFunction();
1863 Value *A1I = nullptr;
1864 if (hasVectorInstrinsicScalarOpd(ID, 1))
1865 A1I = CI->getArgOperand(1);
1866 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1867 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1868 if (!CI2 || CI2->getCalledFunction() != Int ||
1869 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1870 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1871 BS.cancelScheduling(VL, VL0);
1872 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1873 LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1874 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1875 return;
1876 }
1877 // ctlz,cttz and powi are special intrinsics whose second argument
1878 // should be same in order for them to be vectorized.
1879 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1880 Value *A1J = CI2->getArgOperand(1);
1881 if (A1I != A1J) {
1882 BS.cancelScheduling(VL, VL0);
1883 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1884 LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << A1I << "!="
<< A1J << "\n"; } } while (false)
1885 << " argument " << A1I << "!=" << A1J << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument " << A1I << "!="
<< A1J << "\n"; } } while (false)
;
1886 return;
1887 }
1888 }
1889 // Verify that the bundle operands are identical between the two calls.
1890 if (CI->hasOperandBundles() &&
1891 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1892 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1893 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1894 BS.cancelScheduling(VL, VL0);
1895 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1896 LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1897 << *CI << "!=" << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1898 return;
1899 }
1900 }
1901
1902 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1903 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1904 ValueList Operands;
1905 // Prepare the operand vector.
1906 for (Value *j : VL) {
1907 CallInst *CI2 = dyn_cast<CallInst>(j);
1908 Operands.push_back(CI2->getArgOperand(i));
1909 }
1910 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1911 }
1912 return;
1913 }
1914 case Instruction::ShuffleVector:
1915 // If this is not an alternate sequence of opcode like add-sub
1916 // then do not vectorize this instruction.
1917 if (!S.isAltShuffle()) {
1918 BS.cancelScheduling(VL, VL0);
1919 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1920 LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
1921 return;
1922 }
1923 newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1924 LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
1925
1926 // Reorder operands if reordering would enable vectorization.
1927 if (isa<BinaryOperator>(VL0)) {
1928 ValueList Left, Right;
1929 reorderAltShuffleOperands(S, VL, Left, Right);
1930 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1931 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1932 return;
1933 }
1934
1935 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1936 ValueList Operands;
1937 // Prepare the operand vector.
1938 for (Value *j : VL)
1939 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1940
1941 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1942 }
1943 return;
1944
1945 default:
1946 BS.cancelScheduling(VL, VL0);
1947 newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1948 LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
1949 return;
1950 }
1951}
1952
1953unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1954 unsigned N;
1955 Type *EltTy;
1956 auto *ST = dyn_cast<StructType>(T);
1957 if (ST) {
1958 N = ST->getNumElements();
1959 EltTy = *ST->element_begin();
1960 } else {
1961 N = cast<ArrayType>(T)->getNumElements();
1962 EltTy = cast<ArrayType>(T)->getElementType();
1963 }
1964 if (!isValidElementType(EltTy))
1965 return 0;
1966 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1967 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1968 return 0;
1969 if (ST) {
1970 // Check that struct is homogeneous.
1971 for (const auto *Ty : ST->elements())
1972 if (Ty != EltTy)
1973 return 0;
1974 }
1975 return N;
1976}
1977
1978bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1979 SmallVectorImpl<unsigned> &CurrentOrder) const {
1980 Instruction *E0 = cast<Instruction>(OpValue);
1981 assert(E0->getOpcode() == Instruction::ExtractElement ||((E0->getOpcode() == Instruction::ExtractElement || E0->
getOpcode() == Instruction::ExtractValue) ? static_cast<void
> (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1982, __PRETTY_FUNCTION__))
1982 E0->getOpcode() == Instruction::ExtractValue)((E0->getOpcode() == Instruction::ExtractElement || E0->
getOpcode() == Instruction::ExtractValue) ? static_cast<void
> (0) : __assert_fail ("E0->getOpcode() == Instruction::ExtractElement || E0->getOpcode() == Instruction::ExtractValue"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1982, __PRETTY_FUNCTION__))
;
1983 assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode")((E0->getOpcode() == getSameOpcode(VL).getOpcode() &&
"Invalid opcode") ? static_cast<void> (0) : __assert_fail
("E0->getOpcode() == getSameOpcode(VL).getOpcode() && \"Invalid opcode\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1983, __PRETTY_FUNCTION__))
;
1984 // Check if all of the extracts come from the same vector and from the
1985 // correct offset.
1986 Value *Vec = E0->getOperand(0);
1987
1988 CurrentOrder.clear();
1989
1990 // We have to extract from a vector/aggregate with the same number of elements.
1991 unsigned NElts;
1992 if (E0->getOpcode() == Instruction::ExtractValue) {
1993 const DataLayout &DL = E0->getModule()->getDataLayout();
1994 NElts = canMapToVector(Vec->getType(), DL);
1995 if (!NElts)
1996 return false;
1997 // Check if load can be rewritten as load of vector.
1998 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1999 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2000 return false;
2001 } else {
2002 NElts = Vec->getType()->getVectorNumElements();
2003 }
2004
2005 if (NElts != VL.size())
2006 return false;
2007
2008 // Check that all of the indices extract from the correct offset.
2009 bool ShouldKeepOrder = true;
2010 unsigned E = VL.size();
2011 // Assign to all items the initial value E + 1 so we can check if the extract
2012 // instruction index was used already.
2013 // Also, later we can check that all the indices are used and we have a
2014 // consecutive access in the extract instructions, by checking that no
2015 // element of CurrentOrder still has value E + 1.
2016 CurrentOrder.assign(E, E + 1);
2017 unsigned I = 0;
2018 for (; I < E; ++I) {
2019 auto *Inst = cast<Instruction>(VL[I]);
2020 if (Inst->getOperand(0) != Vec)
2021 break;
2022 Optional<unsigned> Idx = getExtractIndex(Inst);
2023 if (!Idx)
2024 break;
2025 const unsigned ExtIdx = *Idx;
2026 if (ExtIdx != I) {
2027 if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
2028 break;
2029 ShouldKeepOrder = false;
2030 CurrentOrder[ExtIdx] = I;
2031 } else {
2032 if (CurrentOrder[I] != E + 1)
2033 break;
2034 CurrentOrder[I] = I;
2035 }
2036 }
2037 if (I < E) {
2038 CurrentOrder.clear();
2039 return false;
2040 }
2041
2042 return ShouldKeepOrder;
2043}
2044
2045bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2046 return I->hasOneUse() ||
2047 std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2048 return ScalarToTreeEntry.count(U) > 0;
2049 });
2050}
2051
2052int BoUpSLP::getEntryCost(TreeEntry *E) {
2053 ArrayRef<Value*> VL = E->Scalars;
2054
2055 Type *ScalarTy = VL[0]->getType();
2056 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2057 ScalarTy = SI->getValueOperand()->getType();
2058 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2059 ScalarTy = CI->getOperand(0)->getType();
2060 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2061
2062 // If we have computed a smaller type for the expression, update VecTy so
2063 // that the costs will be accurate.
2064 if (MinBWs.count(VL[0]))
2065 VecTy = VectorType::get(
2066 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2067
2068 unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2069 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2070 int ReuseShuffleCost = 0;
2071 if (NeedToShuffleReuses) {
2072 ReuseShuffleCost =
2073 TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2074 }
2075 if (E->NeedToGather) {
2076 if (allConstant(VL))
2077 return 0;
2078 if (isSplat(VL)) {
2079 return ReuseShuffleCost +
2080 TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2081 }
2082 if (getSameOpcode(VL).getOpcode() == Instruction::ExtractElement &&
2083 allSameType(VL) && allSameBlock(VL)) {
2084 Optional<TargetTransformInfo::ShuffleKind> ShuffleKind = isShuffle(VL);
2085 if (ShuffleKind.hasValue()) {
2086 int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2087 for (auto *V : VL) {
2088 // If all users of instruction are going to be vectorized and this
2089 // instruction itself is not going to be vectorized, consider this
2090 // instruction as dead and remove its cost from the final cost of the
2091 // vectorized tree.
2092 if (areAllUsersVectorized(cast<Instruction>(V)) &&
2093 !ScalarToTreeEntry.count(V)) {
2094 auto *IO = cast<ConstantInt>(
2095 cast<ExtractElementInst>(V)->getIndexOperand());
2096 Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2097 IO->getZExtValue());
2098 }
2099 }
2100 return ReuseShuffleCost + Cost;
2101 }
2102 }
2103 return ReuseShuffleCost + getGatherCost(VL);
2104 }
2105 InstructionsState S = getSameOpcode(VL);
2106 assert(S.getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL")((S.getOpcode() && allSameType(VL) && allSameBlock
(VL) && "Invalid VL") ? static_cast<void> (0) :
__assert_fail ("S.getOpcode() && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2106, __PRETTY_FUNCTION__))
;
2107 Instruction *VL0 = cast<Instruction>(S.OpValue);
2108 unsigned ShuffleOrOp = S.isAltShuffle() ?
2109 (unsigned) Instruction::ShuffleVector : S.getOpcode();
2110 switch (ShuffleOrOp) {
2111 case Instruction::PHI:
2112 return 0;
2113
2114 case Instruction::ExtractValue:
2115 case Instruction::ExtractElement:
2116 if (NeedToShuffleReuses) {
2117 unsigned Idx = 0;
2118 for (unsigned I : E->ReuseShuffleIndices) {
2119 if (ShuffleOrOp == Instruction::ExtractElement) {
2120 auto *IO = cast<ConstantInt>(
2121 cast<ExtractElementInst>(VL[I])->getIndexOperand());
2122 Idx = IO->getZExtValue();
2123 ReuseShuffleCost -= TTI->getVectorInstrCost(
2124 Instruction::ExtractElement, VecTy, Idx);
2125 } else {
2126 ReuseShuffleCost -= TTI->getVectorInstrCost(
2127 Instruction::ExtractElement, VecTy, Idx);
2128 ++Idx;
2129 }
2130 }
2131 Idx = ReuseShuffleNumbers;
2132 for (Value *V : VL) {
2133 if (ShuffleOrOp == Instruction::ExtractElement) {
2134 auto *IO = cast<ConstantInt>(
2135 cast<ExtractElementInst>(V)->getIndexOperand());
2136 Idx = IO->getZExtValue();
2137 } else {
2138 --Idx;
2139 }
2140 ReuseShuffleCost +=
2141 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2142 }
2143 }
2144 if (!E->NeedToGather) {
2145 int DeadCost = ReuseShuffleCost;
2146 if (!E->ReorderIndices.empty()) {
2147 // TODO: Merge this shuffle with the ReuseShuffleCost.
2148 DeadCost += TTI->getShuffleCost(
2149 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2150 }
2151 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2152 Instruction *E = cast<Instruction>(VL[i]);
2153 // If all users are going to be vectorized, instruction can be
2154 // considered as dead.
2155 // The same, if have only one user, it will be vectorized for sure.
2156 if (areAllUsersVectorized(E)) {
2157 // Take credit for instruction that will become dead.
2158 if (E->hasOneUse()) {
2159 Instruction *Ext = E->user_back();
2160 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2161 all_of(Ext->users(),
2162 [](User *U) { return isa<GetElementPtrInst>(U); })) {
2163 // Use getExtractWithExtendCost() to calculate the cost of
2164 // extractelement/ext pair.
2165 DeadCost -= TTI->getExtractWithExtendCost(
2166 Ext->getOpcode(), Ext->getType(), VecTy, i);
2167 // Add back the cost of s|zext which is subtracted seperately.
2168 DeadCost += TTI->getCastInstrCost(
2169 Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
2170 continue;
2171 }
2172 }
2173 DeadCost -=
2174 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2175 }
2176 }
2177 return DeadCost;
2178 }
2179 return ReuseShuffleCost + getGatherCost(VL);
2180
2181 case Instruction::ZExt:
2182 case Instruction::SExt:
2183 case Instruction::FPToUI:
2184 case Instruction::FPToSI:
2185 case Instruction::FPExt:
2186 case Instruction::PtrToInt:
2187 case Instruction::IntToPtr:
2188 case Instruction::SIToFP:
2189 case Instruction::UIToFP:
2190 case Instruction::Trunc:
2191 case Instruction::FPTrunc:
2192 case Instruction::BitCast: {
2193 Type *SrcTy = VL0->getOperand(0)->getType();
2194 int ScalarEltCost =
2195 TTI->getCastInstrCost(S.getOpcode(), ScalarTy, SrcTy, VL0);
2196 if (NeedToShuffleReuses) {
2197 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2198 }
2199
2200 // Calculate the cost of this instruction.
2201 int ScalarCost = VL.size() * ScalarEltCost;
2202
2203 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2204 int VecCost = 0;
2205 // Check if the values are candidates to demote.
2206 if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2207 VecCost = ReuseShuffleCost +
2208 TTI->getCastInstrCost(S.getOpcode(), VecTy, SrcVecTy, VL0);
2209 }
2210 return VecCost - ScalarCost;
2211 }
2212 case Instruction::FCmp:
2213 case Instruction::ICmp:
2214 case Instruction::Select: {
2215 // Calculate the cost of this instruction.
2216 int ScalarEltCost = TTI->getCmpSelInstrCost(S.getOpcode(), ScalarTy,
2217 Builder.getInt1Ty(), VL0);
2218 if (NeedToShuffleReuses) {
2219 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2220 }
2221 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2222 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2223 int VecCost = TTI->getCmpSelInstrCost(S.getOpcode(), VecTy, MaskTy, VL0);
2224 return ReuseShuffleCost + VecCost - ScalarCost;
2225 }
2226 case Instruction::Add:
2227 case Instruction::FAdd:
2228 case Instruction::Sub:
2229 case Instruction::FSub:
2230 case Instruction::Mul:
2231 case Instruction::FMul:
2232 case Instruction::UDiv:
2233 case Instruction::SDiv:
2234 case Instruction::FDiv:
2235 case Instruction::URem:
2236 case Instruction::SRem:
2237 case Instruction::FRem:
2238 case Instruction::Shl:
2239 case Instruction::LShr:
2240 case Instruction::AShr:
2241 case Instruction::And:
2242 case Instruction::Or:
2243 case Instruction::Xor: {
2244 // Certain instructions can be cheaper to vectorize if they have a
2245 // constant second vector operand.
2246 TargetTransformInfo::OperandValueKind Op1VK =
2247 TargetTransformInfo::OK_AnyValue;
2248 TargetTransformInfo::OperandValueKind Op2VK =
2249 TargetTransformInfo::OK_UniformConstantValue;
2250 TargetTransformInfo::OperandValueProperties Op1VP =
2251 TargetTransformInfo::OP_None;
2252 TargetTransformInfo::OperandValueProperties Op2VP =
2253 TargetTransformInfo::OP_PowerOf2;
2254
2255 // If all operands are exactly the same ConstantInt then set the
2256 // operand kind to OK_UniformConstantValue.
2257 // If instead not all operands are constants, then set the operand kind
2258 // to OK_AnyValue. If all operands are constants but not the same,
2259 // then set the operand kind to OK_NonUniformConstantValue.
2260 ConstantInt *CInt0 = nullptr;
2261 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2262 const Instruction *I = cast<Instruction>(VL[i]);
2263 ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(1));
2264 if (!CInt) {
2265 Op2VK = TargetTransformInfo::OK_AnyValue;
2266 Op2VP = TargetTransformInfo::OP_None;
2267 break;
2268 }
2269 if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
2270 !CInt->getValue().isPowerOf2())
2271 Op2VP = TargetTransformInfo::OP_None;
2272 if (i == 0) {
2273 CInt0 = CInt;
2274 continue;
2275 }
2276 if (CInt0 != CInt)
2277 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
2278 }
2279
2280 SmallVector<const Value *, 4> Operands(VL0->operand_values());
2281 int ScalarEltCost = TTI->getArithmeticInstrCost(
2282 S.getOpcode(), ScalarTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands);
2283 if (NeedToShuffleReuses) {
2284 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2285 }
2286 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2287 int VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy, Op1VK,
2288 Op2VK, Op1VP, Op2VP, Operands);
2289 return ReuseShuffleCost + VecCost - ScalarCost;
2290 }
2291 case Instruction::GetElementPtr: {
2292 TargetTransformInfo::OperandValueKind Op1VK =
2293 TargetTransformInfo::OK_AnyValue;
2294 TargetTransformInfo::OperandValueKind Op2VK =
2295 TargetTransformInfo::OK_UniformConstantValue;
2296
2297 int ScalarEltCost =
2298 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2299 if (NeedToShuffleReuses) {
2300 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2301 }
2302 int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2303 int VecCost =
2304 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2305 return ReuseShuffleCost + VecCost - ScalarCost;
2306 }
2307 case Instruction::Load: {
2308 // Cost of wide load - cost of scalar loads.
2309 unsigned alignment = cast<LoadInst>(VL0)->getAlignment();
2310 int ScalarEltCost =
2311 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2312 if (NeedToShuffleReuses) {
2313 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2314 }
2315 int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
2316 int VecLdCost =
2317 TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, VL0);
2318 if (!E->ReorderIndices.empty()) {
2319 // TODO: Merge this shuffle with the ReuseShuffleCost.
2320 VecLdCost += TTI->getShuffleCost(
2321 TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2322 }
2323 return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2324 }
2325 case Instruction::Store: {
2326 // We know that we can merge the stores. Calculate the cost.
2327 unsigned alignment = cast<StoreInst>(VL0)->getAlignment();
2328 int ScalarEltCost =
2329 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2330 if (NeedToShuffleReuses) {
2331 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2332 }
2333 int ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
2334 int VecStCost =
2335 TTI->getMemoryOpCost(Instruction::Store, VecTy, alignment, 0, VL0);
2336 return ReuseShuffleCost + VecStCost - ScalarStCost;
2337 }
2338 case Instruction::Call: {
2339 CallInst *CI = cast<CallInst>(VL0);
2340 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2341
2342 // Calculate the cost of the scalar and vector calls.
2343 SmallVector<Type *, 4> ScalarTys;
2344 for (unsigned op = 0, opc = CI->getNumArgOperands(); op != opc; ++op)
2345 ScalarTys.push_back(CI->getArgOperand(op)->getType());
2346
2347 FastMathFlags FMF;
2348 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2349 FMF = FPMO->getFastMathFlags();
2350
2351 int ScalarEltCost =
2352 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2353 if (NeedToShuffleReuses) {
2354 ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2355 }
2356 int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
2357
2358 SmallVector<Value *, 4> Args(CI->arg_operands());
2359 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2360 VecTy->getNumElements());
2361
2362 LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2363 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
2364 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost " << VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
2365
2366 return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2367 }
2368 case Instruction::ShuffleVector: {
2369 assert(S.isAltShuffle() &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2374, __PRETTY_FUNCTION__))
2370 ((Instruction::isBinaryOp(S.getOpcode()) &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2374, __PRETTY_FUNCTION__))
2371 Instruction::isBinaryOp(S.getAltOpcode())) ||((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2374, __PRETTY_FUNCTION__))
2372 (Instruction::isCast(S.getOpcode()) &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2374, __PRETTY_FUNCTION__))
2373 Instruction::isCast(S.getAltOpcode()))) &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2374, __PRETTY_FUNCTION__))
2374 "Invalid Shuffle Vector Operand")((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2374, __PRETTY_FUNCTION__))
;
2375 int ScalarCost = 0;
2376 if (NeedToShuffleReuses) {
2377 for (unsigned Idx : E->ReuseShuffleIndices) {
2378 Instruction *I = cast<Instruction>(VL[Idx]);
2379 ReuseShuffleCost -= TTI->getInstructionCost(
2380 I, TargetTransformInfo::TCK_RecipThroughput);
2381 }
2382 for (Value *V : VL) {
2383 Instruction *I = cast<Instruction>(V);
2384 ReuseShuffleCost += TTI->getInstructionCost(
2385 I, TargetTransformInfo::TCK_RecipThroughput);
2386 }
2387 }
2388 for (Value *i : VL) {
2389 Instruction *I = cast<Instruction>(i);
2390 assert(S.isOpcodeOrAlt(I) && "Unexpected main/alternate opcode")((S.isOpcodeOrAlt(I) && "Unexpected main/alternate opcode"
) ? static_cast<void> (0) : __assert_fail ("S.isOpcodeOrAlt(I) && \"Unexpected main/alternate opcode\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2390, __PRETTY_FUNCTION__))
;
2391 ScalarCost += TTI->getInstructionCost(
2392 I, TargetTransformInfo::TCK_RecipThroughput);
2393 }
2394 // VecCost is equal to sum of the cost of creating 2 vectors
2395 // and the cost of creating shuffle.
2396 int VecCost = 0;
2397 if (Instruction::isBinaryOp(S.getOpcode())) {
2398 VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy);
2399 VecCost += TTI->getArithmeticInstrCost(S.getAltOpcode(), VecTy);
2400 } else {
2401 Type *Src0SclTy = S.MainOp->getOperand(0)->getType();
2402 Type *Src1SclTy = S.AltOp->getOperand(0)->getType();
2403 VectorType *Src0Ty = VectorType::get(Src0SclTy, VL.size());
2404 VectorType *Src1Ty = VectorType::get(Src1SclTy, VL.size());
2405 VecCost = TTI->getCastInstrCost(S.getOpcode(), VecTy, Src0Ty);
2406 VecCost += TTI->getCastInstrCost(S.getAltOpcode(), VecTy, Src1Ty);
2407 }
2408 VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
2409 return ReuseShuffleCost + VecCost - ScalarCost;
2410 }
2411 default:
2412 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2412)
;
2413 }
2414}
2415
2416bool BoUpSLP::isFullyVectorizableTinyTree() {
2417 LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
2418 << VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
2419
2420 // We only handle trees of heights 1 and 2.
2421 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2422 return true;
2423
2424 if (VectorizableTree.size() != 2)
2425 return false;
2426
2427 // Handle splat and all-constants stores.
2428 if (!VectorizableTree[0].NeedToGather &&
2429 (allConstant(VectorizableTree[1].Scalars) ||
2430 isSplat(VectorizableTree[1].Scalars)))
2431 return true;
2432
2433 // Gathering cost would be too much for tiny trees.
2434 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2435 return false;
2436
2437 return true;
2438}
2439
2440bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2441 // We can vectorize the tree if its size is greater than or equal to the
2442 // minimum size specified by the MinTreeSize command line option.
2443 if (VectorizableTree.size() >= MinTreeSize)
2444 return false;
2445
2446 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2447 // can vectorize it if we can prove it fully vectorizable.
2448 if (isFullyVectorizableTinyTree())
2449 return false;
2450
2451 assert(VectorizableTree.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2453, __PRETTY_FUNCTION__))
2452 ? ExternalUses.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2453, __PRETTY_FUNCTION__))
2453 : true && "We shouldn't have any external users")((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2453, __PRETTY_FUNCTION__))
;
2454
2455 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2456 // vectorizable.
2457 return true;
2458}
2459
2460int BoUpSLP::getSpillCost() {
2461 // Walk from the bottom of the tree to the top, tracking which values are
2462 // live. When we see a call instruction that is not part of our tree,
2463 // query TTI to see if there is a cost to keeping values live over it
2464 // (for example, if spills and fills are required).
2465 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2466 int Cost = 0;
2467
2468 SmallPtrSet<Instruction*, 4> LiveValues;
2469 Instruction *PrevInst = nullptr;
2470
2471 for (const auto &N : VectorizableTree) {
2472 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2473 if (!Inst)
2474 continue;
2475
2476 if (!PrevInst) {
2477 PrevInst = Inst;
2478 continue;
2479 }
2480
2481 // Update LiveValues.
2482 LiveValues.erase(PrevInst);
2483 for (auto &J : PrevInst->operands()) {
2484 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2485 LiveValues.insert(cast<Instruction>(&*J));
2486 }
2487
2488 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2489 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2490 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2491 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2492 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2493 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
2494 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { { dbgs() << "SLP: #LV: " << LiveValues
.size(); for (auto *X : LiveValues) dbgs() << " " <<
X->getName(); dbgs() << ", Looking at "; Inst->dump
(); }; } } while (false)
;
2495
2496 // Now find the sequence of instructions between PrevInst and Inst.
2497 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2498 PrevInstIt =
2499 PrevInst->getIterator().getReverse();
2500 while (InstIt != PrevInstIt) {
2501 if (PrevInstIt == PrevInst->getParent()->rend()) {
2502 PrevInstIt = Inst->getParent()->rbegin();
2503 continue;
2504 }
2505
2506 // Debug informations don't impact spill cost.
2507 if ((isa<CallInst>(&*PrevInstIt) &&
2508 !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
2509 &*PrevInstIt != PrevInst) {
2510 SmallVector<Type*, 4> V;
2511 for (auto *II : LiveValues)
2512 V.push_back(VectorType::get(II->getType(), BundleWidth));
2513 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2514 }
2515
2516 ++PrevInstIt;
2517 }
2518
2519 PrevInst = Inst;
2520 }
2521
2522 return Cost;
2523}
2524
2525int BoUpSLP::getTreeCost() {
2526 int Cost = 0;
2527 LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
2528 << VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
2529
2530 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2531
2532 for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
2533 TreeEntry &TE = VectorizableTree[I];
2534
2535 // We create duplicate tree entries for gather sequences that have multiple
2536 // uses. However, we should not compute the cost of duplicate sequences.
2537 // For example, if we have a build vector (i.e., insertelement sequence)
2538 // that is used by more than one vector instruction, we only need to
2539 // compute the cost of the insertelement instructions once. The redundent
2540 // instructions will be eliminated by CSE.
2541 //
2542 // We should consider not creating duplicate tree entries for gather
2543 // sequences, and instead add additional edges to the tree representing
2544 // their uses. Since such an approach results in fewer total entries,
2545 // existing heuristics based on tree size may yeild different results.
2546 //
2547 if (TE.NeedToGather &&
2548 std::any_of(std::next(VectorizableTree.begin(), I + 1),
2549 VectorizableTree.end(), [TE](TreeEntry &Entry) {
2550 return Entry.NeedToGather && Entry.isSame(TE.Scalars);
2551 }))
2552 continue;
2553
2554 int C = getEntryCost(&TE);
2555 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
2556 << " for bundle that starts with " << *TE.Scalars[0]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
2557 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
2558 Cost += C;
2559 }
2560
2561 SmallPtrSet<Value *, 16> ExtractCostCalculated;
2562 int ExtractCost = 0;
2563 for (ExternalUser &EU : ExternalUses) {
2564 // We only add extract cost once for the same scalar.
2565 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2566 continue;
2567
2568 // Uses by ephemeral values are free (because the ephemeral value will be
2569 // removed prior to code generation, and so the extraction will be
2570 // removed as well).
2571 if (EphValues.count(EU.User))
2572 continue;
2573
2574 // If we plan to rewrite the tree in a smaller type, we will need to sign
2575 // extend the extracted value back to the original type. Here, we account
2576 // for the extract and the added cost of the sign extend if needed.
2577 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2578 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2579 if (MinBWs.count(ScalarRoot)) {
2580 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2581 auto Extend =
2582 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2583 VecTy = VectorType::get(MinTy, BundleWidth);
2584 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2585 VecTy, EU.Lane);
2586 } else {
2587 ExtractCost +=
2588 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2589 }
2590 }
2591
2592 int SpillCost = getSpillCost();
2593 Cost += SpillCost + ExtractCost;
2594
2595 std::string Str;
2596 {
2597 raw_string_ostream OS(Str);
2598 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2599 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2600 << "SLP: Total Cost = " << Cost << ".\n";
2601 }
2602 LLVM_DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
2603
2604 if (ViewSLPTree)
2605 ViewGraph(this, "SLP" + F->getName(), false, Str);
2606
2607 return Cost;
2608}
2609
2610int BoUpSLP::getGatherCost(Type *Ty,
2611 const DenseSet<unsigned> &ShuffledIndices) {
2612 int Cost = 0;
2613 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2614 if (!ShuffledIndices.count(i))
2615 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2616 if (!ShuffledIndices.empty())
2617 Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2618 return Cost;
2619}
2620
2621int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2622 // Find the type of the operands in VL.
2623 Type *ScalarTy = VL[0]->getType();
2624 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2625 ScalarTy = SI->getValueOperand()->getType();
2626 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2627 // Find the cost of inserting/extracting values from the vector.
2628 // Check if the same elements are inserted several times and count them as
2629 // shuffle candidates.
2630 DenseSet<unsigned> ShuffledElements;
2631 DenseSet<Value *> UniqueElements;
2632 // Iterate in reverse order to consider insert elements with the high cost.
2633 for (unsigned I = VL.size(); I > 0; --I) {
2634 unsigned Idx = I - 1;
2635 if (!UniqueElements.insert(VL[Idx]).second)
2636 ShuffledElements.insert(Idx);
2637 }
2638 return getGatherCost(VecTy, ShuffledElements);
2639}
2640
2641// Reorder commutative operations in alternate shuffle if the resulting vectors
2642// are consecutive loads. This would allow us to vectorize the tree.
2643// If we have something like-
2644// load a[0] - load b[0]
2645// load b[1] + load a[1]
2646// load a[2] - load b[2]
2647// load a[3] + load b[3]
2648// Reordering the second load b[1] load a[1] would allow us to vectorize this
2649// code.
2650void BoUpSLP::reorderAltShuffleOperands(const InstructionsState &S,
2651 ArrayRef<Value *> VL,
2652 SmallVectorImpl<Value *> &Left,
2653 SmallVectorImpl<Value *> &Right) {
2654 // Push left and right operands of binary operation into Left and Right
2655 for (Value *V : VL) {
2656 auto *I = cast<Instruction>(V);
2657 assert(S.isOpcodeOrAlt(I) && "Incorrect instruction in vector")((S.isOpcodeOrAlt(I) && "Incorrect instruction in vector"
) ? static_cast<void> (0) : __assert_fail ("S.isOpcodeOrAlt(I) && \"Incorrect instruction in vector\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2657, __PRETTY_FUNCTION__))
;
2658 Left.push_back(I->getOperand(0));
2659 Right.push_back(I->getOperand(1));
2660 }
2661
2662 // Reorder if we have a commutative operation and consecutive access
2663 // are on either side of the alternate instructions.
2664 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2665 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2666 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2667 Instruction *VL1 = cast<Instruction>(VL[j]);
2668 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2669 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2670 std::swap(Left[j], Right[j]);
2671 continue;
2672 } else if (VL2->isCommutative() &&
2673 isConsecutiveAccess(L, L1, *DL, *SE)) {
2674 std::swap(Left[j + 1], Right[j + 1]);
2675 continue;
2676 }
2677 // else unchanged
2678 }
2679 }
2680 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2681 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2682 Instruction *VL1 = cast<Instruction>(VL[j]);
2683 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2684 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2685 std::swap(Left[j], Right[j]);
2686 continue;
2687 } else if (VL2->isCommutative() &&
2688 isConsecutiveAccess(L, L1, *DL, *SE)) {
2689 std::swap(Left[j + 1], Right[j + 1]);
2690 continue;
2691 }
2692 // else unchanged
2693 }
2694 }
2695 }
2696}
2697
2698// Return true if I should be commuted before adding it's left and right
2699// operands to the arrays Left and Right.
2700//
2701// The vectorizer is trying to either have all elements one side being
2702// instruction with the same opcode to enable further vectorization, or having
2703// a splat to lower the vectorizing cost.
2704static bool shouldReorderOperands(
2705 int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2706 ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2707 bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2708 VLeft = I.getOperand(0);
2709 VRight = I.getOperand(1);
2710 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2711 if (SplatRight) {
2712 if (VRight == Right[i - 1])
2713 // Preserve SplatRight
2714 return false;
2715 if (VLeft == Right[i - 1]) {
2716 // Commuting would preserve SplatRight, but we don't want to break
2717 // SplatLeft either, i.e. preserve the original order if possible.
2718 // (FIXME: why do we care?)
2719 if (SplatLeft && VLeft == Left[i - 1])
2720 return false;
2721 return true;
2722 }
2723 }
2724 // Symmetrically handle Right side.
2725 if (SplatLeft) {
2726 if (VLeft == Left[i - 1])
2727 // Preserve SplatLeft
2728 return false;
2729 if (VRight == Left[i - 1])
2730 return true;
2731 }
2732
2733 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2734 Instruction *IRight = dyn_cast<Instruction>(VRight);
2735
2736 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2737 // it and not the right, in this case we want to commute.
2738 if (AllSameOpcodeRight) {
2739 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2740 if (IRight && RightPrevOpcode == IRight->getOpcode())
2741 // Do not commute, a match on the right preserves AllSameOpcodeRight
2742 return false;
2743 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2744 // We have a match and may want to commute, but first check if there is
2745 // not also a match on the existing operands on the Left to preserve
2746 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2747 // (FIXME: why do we care?)
2748 if (AllSameOpcodeLeft && ILeft &&
2749 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2750 return false;
2751 return true;
2752 }
2753 }
2754 // Symmetrically handle Left side.
2755 if (AllSameOpcodeLeft) {
2756 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2757 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2758 return false;
2759 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2760 return true;
2761 }
2762 return false;
2763}
2764
2765void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2766 ArrayRef<Value *> VL,
2767 SmallVectorImpl<Value *> &Left,
2768 SmallVectorImpl<Value *> &Right) {
2769 if (!VL.empty()) {
2770 // Peel the first iteration out of the loop since there's nothing
2771 // interesting to do anyway and it simplifies the checks in the loop.
2772 auto *I = cast<Instruction>(VL[0]);
2773 Value *VLeft = I->getOperand(0);
2774 Value *VRight = I->getOperand(1);
2775 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2776 // Favor having instruction to the right. FIXME: why?
2777 std::swap(VLeft, VRight);
2778 Left.push_back(VLeft);
2779 Right.push_back(VRight);
2780 }
2781
2782 // Keep track if we have instructions with all the same opcode on one side.
2783 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2784 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2785 // Keep track if we have one side with all the same value (broadcast).
2786 bool SplatLeft = true;
2787 bool SplatRight = true;
2788
2789 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2790 Instruction *I = cast<Instruction>(VL[i]);
2791 assert(((I->getOpcode() == Opcode && I->isCommutative()) ||((((I->getOpcode() == Opcode && I->isCommutative
()) || (I->getOpcode() != Opcode && Instruction::isCommutative
(Opcode))) && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2793, __PRETTY_FUNCTION__))
2792 (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&((((I->getOpcode() == Opcode && I->isCommutative
()) || (I->getOpcode() != Opcode && Instruction::isCommutative
(Opcode))) && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2793, __PRETTY_FUNCTION__))
2793 "Can only process commutative instruction")((((I->getOpcode() == Opcode && I->isCommutative
()) || (I->getOpcode() != Opcode && Instruction::isCommutative
(Opcode))) && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("((I->getOpcode() == Opcode && I->isCommutative()) || (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) && \"Can only process commutative instruction\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2793, __PRETTY_FUNCTION__))
;
2794 // Commute to favor either a splat or maximizing having the same opcodes on
2795 // one side.
2796 Value *VLeft;
2797 Value *VRight;
2798 if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2799 AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2800 VRight)) {
2801 Left.push_back(VRight);
2802 Right.push_back(VLeft);
2803 } else {
2804 Left.push_back(VLeft);
2805 Right.push_back(VRight);
2806 }
2807 // Update Splat* and AllSameOpcode* after the insertion.
2808 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2809 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2810 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2811 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2812 cast<Instruction>(Left[i])->getOpcode());
2813 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2814 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2815 cast<Instruction>(Right[i])->getOpcode());
2816 }
2817
2818 // If one operand end up being broadcast, return this operand order.
2819 if (SplatRight || SplatLeft)
2820 return;
2821
2822 // Finally check if we can get longer vectorizable chain by reordering
2823 // without breaking the good operand order detected above.
2824 // E.g. If we have something like-
2825 // load a[0] load b[0]
2826 // load b[1] load a[1]
2827 // load a[2] load b[2]
2828 // load a[3] load b[3]
2829 // Reordering the second load b[1] load a[1] would allow us to vectorize
2830 // this code and we still retain AllSameOpcode property.
2831 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2832 // such as-
2833 // add a[0],c[0] load b[0]
2834 // add a[1],c[2] load b[1]
2835 // b[2] load b[2]
2836 // add a[3],c[3] load b[3]
2837 for (unsigned j = 0, e = VL.size() - 1; j < e; ++j) {
2838 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2839 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2840 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2841 std::swap(Left[j + 1], Right[j + 1]);
2842 continue;
2843 }
2844 }
2845 }
2846 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2847 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2848 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2849 std::swap(Left[j + 1], Right[j + 1]);
2850 continue;
2851 }
2852 }
2853 }
2854 // else unchanged
2855 }
2856}
2857
2858void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL,
2859 const InstructionsState &S) {
2860 // Get the basic block this bundle is in. All instructions in the bundle
2861 // should be in this block.
2862 auto *Front = cast<Instruction>(S.OpValue);
2863 auto *BB = Front->getParent();
2864 assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { auto *I = cast<Instruction>(V); return !
S.isOpcodeOrAlt(I) || I->getParent() == BB; })) ? static_cast
<void> (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !S.isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2867, __PRETTY_FUNCTION__))
2865 auto *I = cast<Instruction>(V);((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { auto *I = cast<Instruction>(V); return !
S.isOpcodeOrAlt(I) || I->getParent() == BB; })) ? static_cast
<void> (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !S.isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2867, __PRETTY_FUNCTION__))
2866 return !S.isOpcodeOrAlt(I) || I->getParent() == BB;((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { auto *I = cast<Instruction>(V); return !
S.isOpcodeOrAlt(I) || I->getParent() == BB; })) ? static_cast
<void> (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !S.isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2867, __PRETTY_FUNCTION__))
2867 }))((llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V
) -> bool { auto *I = cast<Instruction>(V); return !
S.isOpcodeOrAlt(I) || I->getParent() == BB; })) ? static_cast
<void> (0) : __assert_fail ("llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool { auto *I = cast<Instruction>(V); return !S.isOpcodeOrAlt(I) || I->getParent() == BB; })"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2867, __PRETTY_FUNCTION__))
;
2868
2869 // The last instruction in the bundle in program order.
2870 Instruction *LastInst = nullptr;
2871
2872 // Find the last instruction. The common case should be that BB has been
2873 // scheduled, and the last instruction is VL.back(). So we start with
2874 // VL.back() and iterate over schedule data until we reach the end of the
2875 // bundle. The end of the bundle is marked by null ScheduleData.
2876 if (BlocksSchedules.count(BB)) {
2877 auto *Bundle =
2878 BlocksSchedules[BB]->getScheduleData(isOneOf(S, VL.back()));
2879 if (Bundle && Bundle->isPartOfBundle())
2880 for (; Bundle; Bundle = Bundle->NextInBundle)
2881 if (Bundle->OpValue == Bundle->Inst)
2882 LastInst = Bundle->Inst;
2883 }
2884
2885 // LastInst can still be null at this point if there's either not an entry
2886 // for BB in BlocksSchedules or there's no ScheduleData available for
2887 // VL.back(). This can be the case if buildTree_rec aborts for various
2888 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2889 // size is reached, etc.). ScheduleData is initialized in the scheduling
2890 // "dry-run".
2891 //
2892 // If this happens, we can still find the last instruction by brute force. We
2893 // iterate forwards from Front (inclusive) until we either see all
2894 // instructions in the bundle or reach the end of the block. If Front is the
2895 // last instruction in program order, LastInst will be set to Front, and we
2896 // will visit all the remaining instructions in the block.
2897 //
2898 // One of the reasons we exit early from buildTree_rec is to place an upper
2899 // bound on compile-time. Thus, taking an additional compile-time hit here is
2900 // not ideal. However, this should be exceedingly rare since it requires that
2901 // we both exit early from buildTree_rec and that the bundle be out-of-order
2902 // (causing us to iterate all the way to the end of the block).
2903 if (!LastInst) {
2904 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2905 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2906 if (Bundle.erase(&I) && S.isOpcodeOrAlt(&I))
2907 LastInst = &I;
2908 if (Bundle.empty())
2909 break;
2910 }
2911 }
2912
2913 // Set the insertion point after the last instruction in the bundle. Set the
2914 // debug location to Front.
2915 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2916 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2917}
2918
2919Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2920 Value *Vec = UndefValue::get(Ty);
2921 // Generate the 'InsertElement' instruction.
2922 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2923 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2924 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2925 GatherSeq.insert(Insrt);
2926 CSEBlocks.insert(Insrt->getParent());
2927
2928 // Add to our 'need-to-extract' list.
2929 if (TreeEntry *E = getTreeEntry(VL[i])) {
2930 // Find which lane we need to extract.
2931 int FoundLane = -1;
2932 for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2933 // Is this the lane of the scalar that we are looking for ?
2934 if (E->Scalars[Lane] == VL[i]) {
2935 FoundLane = Lane;
2936 break;
2937 }
2938 }
2939 assert(FoundLane >= 0 && "Could not find the correct lane")((FoundLane >= 0 && "Could not find the correct lane"
) ? static_cast<void> (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2939, __PRETTY_FUNCTION__))
;
2940 if (!E->ReuseShuffleIndices.empty()) {
2941 FoundLane =
2942 std::distance(E->ReuseShuffleIndices.begin(),
2943 llvm::find(E->ReuseShuffleIndices, FoundLane));
2944 }
2945 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2946 }
2947 }
2948 }
2949
2950 return Vec;
2951}
2952
2953Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2954 InstructionsState S = getSameOpcode(VL);
2955 if (S.getOpcode()) {
2956 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2957 if (E->isSame(VL)) {
2958 Value *V = vectorizeTree(E);
2959 if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
2960 // We need to get the vectorized value but without shuffle.
2961 if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
2962 V = SV->getOperand(0);
2963 } else {
2964 // Reshuffle to get only unique values.
2965 SmallVector<unsigned, 4> UniqueIdxs;
2966 SmallSet<unsigned, 4> UsedIdxs;
2967 for(unsigned Idx : E->ReuseShuffleIndices)
2968 if (UsedIdxs.insert(Idx).second)
2969 UniqueIdxs.emplace_back(Idx);
2970 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
2971 UniqueIdxs);
2972 }
2973 }
2974 return V;
2975 }
2976 }
2977 }
2978
2979 Type *ScalarTy = S.OpValue->getType();
2980 if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2981 ScalarTy = SI->getValueOperand()->getType();
2982
2983 // Check that every instruction appears once in this bundle.
2984 SmallVector<unsigned, 4> ReuseShuffleIndicies;
2985 SmallVector<Value *, 4> UniqueValues;
2986 if (VL.size() > 2) {
2987 DenseMap<Value *, unsigned> UniquePositions;
2988 for (Value *V : VL) {
2989 auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2990 ReuseShuffleIndicies.emplace_back(Res.first->second);
2991 if (Res.second || isa<Constant>(V))
2992 UniqueValues.emplace_back(V);
2993 }
2994 // Do not shuffle single element or if number of unique values is not power
2995 // of 2.
2996 if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
2997 !llvm::isPowerOf2_32(UniqueValues.size()))
2998 ReuseShuffleIndicies.clear();
2999 else
3000 VL = UniqueValues;
3001 }
3002 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3003
3004 Value *V = Gather(VL, VecTy);
3005 if (!ReuseShuffleIndicies.empty()) {
3006 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3007 ReuseShuffleIndicies, "shuffle");
3008 if (auto *I = dyn_cast<Instruction>(V)) {
3009 GatherSeq.insert(I);
3010 CSEBlocks.insert(I->getParent());
3011 }
3012 }
3013 return V;
3014}
3015
3016static void inversePermutation(ArrayRef<unsigned> Indices,
3017 SmallVectorImpl<unsigned> &Mask) {
3018 Mask.clear();
3019 const unsigned E = Indices.size();
3020 Mask.resize(E);
3021 for (unsigned I = 0; I < E; ++I)
3022 Mask[Indices[I]] = I;
3023}
3024
3025Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
3026 IRBuilder<>::InsertPointGuard Guard(Builder);
3027
3028 if (E->VectorizedValue) {
3029 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
3030 return E->VectorizedValue;
3031 }
3032
3033 InstructionsState S = getSameOpcode(E->Scalars);
3034 Instruction *VL0 = cast<Instruction>(S.OpValue);
3035 Type *ScalarTy = VL0->getType();
3036 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3037 ScalarTy = SI->getValueOperand()->getType();
3038 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
3039
3040 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3041
3042 if (E->NeedToGather) {
3043 setInsertPointAfterBundle(E->Scalars, S);
3044 auto *V = Gather(E->Scalars, VecTy);
3045 if (NeedToShuffleReuses) {
3046 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3047 E->ReuseShuffleIndices, "shuffle");
3048 if (auto *I = dyn_cast<Instruction>(V)) {
3049 GatherSeq.insert(I);
3050 CSEBlocks.insert(I->getParent());
3051 }
3052 }
3053 E->VectorizedValue = V;
3054 return V;
3055 }
3056
3057 unsigned ShuffleOrOp = S.isAltShuffle() ?
3058 (unsigned) Instruction::ShuffleVector : S.getOpcode();
3059 switch (ShuffleOrOp) {
3060 case Instruction::PHI: {
3061 PHINode *PH = dyn_cast<PHINode>(VL0);
3062 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
3063 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3064 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
3065 Value *V = NewPhi;
3066 if (NeedToShuffleReuses) {
3067 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3068 E->ReuseShuffleIndices, "shuffle");
3069 }
3070 E->VectorizedValue = V;
3071
3072 // PHINodes may have multiple entries from the same block. We want to
3073 // visit every block once.
3074 SmallPtrSet<BasicBlock*, 4> VisitedBBs;
3075
3076 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
3077 ValueList Operands;
3078 BasicBlock *IBB = PH->getIncomingBlock(i);
3079
3080 if (!VisitedBBs.insert(IBB).second) {
3081 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
3082 continue;
3083 }
3084
3085 // Prepare the operand vector.
3086 for (Value *V : E->Scalars)
3087 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
3088
3089 Builder.SetInsertPoint(IBB->getTerminator());
3090 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3091 Value *Vec = vectorizeTree(Operands);
3092 NewPhi->addIncoming(Vec, IBB);
3093 }
3094
3095 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3096, __PRETTY_FUNCTION__))
3096 "Invalid number of incoming values")((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3096, __PRETTY_FUNCTION__))
;
3097 return V;
3098 }
3099
3100 case Instruction::ExtractElement: {
3101 if (!E->NeedToGather) {
3102 Value *V = VL0->getOperand(0);
3103 if (!E->ReorderIndices.empty()) {
3104 OrdersType Mask;
3105 inversePermutation(E->ReorderIndices, Mask);
3106 Builder.SetInsertPoint(VL0);
3107 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
3108 "reorder_shuffle");
3109 }
3110 if (NeedToShuffleReuses) {
3111 // TODO: Merge this shuffle with the ReorderShuffleMask.
3112 if (E->ReorderIndices.empty())
3113 Builder.SetInsertPoint(VL0);
3114 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3115 E->ReuseShuffleIndices, "shuffle");
3116 }
3117 E->VectorizedValue = V;
3118 return V;
3119 }
3120 setInsertPointAfterBundle(E->Scalars, S);
3121 auto *V = Gather(E->Scalars, VecTy);
3122 if (NeedToShuffleReuses) {
3123 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3124 E->ReuseShuffleIndices, "shuffle");
3125 if (auto *I = dyn_cast<Instruction>(V)) {
3126 GatherSeq.insert(I);
3127 CSEBlocks.insert(I->getParent());
3128 }
3129 }
3130 E->VectorizedValue = V;
3131 return V;
3132 }
3133 case Instruction::ExtractValue: {
3134 if (!E->NeedToGather) {
3135 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3136 Builder.SetInsertPoint(LI);
3137 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3138 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3139 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
3140 Value *NewV = propagateMetadata(V, E->Scalars);
3141 if (!E->ReorderIndices.empty()) {
3142 OrdersType Mask;
3143 inversePermutation(E->ReorderIndices, Mask);
3144 NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
3145 "reorder_shuffle");
3146 }
3147 if (NeedToShuffleReuses) {
3148 // TODO: Merge this shuffle with the ReorderShuffleMask.
3149 NewV = Builder.CreateShuffleVector(
3150 NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3151 }
3152 E->VectorizedValue = NewV;
3153 return NewV;
3154 }
3155 setInsertPointAfterBundle(E->Scalars, S);
3156 auto *V = Gather(E->Scalars, VecTy);
3157 if (NeedToShuffleReuses) {
3158 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3159 E->ReuseShuffleIndices, "shuffle");
3160 if (auto *I = dyn_cast<Instruction>(V)) {
3161 GatherSeq.insert(I);
3162 CSEBlocks.insert(I->getParent());
3163 }
3164 }
3165 E->VectorizedValue = V;
3166 return V;
3167 }
3168 case Instruction::ZExt:
3169 case Instruction::SExt:
3170 case Instruction::FPToUI:
3171 case Instruction::FPToSI:
3172 case Instruction::FPExt:
3173 case Instruction::PtrToInt:
3174 case Instruction::IntToPtr:
3175 case Instruction::SIToFP:
3176 case Instruction::UIToFP:
3177 case Instruction::Trunc:
3178 case Instruction::FPTrunc:
3179 case Instruction::BitCast: {
3180 ValueList INVL;
3181 for (Value *V : E->Scalars)
3182 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3183
3184 setInsertPointAfterBundle(E->Scalars, S);
3185
3186 Value *InVec = vectorizeTree(INVL);
3187
3188 if (E->VectorizedValue) {
3189 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3190 return E->VectorizedValue;
3191 }
3192
3193 CastInst *CI = dyn_cast<CastInst>(VL0);
3194 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3195 if (NeedToShuffleReuses) {
3196 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3197 E->ReuseShuffleIndices, "shuffle");
3198 }
3199 E->VectorizedValue = V;
3200 ++NumVectorInstructions;
3201 return V;
3202 }
3203 case Instruction::FCmp:
3204 case Instruction::ICmp: {
3205 ValueList LHSV, RHSV;
3206 for (Value *V : E->Scalars) {
3207 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3208 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3209 }
3210
3211 setInsertPointAfterBundle(E->Scalars, S);
3212
3213 Value *L = vectorizeTree(LHSV);
3214 Value *R = vectorizeTree(RHSV);
3215
3216 if (E->VectorizedValue) {
3217 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3218 return E->VectorizedValue;
3219 }
3220
3221 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3222 Value *V;
3223 if (S.getOpcode() == Instruction::FCmp)
3224 V = Builder.CreateFCmp(P0, L, R);
3225 else
3226 V = Builder.CreateICmp(P0, L, R);
3227
3228 propagateIRFlags(V, E->Scalars, VL0);
3229 if (NeedToShuffleReuses) {
3230 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3231 E->ReuseShuffleIndices, "shuffle");
3232 }
3233 E->VectorizedValue = V;
3234 ++NumVectorInstructions;
3235 return V;
3236 }
3237 case Instruction::Select: {
3238 ValueList TrueVec, FalseVec, CondVec;
3239 for (Value *V : E->Scalars) {
3240 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3241 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3242 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3243 }
3244
3245 setInsertPointAfterBundle(E->Scalars, S);
3246
3247 Value *Cond = vectorizeTree(CondVec);
3248 Value *True = vectorizeTree(TrueVec);
3249 Value *False = vectorizeTree(FalseVec);
3250
3251 if (E->VectorizedValue) {
3252 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3253 return E->VectorizedValue;
3254 }
3255
3256 Value *V = Builder.CreateSelect(Cond, True, False);
3257 if (NeedToShuffleReuses) {
3258 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3259 E->ReuseShuffleIndices, "shuffle");
3260 }
3261 E->VectorizedValue = V;
3262 ++NumVectorInstructions;
3263 return V;
3264 }
3265 case Instruction::Add:
3266 case Instruction::FAdd:
3267 case Instruction::Sub:
3268 case Instruction::FSub:
3269 case Instruction::Mul:
3270 case Instruction::FMul:
3271 case Instruction::UDiv:
3272 case Instruction::SDiv:
3273 case Instruction::FDiv:
3274 case Instruction::URem:
3275 case Instruction::SRem:
3276 case Instruction::FRem:
3277 case Instruction::Shl:
3278 case Instruction::LShr:
3279 case Instruction::AShr:
3280 case Instruction::And:
3281 case Instruction::Or:
3282 case Instruction::Xor: {
3283 ValueList LHSVL, RHSVL;
3284 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3285 reorderInputsAccordingToOpcode(S.getOpcode(), E->Scalars, LHSVL,
3286 RHSVL);
3287 else
3288 for (Value *V : E->Scalars) {
3289 auto *I = cast<Instruction>(V);
3290 LHSVL.push_back(I->getOperand(0));
3291 RHSVL.push_back(I->getOperand(1));
3292 }
3293
3294 setInsertPointAfterBundle(E->Scalars, S);
3295
3296 Value *LHS = vectorizeTree(LHSVL);
3297 Value *RHS = vectorizeTree(RHSVL);
3298
3299 if (E->VectorizedValue) {
3300 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3301 return E->VectorizedValue;
3302 }
3303
3304 Value *V = Builder.CreateBinOp(
3305 static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3306 propagateIRFlags(V, E->Scalars, VL0);
3307 if (auto *I = dyn_cast<Instruction>(V))
3308 V = propagateMetadata(I, E->Scalars);
3309
3310 if (NeedToShuffleReuses) {
3311 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3312 E->ReuseShuffleIndices, "shuffle");
3313 }
3314 E->VectorizedValue = V;
3315 ++NumVectorInstructions;
3316
3317 return V;
3318 }
3319 case Instruction::Load: {
3320 // Loads are inserted at the head of the tree because we don't want to
3321 // sink them all the way down past store instructions.
3322 bool IsReorder = !E->ReorderIndices.empty();
3323 if (IsReorder) {
3324 S = getSameOpcode(E->Scalars, E->ReorderIndices.front());
3325 VL0 = cast<Instruction>(S.OpValue);
3326 }
3327 setInsertPointAfterBundle(E->Scalars, S);
3328
3329 LoadInst *LI = cast<LoadInst>(VL0);
3330 Type *ScalarLoadTy = LI->getType();
3331 unsigned AS = LI->getPointerAddressSpace();
3332
3333 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3334 VecTy->getPointerTo(AS));
3335
3336 // The pointer operand uses an in-tree scalar so we add the new BitCast to
3337 // ExternalUses list to make sure that an extract will be generated in the
3338 // future.
3339 Value *PO = LI->getPointerOperand();
3340 if (getTreeEntry(PO))
3341 ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3342
3343 unsigned Alignment = LI->getAlignment();
3344 LI = Builder.CreateLoad(VecPtr);
3345 if (!Alignment) {
3346 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3347 }
3348 LI->setAlignment(Alignment);
3349 Value *V = propagateMetadata(LI, E->Scalars);
3350 if (IsReorder) {
3351 OrdersType Mask;
3352 inversePermutation(E->ReorderIndices, Mask);
3353 V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3354 Mask, "reorder_shuffle");
3355 }
3356 if (NeedToShuffleReuses) {
3357 // TODO: Merge this shuffle with the ReorderShuffleMask.
3358 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3359 E->ReuseShuffleIndices, "shuffle");
3360 }
3361 E->VectorizedValue = V;
3362 ++NumVectorInstructions;
3363 return V;
3364 }
3365 case Instruction::Store: {
3366 StoreInst *SI = cast<StoreInst>(VL0);
3367 unsigned Alignment = SI->getAlignment();
3368 unsigned AS = SI->getPointerAddressSpace();
3369
3370 ValueList ScalarStoreValues;
3371 for (Value *V : E->Scalars)
3372 ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3373
3374 setInsertPointAfterBundle(E->Scalars, S);
3375
3376 Value *VecValue = vectorizeTree(ScalarStoreValues);
3377 Value *ScalarPtr = SI->getPointerOperand();
3378 Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3379 StoreInst *ST = Builder.CreateStore(VecValue, VecPtr);
3380
3381 // The pointer operand uses an in-tree scalar, so add the new BitCast to
3382 // ExternalUses to make sure that an extract will be generated in the
3383 // future.
3384 if (getTreeEntry(ScalarPtr))
3385 ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3386
3387 if (!Alignment)
3388 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3389
3390 ST->setAlignment(Alignment);
3391 Value *V = propagateMetadata(ST, E->Scalars);
3392 if (NeedToShuffleReuses) {
3393 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3394 E->ReuseShuffleIndices, "shuffle");
3395 }
3396 E->VectorizedValue = V;
3397 ++NumVectorInstructions;
3398 return V;
3399 }
3400 case Instruction::GetElementPtr: {
3401 setInsertPointAfterBundle(E->Scalars, S);
3402
3403 ValueList Op0VL;
3404 for (Value *V : E->Scalars)
3405 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3406
3407 Value *Op0 = vectorizeTree(Op0VL);
3408
3409 std::vector<Value *> OpVecs;
3410 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3411 ++j) {
3412 ValueList OpVL;
3413 for (Value *V : E->Scalars)
3414 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3415
3416 Value *OpVec = vectorizeTree(OpVL);
3417 OpVecs.push_back(OpVec);
3418 }
3419
3420 Value *V = Builder.CreateGEP(
3421 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3422 if (Instruction *I = dyn_cast<Instruction>(V))
3423 V = propagateMetadata(I, E->Scalars);
3424
3425 if (NeedToShuffleReuses) {
3426 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3427 E->ReuseShuffleIndices, "shuffle");
3428 }
3429 E->VectorizedValue = V;
3430 ++NumVectorInstructions;
3431
3432 return V;
3433 }
3434 case Instruction::Call: {
3435 CallInst *CI = cast<CallInst>(VL0);
3436 setInsertPointAfterBundle(E->Scalars, S);
3437 Function *FI;
3438 Intrinsic::ID IID = Intrinsic::not_intrinsic;
3439 Value *ScalarArg = nullptr;
3440 if (CI && (FI = CI->getCalledFunction())) {
3441 IID = FI->getIntrinsicID();
3442 }
3443 std::vector<Value *> OpVecs;
3444 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3445 ValueList OpVL;
3446 // ctlz,cttz and powi are special intrinsics whose second argument is
3447 // a scalar. This argument should not be vectorized.
3448 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3449 CallInst *CEI = cast<CallInst>(VL0);
3450 ScalarArg = CEI->getArgOperand(j);
3451 OpVecs.push_back(CEI->getArgOperand(j));
3452 continue;
3453 }
3454 for (Value *V : E->Scalars) {
3455 CallInst *CEI = cast<CallInst>(V);
3456 OpVL.push_back(CEI->getArgOperand(j));
3457 }
3458
3459 Value *OpVec = vectorizeTree(OpVL);
3460 LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
3461 OpVecs.push_back(OpVec);
3462 }
3463
3464 Module *M = F->getParent();
3465 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
3466 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3467 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3468 SmallVector<OperandBundleDef, 1> OpBundles;
3469 CI->getOperandBundlesAsDefs(OpBundles);
3470 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3471
3472 // The scalar argument uses an in-tree scalar so we add the new vectorized
3473 // call to ExternalUses list to make sure that an extract will be
3474 // generated in the future.
3475 if (ScalarArg && getTreeEntry(ScalarArg))
3476 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3477
3478 propagateIRFlags(V, E->Scalars, VL0);
3479 if (NeedToShuffleReuses) {
3480 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3481 E->ReuseShuffleIndices, "shuffle");
3482 }
3483 E->VectorizedValue = V;
3484 ++NumVectorInstructions;
3485 return V;
3486 }
3487 case Instruction::ShuffleVector: {
3488 ValueList LHSVL, RHSVL;
3489 assert(S.isAltShuffle() &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3494, __PRETTY_FUNCTION__))
3490 ((Instruction::isBinaryOp(S.getOpcode()) &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3494, __PRETTY_FUNCTION__))
3491 Instruction::isBinaryOp(S.getAltOpcode())) ||((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3494, __PRETTY_FUNCTION__))
3492 (Instruction::isCast(S.getOpcode()) &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3494, __PRETTY_FUNCTION__))
3493 Instruction::isCast(S.getAltOpcode()))) &&((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3494, __PRETTY_FUNCTION__))
3494 "Invalid Shuffle Vector Operand")((S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode
()) && Instruction::isBinaryOp(S.getAltOpcode())) || (
Instruction::isCast(S.getOpcode()) && Instruction::isCast
(S.getAltOpcode()))) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("S.isAltShuffle() && ((Instruction::isBinaryOp(S.getOpcode()) && Instruction::isBinaryOp(S.getAltOpcode())) || (Instruction::isCast(S.getOpcode()) && Instruction::isCast(S.getAltOpcode()))) && \"Invalid Shuffle Vector Operand\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3494, __PRETTY_FUNCTION__))
;
3495
3496 Value *LHS, *RHS;
3497 if (Instruction::isBinaryOp(S.getOpcode())) {
3498 reorderAltShuffleOperands(S, E->Scalars, LHSVL, RHSVL);
3499 setInsertPointAfterBundle(E->Scalars, S);
3500 LHS = vectorizeTree(LHSVL);
3501 RHS = vectorizeTree(RHSVL);
3502 } else {
3503 ValueList INVL;
3504 for (Value *V : E->Scalars)
3505 INVL.push_back(cast<Instruction>(V)->getOperand(0));
3506 setInsertPointAfterBundle(E->Scalars, S);
3507 LHS = vectorizeTree(INVL);
3508 }
3509
3510 if (E->VectorizedValue) {
3511 LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*VL0 << ".\n"; } } while (false)
;
3512 return E->VectorizedValue;
3513 }
3514
3515 Value *V0, *V1;
3516 if (Instruction::isBinaryOp(S.getOpcode())) {
3517 V0 = Builder.CreateBinOp(
3518 static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3519 V1 = Builder.CreateBinOp(
3520 static_cast<Instruction::BinaryOps>(S.getAltOpcode()), LHS, RHS);
3521 } else {
3522 V0 = Builder.CreateCast(
3523 static_cast<Instruction::CastOps>(S.getOpcode()), LHS, VecTy);
3524 V1 = Builder.CreateCast(
3525 static_cast<Instruction::CastOps>(S.getAltOpcode()), LHS, VecTy);
3526 }
3527
3528 // Create shuffle to take alternate operations from the vector.
3529 // Also, gather up main and alt scalar ops to propagate IR flags to
3530 // each vector operation.
3531 ValueList OpScalars, AltScalars;
3532 unsigned e = E->Scalars.size();
3533 SmallVector<Constant *, 8> Mask(e);
3534 for (unsigned i = 0; i < e; ++i) {
3535 auto *OpInst = cast<Instruction>(E->Scalars[i]);
3536 assert(S.isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode")((S.isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode"
) ? static_cast<void> (0) : __assert_fail ("S.isOpcodeOrAlt(OpInst) && \"Unexpected main/alternate opcode\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3536, __PRETTY_FUNCTION__))
;
3537 if (OpInst->getOpcode() == S.getAltOpcode()) {
3538 Mask[i] = Builder.getInt32(e + i);
3539 AltScalars.push_back(E->Scalars[i]);
3540 } else {
3541 Mask[i] = Builder.getInt32(i);
3542 OpScalars.push_back(E->Scalars[i]);
3543 }
3544 }
3545
3546 Value *ShuffleMask = ConstantVector::get(Mask);
3547 propagateIRFlags(V0, OpScalars);
3548 propagateIRFlags(V1, AltScalars);
3549
3550 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3551 if (Instruction *I = dyn_cast<Instruction>(V))
3552 V = propagateMetadata(I, E->Scalars);
3553 if (NeedToShuffleReuses) {
3554 V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3555 E->ReuseShuffleIndices, "shuffle");
3556 }
3557 E->VectorizedValue = V;
3558 ++NumVectorInstructions;
3559
3560 return V;
3561 }
3562 default:
3563 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3563)
;
3564 }
3565 return nullptr;
3566}
3567
3568Value *BoUpSLP::vectorizeTree() {
3569 ExtraValueToDebugLocsMap ExternallyUsedValues;
3570 return vectorizeTree(ExternallyUsedValues);
3571}
3572
3573Value *
3574BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3575 // All blocks must be scheduled before any instructions are inserted.
3576 for (auto &BSIter : BlocksSchedules) {
3577 scheduleBlock(BSIter.second.get());
3578 }
3579
3580 Builder.SetInsertPoint(&F->getEntryBlock().front());
3581 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3582
3583 // If the vectorized tree can be rewritten in a smaller type, we truncate the
3584 // vectorized root. InstCombine will then rewrite the entire expression. We
3585 // sign extend the extracted values below.
3586 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3587 if (MinBWs.count(ScalarRoot)) {
3588 if (auto *I = dyn_cast<Instruction>(VectorRoot))
3589 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3590 auto BundleWidth = VectorizableTree[0].Scalars.size();
3591 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3592 auto *VecTy = VectorType::get(MinTy, BundleWidth);
3593 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3594 VectorizableTree[0].VectorizedValue = Trunc;
3595 }
3596
3597 LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
3598 << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
3599
3600 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3601 // specified by ScalarType.
3602 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3603 if (!MinBWs.count(ScalarRoot))
3604 return Ex;
3605 if (MinBWs[ScalarRoot].second)
3606 return Builder.CreateSExt(Ex, ScalarType);
3607 return Builder.CreateZExt(Ex, ScalarType);
3608 };
3609
3610 // Extract all of the elements with the external uses.
3611 for (const auto &ExternalUse : ExternalUses) {
3612 Value *Scalar = ExternalUse.Scalar;
3613 llvm::User *User = ExternalUse.User;
3614
3615 // Skip users that we already RAUW. This happens when one instruction
3616 // has multiple uses of the same value.
3617 if (User && !is_contained(Scalar->users(), User))
3618 continue;
3619 TreeEntry *E = getTreeEntry(Scalar);
3620 assert(E && "Invalid scalar")((E && "Invalid scalar") ? static_cast<void> (0
) : __assert_fail ("E && \"Invalid scalar\"", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3620, __PRETTY_FUNCTION__))
;
3621 assert(!E->NeedToGather && "Extracting from a gather list")((!E->NeedToGather && "Extracting from a gather list"
) ? static_cast<void> (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3621, __PRETTY_FUNCTION__))
;
3622
3623 Value *Vec = E->VectorizedValue;
3624 assert(Vec && "Can't find vectorizable value")((Vec && "Can't find vectorizable value") ? static_cast
<void> (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3624, __PRETTY_FUNCTION__))
;
3625
3626 Value *Lane = Builder.getInt32(ExternalUse.Lane);
3627 // If User == nullptr, the Scalar is used as extra arg. Generate
3628 // ExtractElement instruction and update the record for this scalar in
3629 // ExternallyUsedValues.
3630 if (!User) {
3631 assert(ExternallyUsedValues.count(Scalar) &&((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3633, __PRETTY_FUNCTION__))
3632 "Scalar with nullptr as an external user must be registered in "((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3633, __PRETTY_FUNCTION__))
3633 "ExternallyUsedValues map")((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3633, __PRETTY_FUNCTION__))
;
3634 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3635 Builder.SetInsertPoint(VecI->getParent(),
3636 std::next(VecI->getIterator()));
3637 } else {
3638 Builder.SetInsertPoint(&F->getEntryBlock().front());
3639 }
3640 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3641 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3642 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3643 auto &Locs = ExternallyUsedValues[Scalar];
3644 ExternallyUsedValues.insert({Ex, Locs});
3645 ExternallyUsedValues.erase(Scalar);
3646 // Required to update internally referenced instructions.
3647 Scalar->replaceAllUsesWith(Ex);
3648 continue;
3649 }
3650
3651 // Generate extracts for out-of-tree users.
3652 // Find the insertion point for the extractelement lane.
3653 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3654 if (PHINode *PH = dyn_cast<PHINode>(User)) {
3655 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3656 if (PH->getIncomingValue(i) == Scalar) {
3657 Instruction *IncomingTerminator =
3658 PH->getIncomingBlock(i)->getTerminator();
3659 if (isa<CatchSwitchInst>(IncomingTerminator)) {
3660 Builder.SetInsertPoint(VecI->getParent(),
3661 std::next(VecI->getIterator()));
3662 } else {
3663 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3664 }
3665 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3666 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3667 CSEBlocks.insert(PH->getIncomingBlock(i));
3668 PH->setOperand(i, Ex);
3669 }
3670 }
3671 } else {
3672 Builder.SetInsertPoint(cast<Instruction>(User));
3673 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3674 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3675 CSEBlocks.insert(cast<Instruction>(User)->getParent());
3676 User->replaceUsesOfWith(Scalar, Ex);
3677 }
3678 } else {
3679 Builder.SetInsertPoint(&F->getEntryBlock().front());
3680 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3681 Ex = extend(ScalarRoot, Ex, Scalar->getType());
3682 CSEBlocks.insert(&F->getEntryBlock());
3683 User->replaceUsesOfWith(Scalar, Ex);
3684 }
3685
3686 LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
3687 }
3688
3689 // For each vectorized value:
3690 for (TreeEntry &EIdx : VectorizableTree) {
3691 TreeEntry *Entry = &EIdx;
3692
3693 // No need to handle users of gathered values.
3694 if (Entry->NeedToGather)
3695 continue;
3696
3697 assert(Entry->VectorizedValue && "Can't find vectorizable value")((Entry->VectorizedValue && "Can't find vectorizable value"
) ? static_cast<void> (0) : __assert_fail ("Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3697, __PRETTY_FUNCTION__))
;
3698
3699 // For each lane:
3700 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3701 Value *Scalar = Entry->Scalars[Lane];
3702
3703 Type *Ty = Scalar->getType();
3704 if (!Ty->isVoidTy()) {
3705#ifndef NDEBUG
3706 for (User *U : Scalar->users()) {
3707 LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
3708
3709 // It is legal to replace users in the ignorelist by undef.
3710 assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&(((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
"Replacing out-of-tree value with undef") ? static_cast<void
> (0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3711, __PRETTY_FUNCTION__))
3711 "Replacing out-of-tree value with undef")(((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
"Replacing out-of-tree value with undef") ? static_cast<void
> (0) : __assert_fail ("(getTreeEntry(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3711, __PRETTY_FUNCTION__))
;
3712 }
3713#endif
3714 Value *Undef = UndefValue::get(Ty);
3715 Scalar->replaceAllUsesWith(Undef);
3716 }
3717 LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
3718 eraseInstruction(cast<Instruction>(Scalar));
3719 }
3720 }
3721
3722 Builder.ClearInsertionPoint();
3723
3724 return VectorizableTree[0].VectorizedValue;
3725}
3726
3727void BoUpSLP::optimizeGatherSequence() {
3728 LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
3729 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
3730 // LICM InsertElementInst sequences.
3731 for (Instruction *I : GatherSeq) {
3732 if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3733 continue;
3734
3735 // Check if this block is inside a loop.
3736 Loop *L = LI->getLoopFor(I->getParent());
3737 if (!L)
3738 continue;
3739
3740 // Check if it has a preheader.
3741 BasicBlock *PreHeader = L->getLoopPreheader();
3742 if (!PreHeader)
3743 continue;
3744
3745 // If the vector or the element that we insert into it are
3746 // instructions that are defined in this basic block then we can't
3747 // hoist this instruction.
3748 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3749 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3750 if (Op0 && L->contains(Op0))
3751 continue;
3752 if (Op1 && L->contains(Op1))
3753 continue;
3754
3755 // We can hoist this instruction. Move it to the pre-header.
3756 I->moveBefore(PreHeader->getTerminator());
3757 }
3758
3759 // Make a list of all reachable blocks in our CSE queue.
3760 SmallVector<const DomTreeNode *, 8> CSEWorkList;
3761 CSEWorkList.reserve(CSEBlocks.size());
3762 for (BasicBlock *BB : CSEBlocks)
3763 if (DomTreeNode *N = DT->getNode(BB)) {
3764 assert(DT->isReachableFromEntry(N))((DT->isReachableFromEntry(N)) ? static_cast<void> (
0) : __assert_fail ("DT->isReachableFromEntry(N)", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3764, __PRETTY_FUNCTION__))
;
3765 CSEWorkList.push_back(N);
3766 }
3767
3768 // Sort blocks by domination. This ensures we visit a block after all blocks
3769 // dominating it are visited.
3770 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3771 [this](const DomTreeNode *A, const DomTreeNode *B) {
3772 return DT->properlyDominates(A, B);
3773 });
3774
3775 // Perform O(N^2) search over the gather sequences and merge identical
3776 // instructions. TODO: We can further optimize this scan if we split the
3777 // instructions into different buckets based on the insert lane.
3778 SmallVector<Instruction *, 16> Visited;
3779 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3780 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3781, __PRETTY_FUNCTION__))
3781 "Worklist not sorted properly!")(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3781, __PRETTY_FUNCTION__))
;
3782 BasicBlock *BB = (*I)->getBlock();
3783 // For all instructions in blocks containing gather sequences:
3784 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3785 Instruction *In = &*it++;
3786 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3787 continue;
3788
3789 // Check if we can replace this instruction with any of the
3790 // visited instructions.
3791 for (Instruction *v : Visited) {
3792 if (In->isIdenticalTo(v) &&
3793 DT->dominates(v->getParent(), In->getParent())) {
3794 In->replaceAllUsesWith(v);
3795 eraseInstruction(In);
3796 In = nullptr;
3797 break;
3798 }
3799 }
3800 if (In) {
3801 assert(!is_contained(Visited, In))((!is_contained(Visited, In)) ? static_cast<void> (0) :
__assert_fail ("!is_contained(Visited, In)", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3801, __PRETTY_FUNCTION__))
;
3802 Visited.push_back(In);
3803 }
3804 }
3805 }
3806 CSEBlocks.clear();
3807 GatherSeq.clear();
3808}
3809
3810// Groups the instructions to a bundle (which is then a single scheduling entity)
3811// and schedules instructions until the bundle gets ready.
3812bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3813 BoUpSLP *SLP,
3814 const InstructionsState &S) {
3815 if (isa<PHINode>(S.OpValue))
26
Taking false branch
3816 return true;
3817
3818 // Initialize the instruction bundle.
3819 Instruction *OldScheduleEnd = ScheduleEnd;
3820 ScheduleData *PrevInBundle = nullptr;
3821 ScheduleData *Bundle = nullptr;
27
'Bundle' initialized to a null pointer value
3822 bool ReSchedule = false;
3823 LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *S.OpValue
<< "\n"; } } while (false)
;
28
Assuming 'DebugFlag' is 0
29
Loop condition is false. Exiting loop
3824
3825 // Make sure that the scheduling region contains all
3826 // instructions of the bundle.
3827 for (Value *V : VL) {
30
Assuming '__begin1' is equal to '__end1'
3828 if (!extendSchedulingRegion(V, S))
3829 return false;
3830 }
3831
3832 for (Value *V : VL) {
31
Assuming '__begin1' is equal to '__end1'
3833 ScheduleData *BundleMember = getScheduleData(V);
3834 assert(BundleMember &&((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3835, __PRETTY_FUNCTION__))
3835 "no ScheduleData for bundle member (maybe not in same basic block)")((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3835, __PRETTY_FUNCTION__))
;
3836 if (BundleMember->IsScheduled) {
3837 // A bundle member was scheduled as single instruction before and now
3838 // needs to be scheduled as part of the bundle. We just get rid of the
3839 // existing schedule.
3840 LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3841 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
3842 ReSchedule = true;
3843 }
3844 assert(BundleMember->isSchedulingEntity() &&((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3845, __PRETTY_FUNCTION__))
3845 "bundle member already part of other bundle")((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3845, __PRETTY_FUNCTION__))
;
3846 if (PrevInBundle) {
3847 PrevInBundle->NextInBundle = BundleMember;
3848 } else {
3849 Bundle = BundleMember;
3850 }
3851 BundleMember->UnscheduledDepsInBundle = 0;
3852 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3853
3854 // Group the instructions to a bundle.
3855 BundleMember->FirstInBundle = Bundle;
3856 PrevInBundle = BundleMember;
3857 }
3858 if (ScheduleEnd != OldScheduleEnd) {
32
Taking false branch
3859 // The scheduling region got new instructions at the lower end (or it is a
3860 // new region for the first bundle). This makes it necessary to
3861 // recalculate all dependencies.
3862 // It is seldom that this needs to be done a second time after adding the
3863 // initial bundle to the region.
3864 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3865 doForAllOpcodes(I, [](ScheduleData *SD) {
3866 SD->clearDependencies();
3867 });
3868 }
3869 ReSchedule = true;
3870 }
3871 if (ReSchedule) {
33
Taking false branch
3872 resetSchedule();
3873 initialFillReadyList(ReadyInsts);
3874 }
3875
3876 LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
34
Assuming 'DebugFlag' is 0
35
Loop condition is false. Exiting loop
3877 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
3878
3879 calculateDependencies(Bundle, true, SLP);
3880
3881 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3882 // means that there are no cyclic dependencies and we can schedule it.
3883 // Note that's important that we don't "schedule" the bundle yet (see
3884 // cancelScheduling).
3885 while (!Bundle->isReady() && !ReadyInsts.empty()) {
36
Called C++ object pointer is null
3886
3887 ScheduleData *pickedSD = ReadyInsts.back();
3888 ReadyInsts.pop_back();
3889
3890 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3891 schedule(pickedSD, ReadyInsts);
3892 }
3893 }
3894 if (!Bundle->isReady()) {
3895 cancelScheduling(VL, S.OpValue);
3896 return false;
3897 }
3898 return true;
3899}
3900
3901void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3902 Value *OpValue) {
3903 if (isa<PHINode>(OpValue))
3904 return;
3905
3906 ScheduleData *Bundle = getScheduleData(OpValue);
3907 LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3908 assert(!Bundle->IsScheduled &&((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3909, __PRETTY_FUNCTION__))
3909 "Can't cancel bundle which is already scheduled")((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3909, __PRETTY_FUNCTION__))
;
3910 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3911, __PRETTY_FUNCTION__))
3911 "tried to unbundle something which is not a bundle")((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3911, __PRETTY_FUNCTION__))
;
3912
3913 // Un-bundle: make single instructions out of the bundle.
3914 ScheduleData *BundleMember = Bundle;
3915 while (BundleMember) {
3916 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")((BundleMember->FirstInBundle == Bundle && "corrupt bundle links"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3916, __PRETTY_FUNCTION__))
;
3917 BundleMember->FirstInBundle = BundleMember;
3918 ScheduleData *Next = BundleMember->NextInBundle;
3919 BundleMember->NextInBundle = nullptr;
3920 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3921 if (BundleMember->UnscheduledDepsInBundle == 0) {
3922 ReadyInsts.insert(BundleMember);
3923 }
3924 BundleMember = Next;
3925 }
3926}
3927
3928BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3929 // Allocate a new ScheduleData for the instruction.
3930 if (ChunkPos >= ChunkSize) {
3931 ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3932 ChunkPos = 0;
3933 }
3934 return &(ScheduleDataChunks.back()[ChunkPos++]);
3935}
3936
3937bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3938 const InstructionsState &S) {
3939 if (getScheduleData(V, isOneOf(S, V)))
3940 return true;
3941 Instruction *I = dyn_cast<Instruction>(V);
3942 assert(I && "bundle member must be an instruction")((I && "bundle member must be an instruction") ? static_cast
<void> (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3942, __PRETTY_FUNCTION__))
;
3943 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")((!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3943, __PRETTY_FUNCTION__))
;
3944 auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool {
3945 ScheduleData *ISD = getScheduleData(I);
3946 if (!ISD)
3947 return false;
3948 assert(isInSchedulingRegion(ISD) &&((isInSchedulingRegion(ISD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3949, __PRETTY_FUNCTION__))
3949 "ScheduleData not in scheduling region")((isInSchedulingRegion(ISD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(ISD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3949, __PRETTY_FUNCTION__))
;
3950 ScheduleData *SD = allocateScheduleDataChunks();
3951 SD->Inst = I;
3952 SD->init(SchedulingRegionID, S.OpValue);
3953 ExtraScheduleDataMap[I][S.OpValue] = SD;
3954 return true;
3955 };
3956 if (CheckSheduleForI(I))
3957 return true;
3958 if (!ScheduleStart) {
3959 // It's the first instruction in the new region.
3960 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3961 ScheduleStart = I;
3962 ScheduleEnd = I->getNextNode();
3963 if (isOneOf(S, I) != I)
3964 CheckSheduleForI(I);
3965 assert(ScheduleEnd && "tried to vectorize a terminator?")((ScheduleEnd && "tried to vectorize a terminator?") ?
static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a terminator?\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3965, __PRETTY_FUNCTION__))
;
3966 LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3967 return true;
3968 }
3969 // Search up and down at the same time, because we don't know if the new
3970 // instruction is above or below the existing scheduling region.
3971 BasicBlock::reverse_iterator UpIter =
3972 ++ScheduleStart->getIterator().getReverse();
3973 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3974 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3975 BasicBlock::iterator LowerEnd = BB->end();
3976 while (true) {
3977 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3978 LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3979 return false;
3980 }
3981
3982 if (UpIter != UpperEnd) {
3983 if (&*UpIter == I) {
3984 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3985 ScheduleStart = I;
3986 if (isOneOf(S, I) != I)
3987 CheckSheduleForI(I);
3988 LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
3989 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3990 return true;
3991 }
3992 UpIter++;
3993 }
3994 if (DownIter != LowerEnd) {
3995 if (&*DownIter == I) {
3996 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3997 nullptr);
3998 ScheduleEnd = I->getNextNode();
3999 if (isOneOf(S, I) != I)
4000 CheckSheduleForI(I);
4001 assert(ScheduleEnd && "tried to vectorize a terminator?")((ScheduleEnd && "tried to vectorize a terminator?") ?
static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a terminator?\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4001, __PRETTY_FUNCTION__))
;
4002 LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *Ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
4003 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
4004 return true;
4005 }
4006 DownIter++;
4007 }
4008 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4009, __PRETTY_FUNCTION__))
4009 "instruction not found in block")(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4009, __PRETTY_FUNCTION__))
;
4010 }
4011 return true;
4012}
4013
4014void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
4015 Instruction *ToI,
4016 ScheduleData *PrevLoadStore,
4017 ScheduleData *NextLoadStore) {
4018 ScheduleData *CurrentLoadStore = PrevLoadStore;
4019 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
4020 ScheduleData *SD = ScheduleDataMap[I];
4021 if (!SD) {
4022 SD = allocateScheduleDataChunks();
4023 ScheduleDataMap[I] = SD;
4024 SD->Inst = I;
4025 }
4026 assert(!isInSchedulingRegion(SD) &&((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4027, __PRETTY_FUNCTION__))
4027 "new ScheduleData already in scheduling region")((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4027, __PRETTY_FUNCTION__))
;
4028 SD->init(SchedulingRegionID, I);
4029
4030 if (I->mayReadOrWriteMemory() &&
4031 (!isa<IntrinsicInst>(I) ||
4032 cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
4033 // Update the linked list of memory accessing instructions.
4034 if (CurrentLoadStore) {
4035 CurrentLoadStore->NextLoadStore = SD;
4036 } else {
4037 FirstLoadStoreInRegion = SD;
4038 }
4039 CurrentLoadStore = SD;
4040 }
4041 }
4042 if (NextLoadStore) {
4043 if (CurrentLoadStore)
4044 CurrentLoadStore->NextLoadStore = NextLoadStore;
4045 } else {
4046 LastLoadStoreInRegion = CurrentLoadStore;
4047 }
4048}
4049
4050void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
4051 bool InsertInReadyList,
4052 BoUpSLP *SLP) {
4053 assert(SD->isSchedulingEntity())((SD->isSchedulingEntity()) ? static_cast<void> (0) :
__assert_fail ("SD->isSchedulingEntity()", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4053, __PRETTY_FUNCTION__))
;
4054
4055 SmallVector<ScheduleData *, 10> WorkList;
4056 WorkList.push_back(SD);
4057
4058 while (!WorkList.empty()) {
4059 ScheduleData *SD = WorkList.back();
4060 WorkList.pop_back();
4061
4062 ScheduleData *BundleMember = SD;
4063 while (BundleMember) {
4064 assert(isInSchedulingRegion(BundleMember))((isInSchedulingRegion(BundleMember)) ? static_cast<void>
(0) : __assert_fail ("isInSchedulingRegion(BundleMember)", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4064, __PRETTY_FUNCTION__))
;
4065 if (!BundleMember->hasValidDependencies()) {
4066
4067 LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
4068 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
4069 BundleMember->Dependencies = 0;
4070 BundleMember->resetUnscheduledDeps();
4071
4072 // Handle def-use chain dependencies.
4073 if (BundleMember->OpValue != BundleMember->Inst) {
4074 ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
4075 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4076 BundleMember->Dependencies++;
4077 ScheduleData *DestBundle = UseSD->FirstInBundle;
4078 if (!DestBundle->IsScheduled)
4079 BundleMember->incrementUnscheduledDeps(1);
4080 if (!DestBundle->hasValidDependencies())
4081 WorkList.push_back(DestBundle);
4082 }
4083 } else {
4084 for (User *U : BundleMember->Inst->users()) {
4085 if (isa<Instruction>(U)) {
4086 ScheduleData *UseSD = getScheduleData(U);
4087 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4088 BundleMember->Dependencies++;
4089 ScheduleData *DestBundle = UseSD->FirstInBundle;
4090 if (!DestBundle->IsScheduled)
4091 BundleMember->incrementUnscheduledDeps(1);
4092 if (!DestBundle->hasValidDependencies())
4093 WorkList.push_back(DestBundle);
4094 }
4095 } else {
4096 // I'm not sure if this can ever happen. But we need to be safe.
4097 // This lets the instruction/bundle never be scheduled and
4098 // eventually disable vectorization.
4099 BundleMember->Dependencies++;
4100 BundleMember->incrementUnscheduledDeps(1);
4101 }
4102 }
4103 }
4104
4105 // Handle the memory dependencies.
4106 ScheduleData *DepDest = BundleMember->NextLoadStore;
4107 if (DepDest) {
4108 Instruction *SrcInst = BundleMember->Inst;
4109 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
4110 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
4111 unsigned numAliased = 0;
4112 unsigned DistToSrc = 1;
4113
4114 while (DepDest) {
4115 assert(isInSchedulingRegion(DepDest))((isInSchedulingRegion(DepDest)) ? static_cast<void> (0
) : __assert_fail ("isInSchedulingRegion(DepDest)", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4115, __PRETTY_FUNCTION__))
;
4116
4117 // We have two limits to reduce the complexity:
4118 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
4119 // SLP->isAliased (which is the expensive part in this loop).
4120 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
4121 // the whole loop (even if the loop is fast, it's quadratic).
4122 // It's important for the loop break condition (see below) to
4123 // check this limit even between two read-only instructions.
4124 if (DistToSrc >= MaxMemDepDistance ||
4125 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
4126 (numAliased >= AliasedCheckLimit ||
4127 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
4128
4129 // We increment the counter only if the locations are aliased
4130 // (instead of counting all alias checks). This gives a better
4131 // balance between reduced runtime and accurate dependencies.
4132 numAliased++;
4133
4134 DepDest->MemoryDependencies.push_back(BundleMember);
4135 BundleMember->Dependencies++;
4136 ScheduleData *DestBundle = DepDest->FirstInBundle;
4137 if (!DestBundle->IsScheduled) {
4138 BundleMember->incrementUnscheduledDeps(1);
4139 }
4140 if (!DestBundle->hasValidDependencies()) {
4141 WorkList.push_back(DestBundle);
4142 }
4143 }
4144 DepDest = DepDest->NextLoadStore;
4145
4146 // Example, explaining the loop break condition: Let's assume our
4147 // starting instruction is i0 and MaxMemDepDistance = 3.
4148 //
4149 // +--------v--v--v
4150 // i0,i1,i2,i3,i4,i5,i6,i7,i8
4151 // +--------^--^--^
4152 //
4153 // MaxMemDepDistance let us stop alias-checking at i3 and we add
4154 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
4155 // Previously we already added dependencies from i3 to i6,i7,i8
4156 // (because of MaxMemDepDistance). As we added a dependency from
4157 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4158 // and we can abort this loop at i6.
4159 if (DistToSrc >= 2 * MaxMemDepDistance)
4160 break;
4161 DistToSrc++;
4162 }
4163 }
4164 }
4165 BundleMember = BundleMember->NextInBundle;
4166 }
4167 if (InsertInReadyList && SD->isReady()) {
4168 ReadyInsts.push_back(SD);
4169 LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
4170 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
4171 }
4172 }
4173}
4174
4175void BoUpSLP::BlockScheduling::resetSchedule() {
4176 assert(ScheduleStart &&((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4177, __PRETTY_FUNCTION__))
4177 "tried to reset schedule on block which has not been scheduled")((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4177, __PRETTY_FUNCTION__))
;
4178 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4179 doForAllOpcodes(I, [&](ScheduleData *SD) {
4180 assert(isInSchedulingRegion(SD) &&((isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4181, __PRETTY_FUNCTION__))
4181 "ScheduleData not in scheduling region")((isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("isInSchedulingRegion(SD) && \"ScheduleData not in scheduling region\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4181, __PRETTY_FUNCTION__))
;
4182 SD->IsScheduled = false;
4183 SD->resetUnscheduledDeps();
4184 });
4185 }
4186 ReadyInsts.clear();
4187}
4188
4189void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4190 if (!BS->ScheduleStart)
4191 return;
4192
4193 LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
4194
4195 BS->resetSchedule();
4196
4197 // For the real scheduling we use a more sophisticated ready-list: it is
4198 // sorted by the original instruction location. This lets the final schedule
4199 // be as close as possible to the original instruction order.
4200 struct ScheduleDataCompare {
4201 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4202 return SD2->SchedulingPriority < SD1->SchedulingPriority;
4203 }
4204 };
4205 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4206
4207 // Ensure that all dependency data is updated and fill the ready-list with
4208 // initial instructions.
4209 int Idx = 0;
4210 int NumToSchedule = 0;
4211 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4212 I = I->getNextNode()) {
4213 BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4214 assert(SD->isPartOfBundle() ==((SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr
) && "scheduler and vectorizer bundle mismatch") ? static_cast
<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4216, __PRETTY_FUNCTION__))
4215 (getTreeEntry(SD->Inst) != nullptr) &&((SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr
) && "scheduler and vectorizer bundle mismatch") ? static_cast
<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4216, __PRETTY_FUNCTION__))
4216 "scheduler and vectorizer bundle mismatch")((SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr
) && "scheduler and vectorizer bundle mismatch") ? static_cast
<void> (0) : __assert_fail ("SD->isPartOfBundle() == (getTreeEntry(SD->Inst) != nullptr) && \"scheduler and vectorizer bundle mismatch\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4216, __PRETTY_FUNCTION__))
;
4217 SD->FirstInBundle->SchedulingPriority = Idx++;
4218 if (SD->isSchedulingEntity()) {
4219 BS->calculateDependencies(SD, false, this);
4220 NumToSchedule++;
4221 }
4222 });
4223 }
4224 BS->initialFillReadyList(ReadyInsts);
4225
4226 Instruction *LastScheduledInst = BS->ScheduleEnd;
4227
4228 // Do the "real" scheduling.
4229 while (!ReadyInsts.empty()) {
4230 ScheduleData *picked = *ReadyInsts.begin();
4231 ReadyInsts.erase(ReadyInsts.begin());
4232
4233 // Move the scheduled instruction(s) to their dedicated places, if not
4234 // there yet.
4235 ScheduleData *BundleMember = picked;
4236 while (BundleMember) {
4237 Instruction *pickedInst = BundleMember->Inst;
4238 if (LastScheduledInst->getNextNode() != pickedInst) {
4239 BS->BB->getInstList().remove(pickedInst);
4240 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4241 pickedInst);
4242 }
4243 LastScheduledInst = pickedInst;
4244 BundleMember = BundleMember->NextInBundle;
4245 }
4246
4247 BS->schedule(picked, ReadyInsts);
4248 NumToSchedule--;
4249 }
4250 assert(NumToSchedule == 0 && "could not schedule all instructions")((NumToSchedule == 0 && "could not schedule all instructions"
) ? static_cast<void> (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4250, __PRETTY_FUNCTION__))
;
4251
4252 // Avoid duplicate scheduling of the block.
4253 BS->ScheduleStart = nullptr;
4254}
4255
4256unsigned BoUpSLP::getVectorElementSize(Value *V) {
4257 // If V is a store, just return the width of the stored value without
4258 // traversing the expression tree. This is the common case.
4259 if (auto *Store = dyn_cast<StoreInst>(V))
4260 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4261
4262 // If V is not a store, we can traverse the expression tree to find loads
4263 // that feed it. The type of the loaded value may indicate a more suitable
4264 // width than V's type. We want to base the vector element size on the width
4265 // of memory operations where possible.
4266 SmallVector<Instruction *, 16> Worklist;
4267 SmallPtrSet<Instruction *, 16> Visited;
4268 if (auto *I = dyn_cast<Instruction>(V))
4269 Worklist.push_back(I);
4270
4271 // Traverse the expression tree in bottom-up order looking for loads. If we
4272 // encounter an instruciton we don't yet handle, we give up.
4273 auto MaxWidth = 0u;
4274 auto FoundUnknownInst = false;
4275 while (!Worklist.empty() && !FoundUnknownInst) {
4276 auto *I = Worklist.pop_back_val();
4277 Visited.insert(I);
4278
4279 // We should only be looking at scalar instructions here. If the current
4280 // instruction has a vector type, give up.
4281 auto *Ty = I->getType();
4282 if (isa<VectorType>(Ty))
4283 FoundUnknownInst = true;
4284
4285 // If the current instruction is a load, update MaxWidth to reflect the
4286 // width of the loaded value.
4287 else if (isa<LoadInst>(I))
4288 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4289
4290 // Otherwise, we need to visit the operands of the instruction. We only
4291 // handle the interesting cases from buildTree here. If an operand is an
4292 // instruction we haven't yet visited, we add it to the worklist.
4293 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4294 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4295 for (Use &U : I->operands())
4296 if (auto *J = dyn_cast<Instruction>(U.get()))
4297 if (!Visited.count(J))
4298 Worklist.push_back(J);
4299 }
4300
4301 // If we don't yet handle the instruction, give up.
4302 else
4303 FoundUnknownInst = true;
4304 }
4305
4306 // If we didn't encounter a memory access in the expression tree, or if we
4307 // gave up for some reason, just return the width of V.
4308 if (!MaxWidth || FoundUnknownInst)
4309 return DL->getTypeSizeInBits(V->getType());
4310
4311 // Otherwise, return the maximum width we found.
4312 return MaxWidth;
4313}
4314
4315// Determine if a value V in a vectorizable expression Expr can be demoted to a
4316// smaller type with a truncation. We collect the values that will be demoted
4317// in ToDemote and additional roots that require investigating in Roots.
4318static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
4319 SmallVectorImpl<Value *> &ToDemote,
4320 SmallVectorImpl<Value *> &Roots) {
4321 // We can always demote constants.
4322 if (isa<Constant>(V)) {
4323 ToDemote.push_back(V);
4324 return true;
4325 }
4326
4327 // If the value is not an instruction in the expression with only one use, it
4328 // cannot be demoted.
4329 auto *I = dyn_cast<Instruction>(V);
4330 if (!I || !I->hasOneUse() || !Expr.count(I))
4331 return false;
4332
4333 switch (I->getOpcode()) {
4334
4335 // We can always demote truncations and extensions. Since truncations can
4336 // seed additional demotion, we save the truncated value.
4337 case Instruction::Trunc:
4338 Roots.push_back(I->getOperand(0));
4339 break;
4340 case Instruction::ZExt:
4341 case Instruction::SExt:
4342 break;
4343
4344 // We can demote certain binary operations if we can demote both of their
4345 // operands.
4346 case Instruction::Add:
4347 case Instruction::Sub:
4348 case Instruction::Mul:
4349 case Instruction::And:
4350 case Instruction::Or:
4351 case Instruction::Xor:
4352 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4353 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4354 return false;
4355 break;
4356
4357 // We can demote selects if we can demote their true and false values.
4358 case Instruction::Select: {
4359 SelectInst *SI = cast<SelectInst>(I);
4360 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4361 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4362 return false;
4363 break;
4364 }
4365
4366 // We can demote phis if we can demote all their incoming operands. Note that
4367 // we don't need to worry about cycles since we ensure single use above.
4368 case Instruction::PHI: {
4369 PHINode *PN = cast<PHINode>(I);
4370 for (Value *IncValue : PN->incoming_values())
4371 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4372 return false;
4373 break;
4374 }
4375
4376 // Otherwise, conservatively give up.
4377 default:
4378 return false;
4379 }
4380
4381 // Record the value that we can demote.
4382 ToDemote.push_back(V);
4383 return true;
4384}
4385
4386void BoUpSLP::computeMinimumValueSizes() {
4387 // If there are no external uses, the expression tree must be rooted by a
4388 // store. We can't demote in-memory values, so there is nothing to do here.
4389 if (ExternalUses.empty())
4390 return;
4391
4392 // We only attempt to truncate integer expressions.
4393 auto &TreeRoot = VectorizableTree[0].Scalars;
4394 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4395 if (!TreeRootIT)
4396 return;
4397
4398 // If the expression is not rooted by a store, these roots should have
4399 // external uses. We will rely on InstCombine to rewrite the expression in
4400 // the narrower type. However, InstCombine only rewrites single-use values.
4401 // This means that if a tree entry other than a root is used externally, it
4402 // must have multiple uses and InstCombine will not rewrite it. The code
4403 // below ensures that only the roots are used externally.
4404 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4405 for (auto &EU : ExternalUses)
4406 if (!Expr.erase(EU.Scalar))
4407 return;
4408 if (!Expr.empty())
4409 return;
4410
4411 // Collect the scalar values of the vectorizable expression. We will use this
4412 // context to determine which values can be demoted. If we see a truncation,
4413 // we mark it as seeding another demotion.
4414 for (auto &Entry : VectorizableTree)
4415 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4416
4417 // Ensure the roots of the vectorizable tree don't form a cycle. They must
4418 // have a single external user that is not in the vectorizable tree.
4419 for (auto *Root : TreeRoot)
4420 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4421 return;
4422
4423 // Conservatively determine if we can actually truncate the roots of the
4424 // expression. Collect the values that can be demoted in ToDemote and
4425 // additional roots that require investigating in Roots.
4426 SmallVector<Value *, 32> ToDemote;
4427 SmallVector<Value *, 4> Roots;
4428 for (auto *Root : TreeRoot)
4429 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4430 return;
4431
4432 // The maximum bit width required to represent all the values that can be
4433 // demoted without loss of precision. It would be safe to truncate the roots
4434 // of the expression to this width.
4435 auto MaxBitWidth = 8u;
4436
4437 // We first check if all the bits of the roots are demanded. If they're not,
4438 // we can truncate the roots to this narrower type.
4439 for (auto *Root : TreeRoot) {
4440 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4441 MaxBitWidth = std::max<unsigned>(
4442 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4443 }
4444
4445 // True if the roots can be zero-extended back to their original type, rather
4446 // than sign-extended. We know that if the leading bits are not demanded, we
4447 // can safely zero-extend. So we initialize IsKnownPositive to True.
4448 bool IsKnownPositive = true;
4449
4450 // If all the bits of the roots are demanded, we can try a little harder to
4451 // compute a narrower type. This can happen, for example, if the roots are
4452 // getelementptr indices. InstCombine promotes these indices to the pointer
4453 // width. Thus, all their bits are technically demanded even though the
4454 // address computation might be vectorized in a smaller type.
4455 //
4456 // We start by looking at each entry that can be demoted. We compute the
4457 // maximum bit width required to store the scalar by using ValueTracking to
4458 // compute the number of high-order bits we can truncate.
4459 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
4460 llvm::all_of(TreeRoot, [](Value *R) {
4461 assert(R->hasOneUse() && "Root should have only one use!")((R->hasOneUse() && "Root should have only one use!"
) ? static_cast<void> (0) : __assert_fail ("R->hasOneUse() && \"Root should have only one use!\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4461, __PRETTY_FUNCTION__))
;
4462 return isa<GetElementPtrInst>(R->user_back());
4463 })) {
4464 MaxBitWidth = 8u;
4465
4466 // Determine if the sign bit of all the roots is known to be zero. If not,
4467 // IsKnownPositive is set to False.
4468 IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4469 KnownBits Known = computeKnownBits(R, *DL);
4470 return Known.isNonNegative();
4471 });
4472
4473 // Determine the maximum number of bits required to store the scalar
4474 // values.
4475 for (auto *Scalar : ToDemote) {
4476 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4477 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4478 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4479 }
4480
4481 // If we can't prove that the sign bit is zero, we must add one to the
4482 // maximum bit width to account for the unknown sign bit. This preserves
4483 // the existing sign bit so we can safely sign-extend the root back to the
4484 // original type. Otherwise, if we know the sign bit is zero, we will
4485 // zero-extend the root instead.
4486 //
4487 // FIXME: This is somewhat suboptimal, as there will be cases where adding
4488 // one to the maximum bit width will yield a larger-than-necessary
4489 // type. In general, we need to add an extra bit only if we can't
4490 // prove that the upper bit of the original type is equal to the
4491 // upper bit of the proposed smaller type. If these two bits are the
4492 // same (either zero or one) we know that sign-extending from the
4493 // smaller type will result in the same value. Here, since we can't
4494 // yet prove this, we are just making the proposed smaller type
4495 // larger to ensure correctness.
4496 if (!IsKnownPositive)
4497 ++MaxBitWidth;
4498 }
4499
4500 // Round MaxBitWidth up to the next power-of-two.
4501 if (!isPowerOf2_64(MaxBitWidth))
4502 MaxBitWidth = NextPowerOf2(MaxBitWidth);
4503
4504 // If the maximum bit width we compute is less than the with of the roots'
4505 // type, we can proceed with the narrowing. Otherwise, do nothing.
4506 if (MaxBitWidth >= TreeRootIT->getBitWidth())
4507 return;
4508
4509 // If we can truncate the root, we must collect additional values that might
4510 // be demoted as a result. That is, those seeded by truncations we will
4511 // modify.
4512 while (!Roots.empty())
4513 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4514
4515 // Finally, map the values we can demote to the maximum bit with we computed.
4516 for (auto *Scalar : ToDemote)
4517 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4518}
4519
4520namespace {
4521
4522/// The SLPVectorizer Pass.
4523struct SLPVectorizer : public FunctionPass {
4524 SLPVectorizerPass Impl;
4525
4526 /// Pass identification, replacement for typeid
4527 static char ID;
4528
4529 explicit SLPVectorizer() : FunctionPass(ID) {
4530 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
4531 }
4532
4533 bool doInitialization(Module &M) override {
4534 return false;
4535 }
4536
4537 bool runOnFunction(Function &F) override {
4538 if (skipFunction(F))
4539 return false;
4540
4541 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4542 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4543 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4544 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4545 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4546 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4547 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4548 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4549 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4550 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4551
4552 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4553 }
4554
4555 void getAnalysisUsage(AnalysisUsage &AU) const override {
4556 FunctionPass::getAnalysisUsage(AU);
4557 AU.addRequired<AssumptionCacheTracker>();
4558 AU.addRequired<ScalarEvolutionWrapperPass>();
4559 AU.addRequired<AAResultsWrapperPass>();
4560 AU.addRequired<TargetTransformInfoWrapperPass>();
4561 AU.addRequired<LoopInfoWrapperPass>();
4562 AU.addRequired<DominatorTreeWrapperPass>();
4563 AU.addRequired<DemandedBitsWrapperPass>();
4564 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
4565 AU.addPreserved<LoopInfoWrapperPass>();
4566 AU.addPreserved<DominatorTreeWrapperPass>();
4567 AU.addPreserved<AAResultsWrapperPass>();
4568 AU.addPreserved<GlobalsAAWrapperPass>();
4569 AU.setPreservesCFG();
4570 }
4571};
4572
4573} // end anonymous namespace
4574
4575PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
4576 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4577 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4578 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4579 auto *AA = &AM.getResult<AAManager>(F);
4580 auto *LI = &AM.getResult<LoopAnalysis>(F);
4581 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4582 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4583 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4584 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4585
4586 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4587 if (!Changed)
4588 return PreservedAnalyses::all();
4589
4590 PreservedAnalyses PA;
4591 PA.preserveSet<CFGAnalyses>();
4592 PA.preserve<AAManager>();
4593 PA.preserve<GlobalsAA>();
4594 return PA;
4595}
4596
4597bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
4598 TargetTransformInfo *TTI_,
4599 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4600 LoopInfo *LI_, DominatorTree *DT_,
4601 AssumptionCache *AC_, DemandedBits *DB_,
4602 OptimizationRemarkEmitter *ORE_) {
4603 SE = SE_;
4604 TTI = TTI_;
4605 TLI = TLI_;
4606 AA = AA_;
4607 LI = LI_;
4608 DT = DT_;
4609 AC = AC_;
4610 DB = DB_;
4611 DL = &F.getParent()->getDataLayout();
4612
4613 Stores.clear();
4614 GEPs.clear();
4615 bool Changed = false;
4616
4617 // If the target claims to have no vector registers don't attempt
4618 // vectorization.
4619 if (!TTI->getNumberOfRegisters(true))
4620 return false;
4621
4622 // Don't vectorize when the attribute NoImplicitFloat is used.
4623 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4624 return false;
4625
4626 LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
4627
4628 // Use the bottom up slp vectorizer to construct chains that start with
4629 // store instructions.
4630 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4631
4632 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4633 // delete instructions.
4634
4635 // Scan the blocks in the function in post order.
4636 for (auto BB : post_order(&F.getEntryBlock())) {
4637 collectSeedInstructions(BB);
4638
4639 // Vectorize trees that end at stores.
4640 if (!Stores.empty()) {
4641 LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
4642 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
4643 Changed |= vectorizeStoreChains(R);
4644 }
4645
4646 // Vectorize trees that end at reductions.
4647 Changed |= vectorizeChainsInBlock(BB, R);
4648
4649 // Vectorize the index computations of getelementptr instructions. This
4650 // is primarily intended to catch gather-like idioms ending at
4651 // non-consecutive loads.
4652 if (!GEPs.empty()) {
4653 LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
4654 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
4655 Changed |= vectorizeGEPIndices(BB, R);
4656 }
4657 }
4658
4659 if (Changed) {
4660 R.optimizeGatherSequence();
4661 LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
4662 LLVM_DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
4663 }
4664 return Changed;
4665}
4666
4667/// Check that the Values in the slice in VL array are still existent in
4668/// the WeakTrackingVH array.
4669/// Vectorization of part of the VL array may cause later values in the VL array
4670/// to become invalid. We track when this has happened in the WeakTrackingVH
4671/// array.
4672static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
4673 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4674 unsigned SliceSize) {
4675 VL = VL.slice(SliceBegin, SliceSize);
4676 VH = VH.slice(SliceBegin, SliceSize);
4677 return !std::equal(VL.begin(), VL.end(), VH.begin());
4678}
4679
4680bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4681 unsigned VecRegSize) {
4682 const unsigned ChainLen = Chain.size();
4683 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
4684 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
4685 const unsigned Sz = R.getVectorElementSize(Chain[0]);
4686 const unsigned VF = VecRegSize / Sz;
4687
4688 if (!isPowerOf2_32(Sz) || VF < 2)
4689 return false;
4690
4691 // Keep track of values that were deleted by vectorizing in the loop below.
4692 const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4693
4694 bool Changed = false;
4695 // Look for profitable vectorizable trees at all offsets, starting at zero.
4696 for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4697
4698 // Check that a previous iteration of this loop did not delete the Value.
4699 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4700 continue;
4701
4702 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
4703 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
4704 ArrayRef<Value *> Operands = Chain.slice(i, VF);
4705
4706 R.buildTree(Operands);
4707 if (R.isTreeTinyAndNotFullyVectorizable())
4708 continue;
4709
4710 R.computeMinimumValueSizes();
4711
4712 int Cost = R.getTreeCost();
4713
4714 LLVM_DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VFdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
4715 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
4716 if (Cost < -SLPCostThreshold) {
4717 LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
4718
4719 using namespace ore;
4720
4721 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "StoresVectorized",
4722 cast<StoreInst>(Chain[i]))
4723 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4724 << " and with tree size "
4725 << NV("TreeSize", R.getTreeSize()));
4726
4727 R.vectorizeTree();
4728
4729 // Move to the next bundle.
4730 i += VF - 1;
4731 Changed = true;
4732 }
4733 }
4734
4735 return Changed;
4736}
4737
4738bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4739 BoUpSLP &R) {
4740 SetVector<StoreInst *> Heads;
4741 SmallDenseSet<StoreInst *> Tails;
4742 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4743
4744 // We may run into multiple chains that merge into a single chain. We mark the
4745 // stores that we vectorized so that we don't visit the same store twice.
4746 BoUpSLP::ValueSet VectorizedStores;
4747 bool Changed = false;
4748
4749 // Do a quadratic search on all of the given stores in reverse order and find
4750 // all of the pairs of stores that follow each other.
4751 SmallVector<unsigned, 16> IndexQueue;
4752 unsigned E = Stores.size();
4753 IndexQueue.resize(E - 1);
4754 for (unsigned I = E; I > 0; --I) {
4755 unsigned Idx = I - 1;
4756 // If a store has multiple consecutive store candidates, search Stores
4757 // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4758 // This is because usually pairing with immediate succeeding or preceding
4759 // candidate create the best chance to find slp vectorization opportunity.
4760 unsigned Offset = 1;
4761 unsigned Cnt = 0;
4762 for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4763 if (Idx >= Offset) {
4764 IndexQueue[Cnt] = Idx - Offset;
4765 ++Cnt;
4766 }
4767 if (Idx + Offset < E) {
4768 IndexQueue[Cnt] = Idx + Offset;
4769 ++Cnt;
4770 }
4771 }
4772
4773 for (auto K : IndexQueue) {
4774 if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4775 Tails.insert(Stores[Idx]);
4776 Heads.insert(Stores[K]);
4777 ConsecutiveChain[Stores[K]] = Stores[Idx];
4778 break;
4779 }
4780 }
4781 }
4782
4783 // For stores that start but don't end a link in the chain:
4784 for (auto *SI : llvm::reverse(Heads)) {
4785 if (Tails.count(SI))
4786 continue;
4787
4788 // We found a store instr that starts a chain. Now follow the chain and try
4789 // to vectorize it.
4790 BoUpSLP::ValueList Operands;
4791 StoreInst *I = SI;
4792 // Collect the chain into a list.
4793 while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4794 Operands.push_back(I);
4795 // Move to the next value in the chain.
4796 I = ConsecutiveChain[I];
4797 }
4798
4799 // FIXME: Is division-by-2 the correct step? Should we assert that the
4800 // register size is a power-of-2?
4801 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4802 Size /= 2) {
4803 if (vectorizeStoreChain(Operands, R, Size)) {
4804 // Mark the vectorized stores so that we don't vectorize them again.
4805 VectorizedStores.insert(Operands.begin(), Operands.end());
4806 Changed = true;
4807 break;
4808 }
4809 }
4810 }
4811
4812 return Changed;
4813}
4814
4815void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4816 // Initialize the collections. We will make a single pass over the block.
4817 Stores.clear();
4818 GEPs.clear();
4819
4820 // Visit the store and getelementptr instructions in BB and organize them in
4821 // Stores and GEPs according to the underlying objects of their pointer
4822 // operands.
4823 for (Instruction &I : *BB) {
4824 // Ignore store instructions that are volatile or have a pointer operand
4825 // that doesn't point to a scalar type.
4826 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4827 if (!SI->isSimple())
4828 continue;
4829 if (!isValidElementType(SI->getValueOperand()->getType()))
4830 continue;
4831 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4832 }
4833
4834 // Ignore getelementptr instructions that have more than one index, a
4835 // constant index, or a pointer operand that doesn't point to a scalar
4836 // type.
4837 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4838 auto Idx = GEP->idx_begin()->get();
4839 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4840 continue;
4841 if (!isValidElementType(Idx->getType()))
4842 continue;
4843 if (GEP->getType()->isVectorTy())
4844 continue;
4845 GEPs[GEP->getPointerOperand()].push_back(GEP);
4846 }
4847 }
4848}
4849
4850bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4851 if (!A || !B)
4852 return false;
4853 Value *VL[] = { A, B };
4854 return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4855}
4856
4857bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4858 int UserCost, bool AllowReorder) {
4859 if (VL.size() < 2)
4860 return false;
4861
4862 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4863 << VL.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
4864
4865 // Check that all of the parts are scalar instructions of the same type,
4866 // we permit an alternate opcode via InstructionsState.
4867 InstructionsState S = getSameOpcode(VL);
4868 if (!S.getOpcode())
4869 return false;
4870
4871 Instruction *I0 = cast<Instruction>(S.OpValue);
4872 unsigned Sz = R.getVectorElementSize(I0);
4873 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4874 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4875 if (MaxVF < 2) {
4876 R.getORE()->emit([&]() {
4877 return OptimizationRemarkMissed(SV_NAME"slp-vectorizer", "SmallVF", I0)
4878 << "Cannot SLP vectorize list: vectorization factor "
4879 << "less than 2 is not supported";
4880 });
4881 return false;
4882 }
4883
4884 for (Value *V : VL) {
4885 Type *Ty = V->getType();
4886 if (!isValidElementType(Ty)) {
4887 // NOTE: the following will give user internal llvm type name, which may
4888 // not be useful.
4889 R.getORE()->emit([&]() {
4890 std::string type_str;
4891 llvm::raw_string_ostream rso(type_str);
4892 Ty->print(rso);
4893 return OptimizationRemarkMissed(SV_NAME"slp-vectorizer", "UnsupportedType", I0)
4894 << "Cannot SLP vectorize list: type "
4895 << rso.str() + " is unsupported by vectorizer";
4896 });
4897 return false;
4898 }
4899 }
4900
4901 bool Changed = false;
4902 bool CandidateFound = false;
4903 int MinCost = SLPCostThreshold;
4904
4905 // Keep track of values that were deleted by vectorizing in the loop below.
4906 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4907
4908 unsigned NextInst = 0, MaxInst = VL.size();
4909 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4910 VF /= 2) {
4911 // No actual vectorization should happen, if number of parts is the same as
4912 // provided vectorization factor (i.e. the scalar type is used for vector
4913 // code during codegen).
4914 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4915 if (TTI->getNumberOfParts(VecTy) == VF)
4916 continue;
4917 for (unsigned I = NextInst; I < MaxInst; ++I) {
4918 unsigned OpsWidth = 0;
4919
4920 if (I + VF > MaxInst)
4921 OpsWidth = MaxInst - I;
4922 else
4923 OpsWidth = VF;
4924
4925 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4926 break;
4927
4928 // Check that a previous iteration of this loop did not delete the Value.
4929 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4930 continue;
4931
4932 LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4933 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
4934 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4935
4936 R.buildTree(Ops);
4937 Optional<ArrayRef<unsigned>> Order = R.bestOrder();
4938 // TODO: check if we can allow reordering for more cases.
4939 if (AllowReorder && Order) {
4940 // TODO: reorder tree nodes without tree rebuilding.
4941 // Conceptually, there is nothing actually preventing us from trying to
4942 // reorder a larger list. In fact, we do exactly this when vectorizing
4943 // reductions. However, at this point, we only expect to get here when
4944 // there are exactly two operations.
4945 assert(Ops.size() == 2)((Ops.size() == 2) ? static_cast<void> (0) : __assert_fail
("Ops.size() == 2", "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4945, __PRETTY_FUNCTION__))
;
4946 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4947 R.buildTree(ReorderedOps, None);
4948 }
4949 if (R.isTreeTinyAndNotFullyVectorizable())
4950 continue;
4951
4952 R.computeMinimumValueSizes();
4953 int Cost = R.getTreeCost() - UserCost;
4954 CandidateFound = true;
4955 MinCost = std::min(MinCost, Cost);
4956
4957 if (Cost < -SLPCostThreshold) {
4958 LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
4959 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedList",
4960 cast<Instruction>(Ops[0]))
4961 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4962 << " and with tree size "
4963 << ore::NV("TreeSize", R.getTreeSize()));
4964
4965 R.vectorizeTree();
4966 // Move to the next bundle.
4967 I += VF - 1;
4968 NextInst = I + 1;
4969 Changed = true;
4970 }
4971 }
4972 }
4973
4974 if (!Changed && CandidateFound) {
4975 R.getORE()->emit([&]() {
4976 return OptimizationRemarkMissed(SV_NAME"slp-vectorizer", "NotBeneficial", I0)
4977 << "List vectorization was possible but not beneficial with cost "
4978 << ore::NV("Cost", MinCost) << " >= "
4979 << ore::NV("Treshold", -SLPCostThreshold);
4980 });
4981 } else if (!Changed) {
4982 R.getORE()->emit([&]() {
4983 return OptimizationRemarkMissed(SV_NAME"slp-vectorizer", "NotPossible", I0)
4984 << "Cannot SLP vectorize list: vectorization was impossible"
4985 << " with available vectorization factors";
4986 });
4987 }
4988 return Changed;
4989}
4990
4991bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4992 if (!I)
4993 return false;
4994
4995 if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4996 return false;
4997
4998 Value *P = I->getParent();
4999
5000 // Vectorize in current basic block only.
5001 auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
5002 auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
5003 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
5004 return false;
5005
5006 // Try to vectorize V.
5007 if (tryToVectorizePair(Op0, Op1, R))
5008 return true;
5009
5010 auto *A = dyn_cast<BinaryOperator>(Op0);
5011 auto *B = dyn_cast<BinaryOperator>(Op1);
5012 // Try to skip B.
5013 if (B && B->hasOneUse()) {
5014 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
5015 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
5016 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
5017 return true;
5018 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
5019 return true;
5020 }
5021
5022 // Try to skip A.
5023 if (A && A->hasOneUse()) {
5024 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
5025 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
5026 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
5027 return true;
5028 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
5029 return true;
5030 }
5031 return false;
5032}
5033
5034/// Generate a shuffle mask to be used in a reduction tree.
5035///
5036/// \param VecLen The length of the vector to be reduced.
5037/// \param NumEltsToRdx The number of elements that should be reduced in the
5038/// vector.
5039/// \param IsPairwise Whether the reduction is a pairwise or splitting
5040/// reduction. A pairwise reduction will generate a mask of
5041/// <0,2,...> or <1,3,..> while a splitting reduction will generate
5042/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
5043/// \param IsLeft True will generate a mask of even elements, odd otherwise.
5044static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
5045 bool IsPairwise, bool IsLeft,
5046 IRBuilder<> &Builder) {
5047 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"
) ? static_cast<void> (0) : __assert_fail ("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5047, __PRETTY_FUNCTION__))
;
5048
5049 SmallVector<Constant *, 32> ShuffleMask(
5050 VecLen, UndefValue::get(Builder.getInt32Ty()));
5051
5052 if (IsPairwise)
5053 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
5054 for (unsigned i = 0; i != NumEltsToRdx; ++i)
5055 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
5056 else
5057 // Move the upper half of the vector to the lower half.
5058 for (unsigned i = 0; i != NumEltsToRdx; ++i)
5059 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
5060
5061 return ConstantVector::get(ShuffleMask);
5062}
5063
5064namespace {
5065
5066/// Model horizontal reductions.
5067///
5068/// A horizontal reduction is a tree of reduction operations (currently add and
5069/// fadd) that has operations that can be put into a vector as its leaf.
5070/// For example, this tree:
5071///
5072/// mul mul mul mul
5073/// \ / \ /
5074/// + +
5075/// \ /
5076/// +
5077/// This tree has "mul" as its reduced values and "+" as its reduction
5078/// operations. A reduction might be feeding into a store or a binary operation
5079/// feeding a phi.
5080/// ...
5081/// \ /
5082/// +
5083/// |
5084/// phi +=
5085///
5086/// Or:
5087/// ...
5088/// \ /
5089/// +
5090/// |
5091/// *p =
5092///
5093class HorizontalReduction {
5094 using ReductionOpsType = SmallVector<Value *, 16>;
5095 using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
5096 ReductionOpsListType ReductionOps;
5097 SmallVector<Value *, 32> ReducedVals;
5098 // Use map vector to make stable output.
5099 MapVector<Instruction *, Value *> ExtraArgs;
5100
5101 /// Kind of the reduction data.
5102 enum ReductionKind {
5103 RK_None, /// Not a reduction.
5104 RK_Arithmetic, /// Binary reduction data.
5105 RK_Min, /// Minimum reduction data.
5106 RK_UMin, /// Unsigned minimum reduction data.
5107 RK_Max, /// Maximum reduction data.
5108 RK_UMax, /// Unsigned maximum reduction data.
5109 };
5110
5111 /// Contains info about operation, like its opcode, left and right operands.
5112 class OperationData {
5113 /// Opcode of the instruction.
5114 unsigned Opcode = 0;
5115
5116 /// Left operand of the reduction operation.
5117 Value *LHS = nullptr;
5118
5119 /// Right operand of the reduction operation.
5120 Value *RHS = nullptr;
5121
5122 /// Kind of the reduction operation.
5123 ReductionKind Kind = RK_None;
5124
5125 /// True if float point min/max reduction has no NaNs.
5126 bool NoNaN = false;
5127
5128 /// Checks if the reduction operation can be vectorized.
5129 bool isVectorizable() const {
5130 return LHS && RHS &&
5131 // We currently only support add/mul/logical && min/max reductions.
5132 ((Kind == RK_Arithmetic &&
5133 (Opcode == Instruction::Add || Opcode == Instruction::FAdd ||
5134 Opcode == Instruction::Mul || Opcode == Instruction::FMul ||
5135 Opcode == Instruction::And || Opcode == Instruction::Or ||
5136 Opcode == Instruction::Xor)) ||
5137 ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5138 (Kind == RK_Min || Kind == RK_Max)) ||
5139 (Opcode == Instruction::ICmp &&
5140 (Kind == RK_UMin || Kind == RK_UMax)));
5141 }
5142
5143 /// Creates reduction operation with the current opcode.
5144 Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5145 assert(isVectorizable() &&((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5146, __PRETTY_FUNCTION__))
5146 "Expected add|fadd or min/max reduction operation.")((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5146, __PRETTY_FUNCTION__))
;
5147 Value *Cmp;
5148 switch (Kind) {
5149 case RK_Arithmetic:
5150 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5151 Name);
5152 case RK_Min:
5153 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5154 : Builder.CreateFCmpOLT(LHS, RHS);
5155 break;
5156 case RK_Max:
5157 Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5158 : Builder.CreateFCmpOGT(LHS, RHS);
5159 break;
5160 case RK_UMin:
5161 assert(Opcode == Instruction::ICmp && "Expected integer types.")((Opcode == Instruction::ICmp && "Expected integer types."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5161, __PRETTY_FUNCTION__))
;
5162 Cmp = Builder.CreateICmpULT(LHS, RHS);
5163 break;
5164 case RK_UMax:
5165 assert(Opcode == Instruction::ICmp && "Expected integer types.")((Opcode == Instruction::ICmp && "Expected integer types."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Expected integer types.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5165, __PRETTY_FUNCTION__))
;
5166 Cmp = Builder.CreateICmpUGT(LHS, RHS);
5167 break;
5168 case RK_None:
5169 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5169)
;
5170 }
5171 return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5172 }
5173
5174 public:
5175 explicit OperationData() = default;
5176
5177 /// Construction for reduced values. They are identified by opcode only and
5178 /// don't have associated LHS/RHS values.
5179 explicit OperationData(Value *V) {
5180 if (auto *I = dyn_cast<Instruction>(V))
5181 Opcode = I->getOpcode();
5182 }
5183
5184 /// Constructor for reduction operations with opcode and its left and
5185 /// right operands.
5186 OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5187 bool NoNaN = false)
5188 : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5189 assert(Kind != RK_None && "One of the reduction operations is expected.")((Kind != RK_None && "One of the reduction operations is expected."
) ? static_cast<void> (0) : __assert_fail ("Kind != RK_None && \"One of the reduction operations is expected.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5189, __PRETTY_FUNCTION__))
;
5190 }
5191
5192 explicit operator bool() const { return Opcode; }
5193
5194 /// Get the index of the first operand.
5195 unsigned getFirstOperandIndex() const {
5196 assert(!!*this && "The opcode is not set.")((!!*this && "The opcode is not set.") ? static_cast<
void> (0) : __assert_fail ("!!*this && \"The opcode is not set.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5196, __PRETTY_FUNCTION__))
;
5197 switch (Kind) {
5198 case RK_Min:
5199 case RK_UMin:
5200 case RK_Max:
5201 case RK_UMax:
5202 return 1;
5203 case RK_Arithmetic:
5204 case RK_None:
5205 break;
5206 }
5207 return 0;
5208 }
5209
5210 /// Total number of operands in the reduction operation.
5211 unsigned getNumberOfOperands() const {
5212 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5213, __PRETTY_FUNCTION__))
5213 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5213, __PRETTY_FUNCTION__))
;
5214 switch (Kind) {
5215 case RK_Arithmetic:
5216 return 2;
5217 case RK_Min:
5218 case RK_UMin:
5219 case RK_Max:
5220 case RK_UMax:
5221 return 3;
5222 case RK_None:
5223 break;
5224 }
5225 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5225)
;
5226 }
5227
5228 /// Checks if the operation has the same parent as \p P.
5229 bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5230 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5231, __PRETTY_FUNCTION__))
5231 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5231, __PRETTY_FUNCTION__))
;
5232 if (!IsRedOp)
5233 return I->getParent() == P;
5234 switch (Kind) {
5235 case RK_Arithmetic:
5236 // Arithmetic reduction operation must be used once only.
5237 return I->getParent() == P;
5238 case RK_Min:
5239 case RK_UMin:
5240 case RK_Max:
5241 case RK_UMax: {
5242 // SelectInst must be used twice while the condition op must have single
5243 // use only.
5244 auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5245 return I->getParent() == P && Cmp && Cmp->getParent() == P;
5246 }
5247 case RK_None:
5248 break;
5249 }
5250 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5250)
;
5251 }
5252 /// Expected number of uses for reduction operations/reduced values.
5253 bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5254 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5255, __PRETTY_FUNCTION__))
5255 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5255, __PRETTY_FUNCTION__))
;
5256 switch (Kind) {
5257 case RK_Arithmetic:
5258 return I->hasOneUse();
5259 case RK_Min:
5260 case RK_UMin:
5261 case RK_Max:
5262 case RK_UMax:
5263 return I->hasNUses(2) &&
5264 (!IsReductionOp ||
5265 cast<SelectInst>(I)->getCondition()->hasOneUse());
5266 case RK_None:
5267 break;
5268 }
5269 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5269)
;
5270 }
5271
5272 /// Initializes the list of reduction operations.
5273 void initReductionOps(ReductionOpsListType &ReductionOps) {
5274 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5275, __PRETTY_FUNCTION__))
5275 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5275, __PRETTY_FUNCTION__))
;
5276 switch (Kind) {
5277 case RK_Arithmetic:
5278 ReductionOps.assign(1, ReductionOpsType());
5279 break;
5280 case RK_Min:
5281 case RK_UMin:
5282 case RK_Max:
5283 case RK_UMax:
5284 ReductionOps.assign(2, ReductionOpsType());
5285 break;
5286 case RK_None:
5287 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5287)
;
5288 }
5289 }
5290 /// Add all reduction operations for the reduction instruction \p I.
5291 void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5292 assert(Kind != RK_None && !!*this && LHS && RHS &&((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5293, __PRETTY_FUNCTION__))
5293 "Expected reduction operation.")((Kind != RK_None && !!*this && LHS &&
RHS && "Expected reduction operation.") ? static_cast
<void> (0) : __assert_fail ("Kind != RK_None && !!*this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5293, __PRETTY_FUNCTION__))
;
5294 switch (Kind) {
5295 case RK_Arithmetic:
5296 ReductionOps[0].emplace_back(I);
5297 break;
5298 case RK_Min:
5299 case RK_UMin:
5300 case RK_Max:
5301 case RK_UMax:
5302 ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5303 ReductionOps[1].emplace_back(I);
5304 break;
5305 case RK_None:
5306 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5306)
;
5307 }
5308 }
5309
5310 /// Checks if instruction is associative and can be vectorized.
5311 bool isAssociative(Instruction *I) const {
5312 assert(Kind != RK_None && *this && LHS && RHS &&((Kind != RK_None && *this && LHS && RHS
&& "Expected reduction operation.") ? static_cast<
void> (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5313, __PRETTY_FUNCTION__))
5313 "Expected reduction operation.")((Kind != RK_None && *this && LHS && RHS
&& "Expected reduction operation.") ? static_cast<
void> (0) : __assert_fail ("Kind != RK_None && *this && LHS && RHS && \"Expected reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5313, __PRETTY_FUNCTION__))
;
5314 switch (Kind) {
5315 case RK_Arithmetic:
5316 return I->isAssociative();
5317 case RK_Min:
5318 case RK_Max:
5319 return Opcode == Instruction::ICmp ||
5320 cast<Instruction>(I->getOperand(0))->isFast();
5321 case RK_UMin:
5322 case RK_UMax:
5323 assert(Opcode == Instruction::ICmp &&((Opcode == Instruction::ICmp && "Only integer compare operation is expected."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5324, __PRETTY_FUNCTION__))
5324 "Only integer compare operation is expected.")((Opcode == Instruction::ICmp && "Only integer compare operation is expected."
) ? static_cast<void> (0) : __assert_fail ("Opcode == Instruction::ICmp && \"Only integer compare operation is expected.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5324, __PRETTY_FUNCTION__))
;
5325 return true;
5326 case RK_None:
5327 break;
5328 }
5329 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5329)
;
5330 }
5331
5332 /// Checks if the reduction operation can be vectorized.
5333 bool isVectorizable(Instruction *I) const {
5334 return isVectorizable() && isAssociative(I);
5335 }
5336
5337 /// Checks if two operation data are both a reduction op or both a reduced
5338 /// value.
5339 bool operator==(const OperationData &OD) {
5340 assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&((((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS ==
!OD.RHS))) && "One of the comparing operations is incorrect."
) ? static_cast<void> (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5341, __PRETTY_FUNCTION__))
5341 "One of the comparing operations is incorrect.")((((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS ==
!OD.RHS))) && "One of the comparing operations is incorrect."
) ? static_cast<void> (0) : __assert_fail ("((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) && \"One of the comparing operations is incorrect.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5341, __PRETTY_FUNCTION__))
;
5342 return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5343 }
5344 bool operator!=(const OperationData &OD) { return !(*this == OD); }
5345 void clear() {
5346 Opcode = 0;
5347 LHS = nullptr;
5348 RHS = nullptr;
5349 Kind = RK_None;
5350 NoNaN = false;
5351 }
5352
5353 /// Get the opcode of the reduction operation.
5354 unsigned getOpcode() const {
5355 assert(isVectorizable() && "Expected vectorizable operation.")((isVectorizable() && "Expected vectorizable operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected vectorizable operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5355, __PRETTY_FUNCTION__))
;
5356 return Opcode;
5357 }
5358
5359 /// Get kind of reduction data.
5360 ReductionKind getKind() const { return Kind; }
5361 Value *getLHS() const { return LHS; }
5362 Value *getRHS() const { return RHS; }
5363 Type *getConditionType() const {
5364 switch (Kind) {
5365 case RK_Arithmetic:
5366 return nullptr;
5367 case RK_Min:
5368 case RK_Max:
5369 case RK_UMin:
5370 case RK_UMax:
5371 return CmpInst::makeCmpResultType(LHS->getType());
5372 case RK_None:
5373 break;
5374 }
5375 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5375)
;
5376 }
5377
5378 /// Creates reduction operation with the current opcode with the IR flags
5379 /// from \p ReductionOps.
5380 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5381 const ReductionOpsListType &ReductionOps) const {
5382 assert(isVectorizable() &&((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5383, __PRETTY_FUNCTION__))
5383 "Expected add|fadd or min/max reduction operation.")((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5383, __PRETTY_FUNCTION__))
;
5384 auto *Op = createOp(Builder, Name);
5385 switch (Kind) {
5386 case RK_Arithmetic:
5387 propagateIRFlags(Op, ReductionOps[0]);
5388 return Op;
5389 case RK_Min:
5390 case RK_Max:
5391 case RK_UMin:
5392 case RK_UMax:
5393 if (auto *SI = dyn_cast<SelectInst>(Op))
5394 propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5395 propagateIRFlags(Op, ReductionOps[1]);
5396 return Op;
5397 case RK_None:
5398 break;
5399 }
5400 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5400)
;
5401 }
5402 /// Creates reduction operation with the current opcode with the IR flags
5403 /// from \p I.
5404 Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5405 Instruction *I) const {
5406 assert(isVectorizable() &&((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5407, __PRETTY_FUNCTION__))
5407 "Expected add|fadd or min/max reduction operation.")((isVectorizable() && "Expected add|fadd or min/max reduction operation."
) ? static_cast<void> (0) : __assert_fail ("isVectorizable() && \"Expected add|fadd or min/max reduction operation.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5407, __PRETTY_FUNCTION__))
;
5408 auto *Op = createOp(Builder, Name);
5409 switch (Kind) {
5410 case RK_Arithmetic:
5411 propagateIRFlags(Op, I);
5412 return Op;
5413 case RK_Min:
5414 case RK_Max:
5415 case RK_UMin:
5416 case RK_UMax:
5417 if (auto *SI = dyn_cast<SelectInst>(Op)) {
5418 propagateIRFlags(SI->getCondition(),
5419 cast<SelectInst>(I)->getCondition());
5420 }
5421 propagateIRFlags(Op, I);
5422 return Op;
5423 case RK_None:
5424 break;
5425 }
5426 llvm_unreachable("Unknown reduction operation.")::llvm::llvm_unreachable_internal("Unknown reduction operation."
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5426)
;
5427 }
5428
5429 TargetTransformInfo::ReductionFlags getFlags() const {
5430 TargetTransformInfo::ReductionFlags Flags;
5431 Flags.NoNaN = NoNaN;
5432 switch (Kind) {
5433 case RK_Arithmetic:
5434 break;
5435 case RK_Min:
5436 Flags.IsSigned = Opcode == Instruction::ICmp;
5437 Flags.IsMaxOp = false;
5438 break;
5439 case RK_Max:
5440 Flags.IsSigned = Opcode == Instruction::ICmp;
5441 Flags.IsMaxOp = true;
5442 break;
5443 case RK_UMin:
5444 Flags.IsSigned = false;
5445 Flags.IsMaxOp = false;
5446 break;
5447 case RK_UMax:
5448 Flags.IsSigned = false;
5449 Flags.IsMaxOp = true;
5450 break;
5451 case RK_None:
5452 llvm_unreachable("Reduction kind is not set")::llvm::llvm_unreachable_internal("Reduction kind is not set"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5452)
;
5453 }
5454 return Flags;
5455 }
5456 };
5457
5458 WeakTrackingVH ReductionRoot;
5459
5460 /// The operation data of the reduction operation.
5461 OperationData ReductionData;
5462
5463 /// The operation data of the values we perform a reduction on.
5464 OperationData ReducedValueData;
5465
5466 /// Should we model this reduction as a pairwise reduction tree or a tree that
5467 /// splits the vector in halves and adds those halves.
5468 bool IsPairwiseReduction = false;
5469
5470 /// Checks if the ParentStackElem.first should be marked as a reduction
5471 /// operation with an extra argument or as extra argument itself.
5472 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5473 Value *ExtraArg) {
5474 if (ExtraArgs.count(ParentStackElem.first)) {
5475 ExtraArgs[ParentStackElem.first] = nullptr;
5476 // We ran into something like:
5477 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5478 // The whole ParentStackElem.first should be considered as an extra value
5479 // in this case.
5480 // Do not perform analysis of remaining operands of ParentStackElem.first
5481 // instruction, this whole instruction is an extra argument.
5482 ParentStackElem.second = ParentStackElem.first->getNumOperands();
5483 } else {
5484 // We ran into something like:
5485 // ParentStackElem.first += ... + ExtraArg + ...
5486 ExtraArgs[ParentStackElem.first] = ExtraArg;
5487 }
5488 }
5489
5490 static OperationData getOperationData(Value *V) {
5491 if (!V)
5492 return OperationData();
5493
5494 Value *LHS;
5495 Value *RHS;
5496 if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5497 return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5498 RK_Arithmetic);
5499 }
5500 if (auto *Select = dyn_cast<SelectInst>(V)) {
5501 // Look for a min/max pattern.
5502 if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5503 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5504 } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5505 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5506 } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5507 m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5508 return OperationData(
5509 Instruction::FCmp, LHS, RHS, RK_Min,
5510 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5511 } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5512 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5513 } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5514 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5515 } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5516 m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5517 return OperationData(
5518 Instruction::FCmp, LHS, RHS, RK_Max,
5519 cast<Instruction>(Select->getCondition())->hasNoNaNs());
5520 } else {
5521 // Try harder: look for min/max pattern based on instructions producing
5522 // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2).
5523 // During the intermediate stages of SLP, it's very common to have
5524 // pattern like this (since optimizeGatherSequence is run only once
5525 // at the end):
5526 // %1 = extractelement <2 x i32> %a, i32 0
5527 // %2 = extractelement <2 x i32> %a, i32 1
5528 // %cond = icmp sgt i32 %1, %2
5529 // %3 = extractelement <2 x i32> %a, i32 0
5530 // %4 = extractelement <2 x i32> %a, i32 1
5531 // %select = select i1 %cond, i32 %3, i32 %4
5532 CmpInst::Predicate Pred;
5533 Instruction *L1;
5534 Instruction *L2;
5535
5536 LHS = Select->getTrueValue();
5537 RHS = Select->getFalseValue();
5538 Value *Cond = Select->getCondition();
5539
5540 // TODO: Support inverse predicates.
5541 if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) {
5542 if (!isa<ExtractElementInst>(RHS) ||
5543 !L2->isIdenticalTo(cast<Instruction>(RHS)))
5544 return OperationData(V);
5545 } else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) {
5546 if (!isa<ExtractElementInst>(LHS) ||
5547 !L1->isIdenticalTo(cast<Instruction>(LHS)))
5548 return OperationData(V);
5549 } else {
5550 if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS))
5551 return OperationData(V);
5552 if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) ||
5553 !L1->isIdenticalTo(cast<Instruction>(LHS)) ||
5554 !L2->isIdenticalTo(cast<Instruction>(RHS)))
5555 return OperationData(V);
5556 }
5557 switch (Pred) {
5558 default:
5559 return OperationData(V);
5560
5561 case CmpInst::ICMP_ULT:
5562 case CmpInst::ICMP_ULE:
5563 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5564
5565 case CmpInst::ICMP_SLT:
5566 case CmpInst::ICMP_SLE:
5567 return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5568
5569 case CmpInst::FCMP_OLT:
5570 case CmpInst::FCMP_OLE:
5571 case CmpInst::FCMP_ULT:
5572 case CmpInst::FCMP_ULE:
5573 return OperationData(Instruction::FCmp, LHS, RHS, RK_Min,
5574 cast<Instruction>(Cond)->hasNoNaNs());
5575
5576 case CmpInst::ICMP_UGT:
5577 case CmpInst::ICMP_UGE:
5578 return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5579
5580 case CmpInst::ICMP_SGT:
5581 case CmpInst::ICMP_SGE:
5582 return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5583
5584 case CmpInst::FCMP_OGT:
5585 case CmpInst::FCMP_OGE:
5586 case CmpInst::FCMP_UGT:
5587 case CmpInst::FCMP_UGE:
5588 return OperationData(Instruction::FCmp, LHS, RHS, RK_Max,
5589 cast<Instruction>(Cond)->hasNoNaNs());
5590 }
5591 }
5592 }
5593 return OperationData(V);
5594 }
5595
5596public:
5597 HorizontalReduction() = default;
5598
5599 /// Try to find a reduction tree.
5600 bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5601 assert((!Phi || is_contained(Phi->operands(), B)) &&(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5602, __PRETTY_FUNCTION__))
5602 "Thi phi needs to use the binary operator")(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5602, __PRETTY_FUNCTION__))
;
5603
5604 ReductionData = getOperationData(B);
5605
5606 // We could have a initial reductions that is not an add.
5607 // r *= v1 + v2 + v3 + v4
5608 // In such a case start looking for a tree rooted in the first '+'.
5609 if (Phi) {
5610 if (ReductionData.getLHS() == Phi) {
5611 Phi = nullptr;
5612 B = dyn_cast<Instruction>(ReductionData.getRHS());
5613 ReductionData = getOperationData(B);
5614 } else if (ReductionData.getRHS() == Phi) {
5615 Phi = nullptr;
5616 B = dyn_cast<Instruction>(ReductionData.getLHS());
5617 ReductionData = getOperationData(B);
5618 }
5619 }
5620
5621 if (!ReductionData.isVectorizable(B))
5622 return false;
5623
5624 Type *Ty = B->getType();
5625 if (!isValidElementType(Ty))
5626 return false;
5627 if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy())
5628 return false;
5629
5630 ReducedValueData.clear();
5631 ReductionRoot = B;
5632
5633 // Post order traverse the reduction tree starting at B. We only handle true
5634 // trees containing only binary operators.
5635 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
5636 Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5637 ReductionData.initReductionOps(ReductionOps);
5638 while (!Stack.empty()) {
5639 Instruction *TreeN = Stack.back().first;
5640 unsigned EdgeToVist = Stack.back().second++;
5641 OperationData OpData = getOperationData(TreeN);
5642 bool IsReducedValue = OpData != ReductionData;
5643
5644 // Postorder vist.
5645 if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5646 if (IsReducedValue)
5647 ReducedVals.push_back(TreeN);
5648 else {
5649 auto I = ExtraArgs.find(TreeN);
5650 if (I != ExtraArgs.end() && !I->second) {
5651 // Check if TreeN is an extra argument of its parent operation.
5652 if (Stack.size() <= 1) {
5653 // TreeN can't be an extra argument as it is a root reduction
5654 // operation.
5655 return false;
5656 }
5657 // Yes, TreeN is an extra argument, do not add it to a list of
5658 // reduction operations.
5659 // Stack[Stack.size() - 2] always points to the parent operation.
5660 markExtraArg(Stack[Stack.size() - 2], TreeN);
5661 ExtraArgs.erase(TreeN);
5662 } else
5663 ReductionData.addReductionOps(TreeN, ReductionOps);
5664 }
5665 // Retract.
5666 Stack.pop_back();
5667 continue;
5668 }
5669
5670 // Visit left or right.
5671 Value *NextV = TreeN->getOperand(EdgeToVist);
5672 if (NextV != Phi) {
5673 auto *I = dyn_cast<Instruction>(NextV);
5674 OpData = getOperationData(I);
5675 // Continue analysis if the next operand is a reduction operation or
5676 // (possibly) a reduced value. If the reduced value opcode is not set,
5677 // the first met operation != reduction operation is considered as the
5678 // reduced value class.
5679 if (I && (!ReducedValueData || OpData == ReducedValueData ||
5680 OpData == ReductionData)) {
5681 const bool IsReductionOperation = OpData == ReductionData;
5682 // Only handle trees in the current basic block.
5683 if (!ReductionData.hasSameParent(I, B->getParent(),
5684 IsReductionOperation)) {
5685 // I is an extra argument for TreeN (its parent operation).
5686 markExtraArg(Stack.back(), I);
5687 continue;
5688 }
5689
5690 // Each tree node needs to have minimal number of users except for the
5691 // ultimate reduction.
5692 if (!ReductionData.hasRequiredNumberOfUses(I,
5693 OpData == ReductionData) &&
5694 I != B) {
5695 // I is an extra argument for TreeN (its parent operation).
5696 markExtraArg(Stack.back(), I);
5697 continue;
5698 }
5699
5700 if (IsReductionOperation) {
5701 // We need to be able to reassociate the reduction operations.
5702 if (!OpData.isAssociative(I)) {
5703 // I is an extra argument for TreeN (its parent operation).
5704 markExtraArg(Stack.back(), I);
5705 continue;
5706 }
5707 } else if (ReducedValueData &&
5708 ReducedValueData != OpData) {
5709 // Make sure that the opcodes of the operations that we are going to
5710 // reduce match.
5711 // I is an extra argument for TreeN (its parent operation).
5712 markExtraArg(Stack.back(), I);
5713 continue;
5714 } else if (!ReducedValueData)
5715 ReducedValueData = OpData;
5716
5717 Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5718 continue;
5719 }
5720 }
5721 // NextV is an extra argument for TreeN (its parent operation).
5722 markExtraArg(Stack.back(), NextV);
5723 }
5724 return true;
5725 }
5726
5727 /// Attempt to vectorize the tree found by
5728 /// matchAssociativeReduction.
5729 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5730 if (ReducedVals.empty())
5731 return false;
5732
5733 // If there is a sufficient number of reduction values, reduce
5734 // to a nearby power-of-2. Can safely generate oversized
5735 // vectors and rely on the backend to split them to legal sizes.
5736 unsigned NumReducedVals = ReducedVals.size();
5737 if (NumReducedVals < 4)
5738 return false;
5739
5740 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5741
5742 Value *VectorizedTree = nullptr;
5743 IRBuilder<> Builder(cast<Instruction>(ReductionRoot));
5744 FastMathFlags Unsafe;
5745 Unsafe.setFast();
5746 Builder.setFastMathFlags(Unsafe);
5747 unsigned i = 0;
5748
5749 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5750 // The same extra argument may be used several time, so log each attempt
5751 // to use it.
5752 for (auto &Pair : ExtraArgs) {
5753 assert(Pair.first && "DebugLoc must be set.")((Pair.first && "DebugLoc must be set.") ? static_cast
<void> (0) : __assert_fail ("Pair.first && \"DebugLoc must be set.\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5753, __PRETTY_FUNCTION__))
;
5754 ExternallyUsedValues[Pair.second].push_back(Pair.first);
5755 }
5756 // The reduction root is used as the insertion point for new instructions,
5757 // so set it as externally used to prevent it from being deleted.
5758 ExternallyUsedValues[ReductionRoot];
5759 SmallVector<Value *, 16> IgnoreList;
5760 for (auto &V : ReductionOps)
5761 IgnoreList.append(V.begin(), V.end());
5762 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5763 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5764 V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5765 Optional<ArrayRef<unsigned>> Order = V.bestOrder();
5766 // TODO: Handle orders of size less than number of elements in the vector.
5767 if (Order && Order->size() == VL.size()) {
5768 // TODO: reorder tree nodes without tree rebuilding.
5769 SmallVector<Value *, 4> ReorderedOps(VL.size());
5770 llvm::transform(*Order, ReorderedOps.begin(),
5771 [VL](const unsigned Idx) { return VL[Idx]; });
5772 V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList);
5773 }
5774 if (V.isTreeTinyAndNotFullyVectorizable())
5775 break;
5776
5777 V.computeMinimumValueSizes();
5778
5779 // Estimate cost.
5780 int TreeCost = V.getTreeCost();
5781 int ReductionCost = getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5782 int Cost = TreeCost + ReductionCost;
5783 if (Cost >= -SLPCostThreshold) {
5784 V.getORE()->emit([&]() {
5785 return OptimizationRemarkMissed(
5786 SV_NAME"slp-vectorizer", "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5787 << "Vectorizing horizontal reduction is possible"
5788 << "but not beneficial with cost "
5789 << ore::NV("Cost", Cost) << " and threshold "
5790 << ore::NV("Threshold", -SLPCostThreshold);
5791 });
5792 break;
5793 }
5794
5795 LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
5796 << Cost << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
;
5797 V.getORE()->emit([&]() {
5798 return OptimizationRemark(
5799 SV_NAME"slp-vectorizer", "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5800 << "Vectorized horizontal reduction with cost "
5801 << ore::NV("Cost", Cost) << " and with tree size "
5802 << ore::NV("TreeSize", V.getTreeSize());
5803 });
5804
5805 // Vectorize a tree.
5806 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5807 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5808
5809 // Emit a reduction.
5810 Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
5811 Value *ReducedSubTree =
5812 emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5813 if (VectorizedTree) {
5814 Builder.SetCurrentDebugLocation(Loc);
5815 OperationData VectReductionData(ReductionData.getOpcode(),
5816 VectorizedTree, ReducedSubTree,
5817 ReductionData.getKind());
5818 VectorizedTree =
5819 VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5820 } else
5821 VectorizedTree = ReducedSubTree;
5822 i += ReduxWidth;
5823 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5824 }
5825
5826 if (VectorizedTree) {
5827 // Finish the reduction.
5828 for (; i < NumReducedVals; ++i) {
5829 auto *I = cast<Instruction>(ReducedVals[i]);
5830 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5831 OperationData VectReductionData(ReductionData.getOpcode(),
5832 VectorizedTree, I,
5833 ReductionData.getKind());
5834 VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5835 }
5836 for (auto &Pair : ExternallyUsedValues) {
5837 // Add each externally used value to the final reduction.
5838 for (auto *I : Pair.second) {
5839 Builder.SetCurrentDebugLocation(I->getDebugLoc());
5840 OperationData VectReductionData(ReductionData.getOpcode(),
5841 VectorizedTree, Pair.first,
5842 ReductionData.getKind());
5843 VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5844 }
5845 }
5846 // Update users.
5847 ReductionRoot->replaceAllUsesWith(VectorizedTree);
5848 }
5849 return VectorizedTree != nullptr;
5850 }
5851
5852 unsigned numReductionValues() const {
5853 return ReducedVals.size();
5854 }
5855
5856private:
5857 /// Calculate the cost of a reduction.
5858 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5859 unsigned ReduxWidth) {
5860 Type *ScalarTy = FirstReducedVal->getType();
5861 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5862
5863 int PairwiseRdxCost;
5864 int SplittingRdxCost;
5865 switch (ReductionData.getKind()) {
5866 case RK_Arithmetic:
5867 PairwiseRdxCost =
5868 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5869 /*IsPairwiseForm=*/true);
5870 SplittingRdxCost =
5871 TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5872 /*IsPairwiseForm=*/false);
5873 break;
5874 case RK_Min:
5875 case RK_Max:
5876 case RK_UMin:
5877 case RK_UMax: {
5878 Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5879 bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5880 ReductionData.getKind() == RK_UMax;
5881 PairwiseRdxCost =
5882 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5883 /*IsPairwiseForm=*/true, IsUnsigned);
5884 SplittingRdxCost =
5885 TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5886 /*IsPairwiseForm=*/false, IsUnsigned);
5887 break;
5888 }
5889 case RK_None:
5890 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5890)
;
5891 }
5892
5893 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5894 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5895
5896 int ScalarReduxCost;
5897 switch (ReductionData.getKind()) {
5898 case RK_Arithmetic:
5899 ScalarReduxCost =
5900 TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5901 break;
5902 case RK_Min:
5903 case RK_Max:
5904 case RK_UMin:
5905 case RK_UMax:
5906 ScalarReduxCost =
5907 TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5908 TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5909 CmpInst::makeCmpResultType(ScalarTy));
5910 break;
5911 case RK_None:
5912 llvm_unreachable("Expected arithmetic or min/max reduction operation")::llvm::llvm_unreachable_internal("Expected arithmetic or min/max reduction operation"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5912)
;
5913 }
5914 ScalarReduxCost *= (ReduxWidth - 1);
5915
5916 LLVM_DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5917 << " for reduction that starts with " << *FirstReducedValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5918 << " (It is a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5919 << (IsPairwiseReduction ? "pairwise" : "splitting")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
5920 << " reduction)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
;
5921
5922 return VecReduxCost - ScalarReduxCost;
5923 }
5924
5925 /// Emit a horizontal reduction of the vectorized value.
5926 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5927 unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5928 assert(VectorizedValue && "Need to have a vectorized tree node")((VectorizedValue && "Need to have a vectorized tree node"
) ? static_cast<void> (0) : __assert_fail ("VectorizedValue && \"Need to have a vectorized tree node\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5928, __PRETTY_FUNCTION__))
;
5929 assert(isPowerOf2_32(ReduxWidth) &&((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5930, __PRETTY_FUNCTION__))
5930 "We only handle power-of-two reductions for now")((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5930, __PRETTY_FUNCTION__))
;
5931
5932 if (!IsPairwiseReduction)
5933 return createSimpleTargetReduction(
5934 Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5935 ReductionData.getFlags(), ReductionOps.back());
5936
5937 Value *TmpVec = VectorizedValue;
5938 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5939 Value *LeftMask =
5940 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5941 Value *RightMask =
5942 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5943
5944 Value *LeftShuf = Builder.CreateShuffleVector(
5945 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5946 Value *RightShuf = Builder.CreateShuffleVector(
5947 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5948 "rdx.shuf.r");
5949 OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5950 RightShuf, ReductionData.getKind());
5951 TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5952 }
5953
5954 // The result is in the first element of the vector.
5955 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5956 }
5957};
5958
5959} // end anonymous namespace
5960
5961/// Recognize construction of vectors like
5962/// %ra = insertelement <4 x float> undef, float %s0, i32 0
5963/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5964/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5965/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5966/// starting from the last insertelement instruction.
5967///
5968/// Returns true if it matches
5969static bool findBuildVector(InsertElementInst *LastInsertElem,
5970 TargetTransformInfo *TTI,
5971 SmallVectorImpl<Value *> &BuildVectorOpds,
5972 int &UserCost) {
5973 UserCost = 0;
5974 Value *V = nullptr;
5975 do {
5976 if (auto *CI = dyn_cast<ConstantInt>(LastInsertElem->getOperand(2))) {
5977 UserCost += TTI->getVectorInstrCost(Instruction::InsertElement,
5978 LastInsertElem->getType(),
5979 CI->getZExtValue());
5980 }
5981 BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5982 V = LastInsertElem->getOperand(0);
5983 if (isa<UndefValue>(V))
5984 break;
5985 LastInsertElem = dyn_cast<InsertElementInst>(V);
5986 if (!LastInsertElem || !LastInsertElem->hasOneUse())
5987 return false;
5988 } while (true);
5989 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5990 return true;
5991}
5992
5993/// Like findBuildVector, but looks for construction of aggregate.
5994///
5995/// \return true if it matches.
5996static bool findBuildAggregate(InsertValueInst *IV,
5997 SmallVectorImpl<Value *> &BuildVectorOpds) {
5998 Value *V;
5999 do {
6000 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
6001 V = IV->getAggregateOperand();
6002 if (isa<UndefValue>(V))
6003 break;
6004 IV = dyn_cast<InsertValueInst>(V);
6005 if (!IV || !IV->hasOneUse())
6006 return false;
6007 } while (true);
6008 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
6009 return true;
6010}
6011
6012static bool PhiTypeSorterFunc(Value *V, Value *V2) {
6013 return V->getType() < V2->getType();
6014}
6015
6016/// Try and get a reduction value from a phi node.
6017///
6018/// Given a phi node \p P in a block \p ParentBB, consider possible reductions
6019/// if they come from either \p ParentBB or a containing loop latch.
6020///
6021/// \returns A candidate reduction value if possible, or \code nullptr \endcode
6022/// if not possible.
6023static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
6024 BasicBlock *ParentBB, LoopInfo *LI) {
6025 // There are situations where the reduction value is not dominated by the
6026 // reduction phi. Vectorizing such cases has been reported to cause
6027 // miscompiles. See PR25787.
6028 auto DominatedReduxValue = [&](Value *R) {
6029 return isa<Instruction>(R) &&
6030 DT->dominates(P->getParent(), cast<Instruction>(R)->getParent());
6031 };
6032
6033 Value *Rdx = nullptr;
6034
6035 // Return the incoming value if it comes from the same BB as the phi node.
6036 if (P->getIncomingBlock(0) == ParentBB) {
6037 Rdx = P->getIncomingValue(0);
6038 } else if (P->getIncomingBlock(1) == ParentBB) {
6039 Rdx = P->getIncomingValue(1);
6040 }
6041
6042 if (Rdx && DominatedReduxValue(Rdx))
6043 return Rdx;
6044
6045 // Otherwise, check whether we have a loop latch to look at.
6046 Loop *BBL = LI->getLoopFor(ParentBB);
6047 if (!BBL)
6048 return nullptr;
6049 BasicBlock *BBLatch = BBL->getLoopLatch();
6050 if (!BBLatch)
6051 return nullptr;
6052
6053 // There is a loop latch, return the incoming value if it comes from
6054 // that. This reduction pattern occasionally turns up.
6055 if (P->getIncomingBlock(0) == BBLatch) {
6056 Rdx = P->getIncomingValue(0);
6057 } else if (P->getIncomingBlock(1) == BBLatch) {
6058 Rdx = P->getIncomingValue(1);
6059 }
6060
6061 if (Rdx && DominatedReduxValue(Rdx))
6062 return Rdx;
6063
6064 return nullptr;
6065}
6066
6067/// Attempt to reduce a horizontal reduction.
6068/// If it is legal to match a horizontal reduction feeding the phi node \a P
6069/// with reduction operators \a Root (or one of its operands) in a basic block
6070/// \a BB, then check if it can be done. If horizontal reduction is not found
6071/// and root instruction is a binary operation, vectorization of the operands is
6072/// attempted.
6073/// \returns true if a horizontal reduction was matched and reduced or operands
6074/// of one of the binary instruction were vectorized.
6075/// \returns false if a horizontal reduction was not matched (or not possible)
6076/// or no vectorization of any binary operation feeding \a Root instruction was
6077/// performed.
6078static bool tryToVectorizeHorReductionOrInstOperands(
6079 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
6080 TargetTransformInfo *TTI,
6081 const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
6082 if (!ShouldVectorizeHor)
6083 return false;
6084
6085 if (!Root)
6086 return false;
6087
6088 if (Root->getParent() != BB || isa<PHINode>(Root))
6089 return false;
6090 // Start analysis starting from Root instruction. If horizontal reduction is
6091 // found, try to vectorize it. If it is not a horizontal reduction or
6092 // vectorization is not possible or not effective, and currently analyzed
6093 // instruction is a binary operation, try to vectorize the operands, using
6094 // pre-order DFS traversal order. If the operands were not vectorized, repeat
6095 // the same procedure considering each operand as a possible root of the
6096 // horizontal reduction.
6097 // Interrupt the process if the Root instruction itself was vectorized or all
6098 // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
6099 SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
6100 SmallPtrSet<Value *, 8> VisitedInstrs;
6101 bool Res = false;
6102 while (!Stack.empty()) {
6103 Value *V;
6104 unsigned Level;
6105 std::tie(V, Level) = Stack.pop_back_val();
6106 if (!V)
6107 continue;
6108 auto *Inst = dyn_cast<Instruction>(V);
6109 if (!Inst)
6110 continue;
6111 auto *BI = dyn_cast<BinaryOperator>(Inst);
6112 auto *SI = dyn_cast<SelectInst>(Inst);
6113 if (BI || SI) {
6114 HorizontalReduction HorRdx;
6115 if (HorRdx.matchAssociativeReduction(P, Inst)) {
6116 if (HorRdx.tryToReduce(R, TTI)) {
6117 Res = true;
6118 // Set P to nullptr to avoid re-analysis of phi node in
6119 // matchAssociativeReduction function unless this is the root node.
6120 P = nullptr;
6121 continue;
6122 }
6123 }
6124 if (P && BI) {
6125 Inst = dyn_cast<Instruction>(BI->getOperand(0));
6126 if (Inst == P)
6127 Inst = dyn_cast<Instruction>(BI->getOperand(1));
6128 if (!Inst) {
6129 // Set P to nullptr to avoid re-analysis of phi node in
6130 // matchAssociativeReduction function unless this is the root node.
6131 P = nullptr;
6132 continue;
6133 }
6134 }
6135 }
6136 // Set P to nullptr to avoid re-analysis of phi node in
6137 // matchAssociativeReduction function unless this is the root node.
6138 P = nullptr;
6139 if (Vectorize(Inst, R)) {
6140 Res = true;
6141 continue;
6142 }
6143
6144 // Try to vectorize operands.
6145 // Continue analysis for the instruction from the same basic block only to
6146 // save compile time.
6147 if (++Level < RecursionMaxDepth)
6148 for (auto *Op : Inst->operand_values())
6149 if (VisitedInstrs.insert(Op).second)
6150 if (auto *I = dyn_cast<Instruction>(Op))
6151 if (!isa<PHINode>(I) && I->getParent() == BB)
6152 Stack.emplace_back(Op, Level);
6153 }
6154 return Res;
6155}
6156
6157bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
6158 BasicBlock *BB, BoUpSLP &R,
6159 TargetTransformInfo *TTI) {
6160 if (!V)
6161 return false;
6162 auto *I = dyn_cast<Instruction>(V);
6163 if (!I)
6164 return false;
6165
6166 if (!isa<BinaryOperator>(I))
6167 P = nullptr;
6168 // Try to match and vectorize a horizontal reduction.
6169 auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
6170 return tryToVectorize(I, R);
6171 };
6172 return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
6173 ExtraVectorization);
6174}
6175
6176bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
6177 BasicBlock *BB, BoUpSLP &R) {
6178 const DataLayout &DL = BB->getModule()->getDataLayout();
6179 if (!R.canMapToVector(IVI->getType(), DL))
6180 return false;
6181
6182 SmallVector<Value *, 16> BuildVectorOpds;
6183 if (!findBuildAggregate(IVI, BuildVectorOpds))
6184 return false;
6185
6186 LLVM_DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: array mappable to vector: " <<
*IVI << "\n"; } } while (false)
;
6187 // Aggregate value is unlikely to be processed in vector register, we need to
6188 // extract scalars into scalar registers, so NeedExtraction is set true.
6189 return tryToVectorizeList(BuildVectorOpds, R);
6190}
6191
6192bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
6193 BasicBlock *BB, BoUpSLP &R) {
6194 int UserCost;
6195 SmallVector<Value *, 16> BuildVectorOpds;
6196 if (!findBuildVector(IEI, TTI, BuildVectorOpds, UserCost) ||
6197 (llvm::all_of(BuildVectorOpds,
6198 [](Value *V) { return isa<ExtractElementInst>(V); }) &&
6199 isShuffle(BuildVectorOpds)))
6200 return false;
6201
6202 // Vectorize starting with the build vector operands ignoring the BuildVector
6203 // instructions for the purpose of scheduling and user extraction.
6204 return tryToVectorizeList(BuildVectorOpds, R, UserCost);
6205}
6206
6207bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
6208 BoUpSLP &R) {
6209 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
6210 return true;
6211
6212 bool OpsChanged = false;
6213 for (int Idx = 0; Idx < 2; ++Idx) {
6214 OpsChanged |=
6215 vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
6216 }
6217 return OpsChanged;
6218}
6219
6220bool SLPVectorizerPass::vectorizeSimpleInstructions(
6221 SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
6222 bool OpsChanged = false;
6223 for (auto &VH : reverse(Instructions)) {
6224 auto *I = dyn_cast_or_null<Instruction>(VH);
6225 if (!I)
6226 continue;
6227 if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
6228 OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
6229 else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
6230 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
6231 else if (auto *CI = dyn_cast<CmpInst>(I))
6232 OpsChanged |= vectorizeCmpInst(CI, BB, R);
6233 }
6234 Instructions.clear();
6235 return OpsChanged;
6236}
6237
6238bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
6239 bool Changed = false;
6240 SmallVector<Value *, 4> Incoming;
6241 SmallPtrSet<Value *, 16> VisitedInstrs;
6242
6243 bool HaveVectorizedPhiNodes = true;
6244 while (HaveVectorizedPhiNodes) {
6245 HaveVectorizedPhiNodes = false;
6246
6247 // Collect the incoming values from the PHIs.
6248 Incoming.clear();
6249 for (Instruction &I : *BB) {
6250 PHINode *P = dyn_cast<PHINode>(&I);
6251 if (!P)
6252 break;
6253
6254 if (!VisitedInstrs.count(P))
6255 Incoming.push_back(P);
6256 }
6257
6258 // Sort by type.
6259 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
6260
6261 // Try to vectorize elements base on their type.
6262 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
6263 E = Incoming.end();
6264 IncIt != E;) {
6265
6266 // Look for the next elements with the same type.
6267 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
6268 while (SameTypeIt != E &&
6269 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
6270 VisitedInstrs.insert(*SameTypeIt);
6271 ++SameTypeIt;
6272 }
6273
6274 // Try to vectorize them.
6275 unsigned NumElts = (SameTypeIt - IncIt);
6276 LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize starting at PHIs ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false)
6277 << NumElts << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false)
;
6278 // The order in which the phi nodes appear in the program does not matter.
6279 // So allow tryToVectorizeList to reorder them if it is beneficial. This
6280 // is done when there are exactly two elements since tryToVectorizeList
6281 // asserts that there are only two values when AllowReorder is true.
6282 bool AllowReorder = NumElts == 2;
6283 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
6284 /*UserCost=*/0, AllowReorder)) {
6285 // Success start over because instructions might have been changed.
6286 HaveVectorizedPhiNodes = true;
6287 Changed = true;
6288 break;
6289 }
6290
6291 // Start over at the next instruction of a different type (or the end).
6292 IncIt = SameTypeIt;
6293 }
6294 }
6295
6296 VisitedInstrs.clear();
6297
6298 SmallVector<WeakVH, 8> PostProcessInstructions;
6299 SmallDenseSet<Instruction *, 4> KeyNodes;
6300 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
6301 // We may go through BB multiple times so skip the one we have checked.
6302 if (!VisitedInstrs.insert(&*it).second) {
6303 if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
6304 vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
6305 // We would like to start over since some instructions are deleted
6306 // and the iterator may become invalid value.
6307 Changed = true;
6308 it = BB->begin();
6309 e = BB->end();
6310 }
6311 continue;
6312 }
6313
6314 if (isa<DbgInfoIntrinsic>(it))
6315 continue;
6316
6317 // Try to vectorize reductions that use PHINodes.
6318 if (PHINode *P = dyn_cast<PHINode>(it)) {
6319 // Check that the PHI is a reduction PHI.
6320 if (P->getNumIncomingValues() != 2)
6321 return Changed;
6322
6323 // Try to match and vectorize a horizontal reduction.
6324 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
6325 TTI)) {
6326 Changed = true;
6327 it = BB->begin();
6328 e = BB->end();
6329 continue;
6330 }
6331 continue;
6332 }
6333
6334 // Ran into an instruction without users, like terminator, or function call
6335 // with ignored return value, store. Ignore unused instructions (basing on
6336 // instruction type, except for CallInst and InvokeInst).
6337 if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
6338 isa<InvokeInst>(it))) {
6339 KeyNodes.insert(&*it);
6340 bool OpsChanged = false;
6341 if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
6342 for (auto *V : it->operand_values()) {
6343 // Try to match and vectorize a horizontal reduction.
6344 OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
6345 }
6346 }
6347 // Start vectorization of post-process list of instructions from the
6348 // top-tree instructions to try to vectorize as many instructions as
6349 // possible.
6350 OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
6351 if (OpsChanged) {
6352 // We would like to start over since some instructions are deleted
6353 // and the iterator may become invalid value.
6354 Changed = true;
6355 it = BB->begin();
6356 e = BB->end();
6357 continue;
6358 }
6359 }
6360
6361 if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
6362 isa<InsertValueInst>(it))
6363 PostProcessInstructions.push_back(&*it);
6364 }
6365
6366 return Changed;
6367}
6368
6369bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
6370 auto Changed = false;
6371 for (auto &Entry : GEPs) {
6372 // If the getelementptr list has fewer than two elements, there's nothing
6373 // to do.
6374 if (Entry.second.size() < 2)
6375 continue;
6376
6377 LLVM_DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
6378 << Entry.second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
;
6379
6380 // We process the getelementptr list in chunks of 16 (like we do for
6381 // stores) to minimize compile-time.
6382 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
6383 auto Len = std::min<unsigned>(BE - BI, 16);
6384 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
6385
6386 // Initialize a set a candidate getelementptrs. Note that we use a
6387 // SetVector here to preserve program order. If the index computations
6388 // are vectorizable and begin with loads, we want to minimize the chance
6389 // of having to reorder them later.
6390 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
6391
6392 // Some of the candidates may have already been vectorized after we
6393 // initially collected them. If so, the WeakTrackingVHs will have
6394 // nullified the
6395 // values, so remove them from the set of candidates.
6396 Candidates.remove(nullptr);
6397
6398 // Remove from the set of candidates all pairs of getelementptrs with
6399 // constant differences. Such getelementptrs are likely not good
6400 // candidates for vectorization in a bottom-up phase since one can be
6401 // computed from the other. We also ensure all candidate getelementptr
6402 // indices are unique.
6403 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
6404 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
6405 if (!Candidates.count(GEPI))
6406 continue;
6407 auto *SCEVI = SE->getSCEV(GEPList[I]);
6408 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
6409 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
6410 auto *SCEVJ = SE->getSCEV(GEPList[J]);
6411 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
6412 Candidates.remove(GEPList[I]);
6413 Candidates.remove(GEPList[J]);
6414 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
6415 Candidates.remove(GEPList[J]);
6416 }
6417 }
6418 }
6419
6420 // We break out of the above computation as soon as we know there are
6421 // fewer than two candidates remaining.
6422 if (Candidates.size() < 2)
6423 continue;
6424
6425 // Add the single, non-constant index of each candidate to the bundle. We
6426 // ensured the indices met these constraints when we originally collected
6427 // the getelementptrs.
6428 SmallVector<Value *, 16> Bundle(Candidates.size());
6429 auto BundleIndex = 0u;
6430 for (auto *V : Candidates) {
6431 auto *GEP = cast<GetElementPtrInst>(V);
6432 auto *GEPIdx = GEP->idx_begin()->get();
6433 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx))((GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx
)) ? static_cast<void> (0) : __assert_fail ("GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)"
, "/build/llvm-toolchain-snapshot-8~svn348900/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 6433, __PRETTY_FUNCTION__))
;
6434 Bundle[BundleIndex++] = GEPIdx;
6435 }
6436
6437 // Try and vectorize the indices. We are currently only interested in
6438 // gather-like cases of the form:
6439 //
6440 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
6441 //
6442 // where the loads of "a", the loads of "b", and the subtractions can be
6443 // performed in parallel. It's likely that detecting this pattern in a
6444 // bottom-up phase will be simpler and less costly than building a
6445 // full-blown top-down phase beginning at the consecutive loads.
6446 Changed |= tryToVectorizeList(Bundle, R);
6447 }
6448 }
6449 return Changed;
6450}
6451
6452bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
6453 bool Changed = false;
6454 // Attempt to sort and vectorize each of the store-groups.
6455 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
6456 ++it) {
6457 if (it->second.size() < 2)
6458 continue;
6459
6460 LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
6461 << it->second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
;
6462
6463 // Process the stores in chunks of 16.
6464 // TODO: The limit of 16 inhibits greater vectorization factors.
6465 // For example, AVX2 supports v32i8. Increasing this limit, however,
6466 // may cause a significant compile-time increase.
6467 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI += 16) {
6468 unsigned Len = std::min<unsigned>(CE - CI, 16);
6469 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
6470 }
6471 }
6472 return Changed;
6473}
6474
6475char SLPVectorizer::ID = 0;
6476
6477static const char lv_name[] = "SLP Vectorizer";
6478
6479INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)static void *initializeSLPVectorizerPassOnce(PassRegistry &
Registry) {
6480INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
6481INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
6482INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
6483INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
6484INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry);
6485INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
6486INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
6487INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "slp-vectorizer", &
SLPVectorizer::ID, PassInfo::NormalCtor_t(callDefaultCtor<
SLPVectorizer>), false, false); Registry.registerPass(*PI,
true); return PI; } static llvm::once_flag InitializeSLPVectorizerPassFlag
; void llvm::initializeSLPVectorizerPass(PassRegistry &Registry
) { llvm::call_once(InitializeSLPVectorizerPassFlag, initializeSLPVectorizerPassOnce
, std::ref(Registry)); }
6488
6489Pass *llvm::createSLPVectorizerPass() { return new SLPVectorizer(); }