Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 3020, column 22
Called C++ object pointer is null

Annotated Source Code

1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10// stores that can be put together into vector-stores. Next, it attempts to
11// construct vectorizable tree using the use-def chains. If a profitable tree
12// was found, the SLP vectorizer performs vectorization on the tree.
13//
14// The pass is inspired by the work described in the paper:
15// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16//
17//===----------------------------------------------------------------------===//
18#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
19#include "llvm/ADT/Optional.h"
20#include "llvm/ADT/PostOrderIterator.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/CodeMetrics.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/LoopAccessAnalysis.h"
26#include "llvm/Analysis/ScalarEvolutionExpressions.h"
27#include "llvm/Analysis/ValueTracking.h"
28#include "llvm/Analysis/VectorUtils.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/Instructions.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NoFolder.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/Value.h"
38#include "llvm/IR/Verifier.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/CommandLine.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/raw_ostream.h"
43#include "llvm/Transforms/Vectorize.h"
44#include <algorithm>
45#include <memory>
46
47using namespace llvm;
48using namespace slpvectorizer;
49
50#define SV_NAME"slp-vectorizer" "slp-vectorizer"
51#define DEBUG_TYPE"SLP" "SLP"
52
53STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, false}
;
54
55static cl::opt<int>
56 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
57 cl::desc("Only vectorize if you gain more than this "
58 "number "));
59
60static cl::opt<bool>
61ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
62 cl::desc("Attempt to vectorize horizontal reductions"));
63
64static cl::opt<bool> ShouldStartVectorizeHorAtStore(
65 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
66 cl::desc(
67 "Attempt to vectorize horizontal reductions feeding into a store"));
68
69static cl::opt<int>
70MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
71 cl::desc("Attempt to vectorize for this register size in bits"));
72
73/// Limits the size of scheduling regions in a block.
74/// It avoid long compile times for _very_ large blocks where vector
75/// instructions are spread over a wide range.
76/// This limit is way higher than needed by real-world functions.
77static cl::opt<int>
78ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
79 cl::desc("Limit the size of the SLP scheduling region per block"));
80
81static cl::opt<int> MinVectorRegSizeOption(
82 "slp-min-reg-size", cl::init(128), cl::Hidden,
83 cl::desc("Attempt to vectorize for this register size in bits"));
84
85static cl::opt<unsigned> RecursionMaxDepth(
86 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
87 cl::desc("Limit the recursion depth when building a vectorizable tree"));
88
89static cl::opt<unsigned> MinTreeSize(
90 "slp-min-tree-size", cl::init(3), cl::Hidden,
91 cl::desc("Only vectorize small trees if they are fully vectorizable"));
92
93// Limit the number of alias checks. The limit is chosen so that
94// it has no negative effect on the llvm benchmarks.
95static const unsigned AliasedCheckLimit = 10;
96
97// Another limit for the alias checks: The maximum distance between load/store
98// instructions where alias checks are done.
99// This limit is useful for very large basic blocks.
100static const unsigned MaxMemDepDistance = 160;
101
102/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
103/// regions to be handled.
104static const int MinScheduleRegionSize = 16;
105
106/// \brief Predicate for the element types that the SLP vectorizer supports.
107///
108/// The most important thing to filter here are types which are invalid in LLVM
109/// vectors. We also filter target specific types which have absolutely no
110/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
111/// avoids spending time checking the cost model and realizing that they will
112/// be inevitably scalarized.
113static bool isValidElementType(Type *Ty) {
114 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
115 !Ty->isPPC_FP128Ty();
116}
117
118/// \returns true if all of the instructions in \p VL are in the same block or
119/// false otherwise.
120static bool allSameBlock(ArrayRef<Value *> VL) {
121 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
122 if (!I0)
123 return false;
124 BasicBlock *BB = I0->getParent();
125 for (int i = 1, e = VL.size(); i < e; i++) {
126 Instruction *I = dyn_cast<Instruction>(VL[i]);
127 if (!I)
128 return false;
129
130 if (BB != I->getParent())
131 return false;
132 }
133 return true;
134}
135
136/// \returns True if all of the values in \p VL are constants.
137static bool allConstant(ArrayRef<Value *> VL) {
138 for (Value *i : VL)
139 if (!isa<Constant>(i))
140 return false;
141 return true;
142}
143
144/// \returns True if all of the values in \p VL are identical.
145static bool isSplat(ArrayRef<Value *> VL) {
146 for (unsigned i = 1, e = VL.size(); i < e; ++i)
147 if (VL[i] != VL[0])
148 return false;
149 return true;
150}
151
152///\returns Opcode that can be clubbed with \p Op to create an alternate
153/// sequence which can later be merged as a ShuffleVector instruction.
154static unsigned getAltOpcode(unsigned Op) {
155 switch (Op) {
156 case Instruction::FAdd:
157 return Instruction::FSub;
158 case Instruction::FSub:
159 return Instruction::FAdd;
160 case Instruction::Add:
161 return Instruction::Sub;
162 case Instruction::Sub:
163 return Instruction::Add;
164 default:
165 return 0;
166 }
167}
168
169///\returns bool representing if Opcode \p Op can be part
170/// of an alternate sequence which can later be merged as
171/// a ShuffleVector instruction.
172static bool canCombineAsAltInst(unsigned Op) {
173 return Op == Instruction::FAdd || Op == Instruction::FSub ||
174 Op == Instruction::Sub || Op == Instruction::Add;
175}
176
177/// \returns ShuffleVector instruction if instructions in \p VL have
178/// alternate fadd,fsub / fsub,fadd/add,sub/sub,add sequence.
179/// (i.e. e.g. opcodes of fadd,fsub,fadd,fsub...)
180static unsigned isAltInst(ArrayRef<Value *> VL) {
181 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
182 unsigned Opcode = I0->getOpcode();
183 unsigned AltOpcode = getAltOpcode(Opcode);
184 for (int i = 1, e = VL.size(); i < e; i++) {
185 Instruction *I = dyn_cast<Instruction>(VL[i]);
186 if (!I || I->getOpcode() != ((i & 1) ? AltOpcode : Opcode))
187 return 0;
188 }
189 return Instruction::ShuffleVector;
190}
191
192/// \returns The opcode if all of the Instructions in \p VL have the same
193/// opcode, or zero.
194static unsigned getSameOpcode(ArrayRef<Value *> VL) {
195 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
196 if (!I0)
197 return 0;
198 unsigned Opcode = I0->getOpcode();
199 for (int i = 1, e = VL.size(); i < e; i++) {
200 Instruction *I = dyn_cast<Instruction>(VL[i]);
201 if (!I || Opcode != I->getOpcode()) {
202 if (canCombineAsAltInst(Opcode) && i == 1)
203 return isAltInst(VL);
204 return 0;
205 }
206 }
207 return Opcode;
208}
209
210/// Get the intersection (logical and) of all of the potential IR flags
211/// of each scalar operation (VL) that will be converted into a vector (I).
212/// Flag set: NSW, NUW, exact, and all of fast-math.
213static void propagateIRFlags(Value *I, ArrayRef<Value *> VL) {
214 if (auto *VecOp = dyn_cast<Instruction>(I)) {
215 if (auto *Intersection = dyn_cast<Instruction>(VL[0])) {
216 // Intersection is initialized to the 0th scalar,
217 // so start counting from index '1'.
218 for (int i = 1, e = VL.size(); i < e; ++i) {
219 if (auto *Scalar = dyn_cast<Instruction>(VL[i]))
220 Intersection->andIRFlags(Scalar);
221 }
222 VecOp->copyIRFlags(Intersection);
223 }
224 }
225}
226
227/// \returns true if all of the values in \p VL have the same type or false
228/// otherwise.
229static bool allSameType(ArrayRef<Value *> VL) {
230 Type *Ty = VL[0]->getType();
231 for (int i = 1, e = VL.size(); i < e; i++)
232 if (VL[i]->getType() != Ty)
233 return false;
234
235 return true;
236}
237
238/// \returns True if Extract{Value,Element} instruction extracts element Idx.
239static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
240 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 241, __PRETTY_FUNCTION__))
241 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 241, __PRETTY_FUNCTION__))
;
242 if (Opcode == Instruction::ExtractElement) {
243 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
244 return CI && CI->getZExtValue() == Idx;
245 } else {
246 ExtractValueInst *EI = cast<ExtractValueInst>(E);
247 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
248 }
249}
250
251/// \returns True if in-tree use also needs extract. This refers to
252/// possible scalar operand in vectorized instruction.
253static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
254 TargetLibraryInfo *TLI) {
255
256 unsigned Opcode = UserInst->getOpcode();
257 switch (Opcode) {
258 case Instruction::Load: {
259 LoadInst *LI = cast<LoadInst>(UserInst);
260 return (LI->getPointerOperand() == Scalar);
261 }
262 case Instruction::Store: {
263 StoreInst *SI = cast<StoreInst>(UserInst);
264 return (SI->getPointerOperand() == Scalar);
265 }
266 case Instruction::Call: {
267 CallInst *CI = cast<CallInst>(UserInst);
268 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
269 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
270 return (CI->getArgOperand(1) == Scalar);
271 }
272 }
273 default:
274 return false;
275 }
276}
277
278/// \returns the AA location that is being access by the instruction.
279static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
280 if (StoreInst *SI = dyn_cast<StoreInst>(I))
281 return MemoryLocation::get(SI);
282 if (LoadInst *LI = dyn_cast<LoadInst>(I))
283 return MemoryLocation::get(LI);
284 return MemoryLocation();
285}
286
287/// \returns True if the instruction is not a volatile or atomic load/store.
288static bool isSimple(Instruction *I) {
289 if (LoadInst *LI = dyn_cast<LoadInst>(I))
290 return LI->isSimple();
291 if (StoreInst *SI = dyn_cast<StoreInst>(I))
292 return SI->isSimple();
293 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
294 return !MI->isVolatile();
295 return true;
296}
297
298namespace llvm {
299namespace slpvectorizer {
300/// Bottom Up SLP Vectorizer.
301class BoUpSLP {
302public:
303 typedef SmallVector<Value *, 8> ValueList;
304 typedef SmallVector<Instruction *, 16> InstrList;
305 typedef SmallPtrSet<Value *, 16> ValueSet;
306 typedef SmallVector<StoreInst *, 8> StoreList;
307 typedef MapVector<Value *, SmallVector<DebugLoc, 2>> ExtraValueToDebugLocsMap;
308
309 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
310 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
311 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
312 const DataLayout *DL)
313 : NumLoadsWantToKeepOrder(0), NumLoadsWantToChangeOrder(0), F(Func),
314 SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC), DB(DB),
315 DL(DL), Builder(Se->getContext()) {
316 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
317 // Use the vector register size specified by the target unless overridden
318 // by a command-line option.
319 // TODO: It would be better to limit the vectorization factor based on
320 // data type rather than just register size. For example, x86 AVX has
321 // 256-bit registers, but it does not support integer operations
322 // at that width (that requires AVX2).
323 if (MaxVectorRegSizeOption.getNumOccurrences())
324 MaxVecRegSize = MaxVectorRegSizeOption;
325 else
326 MaxVecRegSize = TTI->getRegisterBitWidth(true);
327
328 MinVecRegSize = MinVectorRegSizeOption;
329 }
330
331 /// \brief Vectorize the tree that starts with the elements in \p VL.
332 /// Returns the vectorized root.
333 Value *vectorizeTree();
334 /// Vectorize the tree but with the list of externally used values \p
335 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
336 /// generated extractvalue instructions.
337 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
338
339 /// \returns the cost incurred by unwanted spills and fills, caused by
340 /// holding live values over call sites.
341 int getSpillCost();
342
343 /// \returns the vectorization cost of the subtree that starts at \p VL.
344 /// A negative number means that this is profitable.
345 int getTreeCost();
346
347 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
348 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
349 void buildTree(ArrayRef<Value *> Roots,
350 ArrayRef<Value *> UserIgnoreLst = None);
351 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
352 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
353 /// into account (anf updating it, if required) list of externally used
354 /// values stored in \p ExternallyUsedValues.
355 void buildTree(ArrayRef<Value *> Roots,
356 ExtraValueToDebugLocsMap &ExternallyUsedValues,
357 ArrayRef<Value *> UserIgnoreLst = None);
358
359 /// Clear the internal data structures that are created by 'buildTree'.
360 void deleteTree() {
361 VectorizableTree.clear();
362 ScalarToTreeEntry.clear();
363 MustGather.clear();
364 ExternalUses.clear();
365 NumLoadsWantToKeepOrder = 0;
366 NumLoadsWantToChangeOrder = 0;
367 for (auto &Iter : BlocksSchedules) {
368 BlockScheduling *BS = Iter.second.get();
369 BS->clear();
370 }
371 MinBWs.clear();
372 }
373
374 /// \brief Perform LICM and CSE on the newly generated gather sequences.
375 void optimizeGatherSequence();
376
377 /// \returns true if it is beneficial to reverse the vector order.
378 bool shouldReorder() const {
379 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
380 }
381
382 /// \return The vector element size in bits to use when vectorizing the
383 /// expression tree ending at \p V. If V is a store, the size is the width of
384 /// the stored value. Otherwise, the size is the width of the largest loaded
385 /// value reaching V. This method is used by the vectorizer to calculate
386 /// vectorization factors.
387 unsigned getVectorElementSize(Value *V);
388
389 /// Compute the minimum type sizes required to represent the entries in a
390 /// vectorizable tree.
391 void computeMinimumValueSizes();
392
393 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
394 unsigned getMaxVecRegSize() const {
395 return MaxVecRegSize;
396 }
397
398 // \returns minimum vector register size as set by cl::opt.
399 unsigned getMinVecRegSize() const {
400 return MinVecRegSize;
401 }
402
403 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
404 ///
405 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
406 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
407
408 /// \returns True if the VectorizableTree is both tiny and not fully
409 /// vectorizable. We do not vectorize such trees.
410 bool isTreeTinyAndNotFullyVectorizable();
411
412private:
413 struct TreeEntry;
414
415 /// \returns the cost of the vectorizable entry.
416 int getEntryCost(TreeEntry *E);
417
418 /// This is the recursive part of buildTree.
419 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth);
420
421 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
422 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
423 bool canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const;
424
425 /// Vectorize a single entry in the tree. VL icontains all isomorphic scalars
426 /// in order of its usage in a user program, for example ADD1, ADD2 and so on
427 /// or LOAD1 , LOAD2 etc.
428 Value *vectorizeTree(ArrayRef<Value *> VL, TreeEntry *E);
429
430 /// Vectorize a single entry in the tree, starting in \p VL.
431 Value *vectorizeTree(ArrayRef<Value *> VL);
432
433 /// \returns the pointer to the vectorized value if \p VL is already
434 /// vectorized, or NULL. They may happen in cycles.
435 Value *alreadyVectorized(ArrayRef<Value *> VL) const;
436
437 /// \returns the scalarization cost for this type. Scalarization in this
438 /// context means the creation of vectors from a group of scalars.
439 int getGatherCost(Type *Ty);
440
441 /// \returns the scalarization cost for this list of values. Assuming that
442 /// this subtree gets vectorized, we may need to extract the values from the
443 /// roots. This method calculates the cost of extracting the values.
444 int getGatherCost(ArrayRef<Value *> VL);
445
446 /// \brief Set the Builder insert point to one after the last instruction in
447 /// the bundle
448 void setInsertPointAfterBundle(ArrayRef<Value *> VL);
449
450 /// \returns a vector from a collection of scalars in \p VL.
451 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
452
453 /// \returns whether the VectorizableTree is fully vectorizable and will
454 /// be beneficial even the tree height is tiny.
455 bool isFullyVectorizableTinyTree();
456
457 /// \reorder commutative operands in alt shuffle if they result in
458 /// vectorized code.
459 void reorderAltShuffleOperands(ArrayRef<Value *> VL,
460 SmallVectorImpl<Value *> &Left,
461 SmallVectorImpl<Value *> &Right);
462 /// \reorder commutative operands to get better probability of
463 /// generating vectorized code.
464 void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
465 SmallVectorImpl<Value *> &Left,
466 SmallVectorImpl<Value *> &Right);
467 struct TreeEntry {
468 TreeEntry() : Scalars(), VectorizedValue(nullptr),
469 NeedToGather(0), NeedToShuffle(0) {}
470
471 /// \returns true if the scalars in VL are equal to this entry.
472 bool isSame(ArrayRef<Value *> VL) const {
473 assert(VL.size() == Scalars.size() && "Invalid size")((VL.size() == Scalars.size() && "Invalid size") ? static_cast
<void> (0) : __assert_fail ("VL.size() == Scalars.size() && \"Invalid size\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 473, __PRETTY_FUNCTION__))
;
474 return std::equal(VL.begin(), VL.end(), Scalars.begin());
475 }
476
477 /// \returns true if the scalars in VL are found in this tree entry.
478 bool isFoundJumbled(ArrayRef<Value *> VL, const DataLayout &DL,
479 ScalarEvolution &SE) const {
480 assert(VL.size() == Scalars.size() && "Invalid size")((VL.size() == Scalars.size() && "Invalid size") ? static_cast
<void> (0) : __assert_fail ("VL.size() == Scalars.size() && \"Invalid size\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 480, __PRETTY_FUNCTION__))
;
481 SmallVector<Value *, 8> List;
482 if (!sortMemAccesses(VL, DL, SE, List))
483 return false;
484
485 return std::equal(List.begin(), List.end(), Scalars.begin());
486 }
487
488 /// A vector of scalars.
489 ValueList Scalars;
490
491 /// The Scalars are vectorized into this value. It is initialized to Null.
492 Value *VectorizedValue;
493
494 /// Do we need to gather this sequence ?
495 bool NeedToGather;
496
497 /// Do we need to shuffle the load ?
498 bool NeedToShuffle;
499 };
500
501 /// Create a new VectorizableTree entry.
502 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
503 bool NeedToShuffle) {
504 VectorizableTree.emplace_back();
505 int idx = VectorizableTree.size() - 1;
506 TreeEntry *Last = &VectorizableTree[idx];
507 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
508 Last->NeedToGather = !Vectorized;
509 Last->NeedToShuffle = NeedToShuffle;
510 if (Vectorized) {
511 for (int i = 0, e = VL.size(); i != e; ++i) {
512 assert(!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!")((!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!"
) ? static_cast<void> (0) : __assert_fail ("!ScalarToTreeEntry.count(VL[i]) && \"Scalar already in tree!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 512, __PRETTY_FUNCTION__))
;
513 ScalarToTreeEntry[VL[i]] = idx;
514 }
515 } else {
516 MustGather.insert(VL.begin(), VL.end());
517 }
518 return Last;
519 }
520
521 /// -- Vectorization State --
522 /// Holds all of the tree entries.
523 std::vector<TreeEntry> VectorizableTree;
524
525 /// Maps a specific scalar to its tree entry.
526 SmallDenseMap<Value*, int> ScalarToTreeEntry;
527
528 /// A list of scalars that we found that we need to keep as scalars.
529 ValueSet MustGather;
530
531 /// This POD struct describes one external user in the vectorized tree.
532 struct ExternalUser {
533 ExternalUser (Value *S, llvm::User *U, int L) :
534 Scalar(S), User(U), Lane(L){}
535 // Which scalar in our function.
536 Value *Scalar;
537 // Which user that uses the scalar.
538 llvm::User *User;
539 // Which lane does the scalar belong to.
540 int Lane;
541 };
542 typedef SmallVector<ExternalUser, 16> UserList;
543
544 /// Checks if two instructions may access the same memory.
545 ///
546 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
547 /// is invariant in the calling loop.
548 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
549 Instruction *Inst2) {
550
551 // First check if the result is already in the cache.
552 AliasCacheKey key = std::make_pair(Inst1, Inst2);
553 Optional<bool> &result = AliasCache[key];
554 if (result.hasValue()) {
555 return result.getValue();
556 }
557 MemoryLocation Loc2 = getLocation(Inst2, AA);
558 bool aliased = true;
559 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
560 // Do the alias check.
561 aliased = AA->alias(Loc1, Loc2);
562 }
563 // Store the result in the cache.
564 result = aliased;
565 return aliased;
566 }
567
568 typedef std::pair<Instruction *, Instruction *> AliasCacheKey;
569
570 /// Cache for alias results.
571 /// TODO: consider moving this to the AliasAnalysis itself.
572 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
573
574 /// Removes an instruction from its block and eventually deletes it.
575 /// It's like Instruction::eraseFromParent() except that the actual deletion
576 /// is delayed until BoUpSLP is destructed.
577 /// This is required to ensure that there are no incorrect collisions in the
578 /// AliasCache, which can happen if a new instruction is allocated at the
579 /// same address as a previously deleted instruction.
580 void eraseInstruction(Instruction *I) {
581 I->removeFromParent();
582 I->dropAllReferences();
583 DeletedInstructions.push_back(std::unique_ptr<Instruction>(I));
584 }
585
586 /// Temporary store for deleted instructions. Instructions will be deleted
587 /// eventually when the BoUpSLP is destructed.
588 SmallVector<std::unique_ptr<Instruction>, 8> DeletedInstructions;
589
590 /// A list of values that need to extracted out of the tree.
591 /// This list holds pairs of (Internal Scalar : External User). External User
592 /// can be nullptr, it means that this Internal Scalar will be used later,
593 /// after vectorization.
594 UserList ExternalUses;
595
596 /// Values used only by @llvm.assume calls.
597 SmallPtrSet<const Value *, 32> EphValues;
598
599 /// Holds all of the instructions that we gathered.
600 SetVector<Instruction *> GatherSeq;
601 /// A list of blocks that we are going to CSE.
602 SetVector<BasicBlock *> CSEBlocks;
603
604 /// Contains all scheduling relevant data for an instruction.
605 /// A ScheduleData either represents a single instruction or a member of an
606 /// instruction bundle (= a group of instructions which is combined into a
607 /// vector instruction).
608 struct ScheduleData {
609
610 // The initial value for the dependency counters. It means that the
611 // dependencies are not calculated yet.
612 enum { InvalidDeps = -1 };
613
614 ScheduleData()
615 : Inst(nullptr), FirstInBundle(nullptr), NextInBundle(nullptr),
616 NextLoadStore(nullptr), SchedulingRegionID(0), SchedulingPriority(0),
617 Dependencies(InvalidDeps), UnscheduledDeps(InvalidDeps),
618 UnscheduledDepsInBundle(InvalidDeps), IsScheduled(false) {}
619
620 void init(int BlockSchedulingRegionID) {
621 FirstInBundle = this;
622 NextInBundle = nullptr;
623 NextLoadStore = nullptr;
624 IsScheduled = false;
625 SchedulingRegionID = BlockSchedulingRegionID;
626 UnscheduledDepsInBundle = UnscheduledDeps;
627 clearDependencies();
628 }
629
630 /// Returns true if the dependency information has been calculated.
631 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
632
633 /// Returns true for single instructions and for bundle representatives
634 /// (= the head of a bundle).
635 bool isSchedulingEntity() const { return FirstInBundle == this; }
636
637 /// Returns true if it represents an instruction bundle and not only a
638 /// single instruction.
639 bool isPartOfBundle() const {
640 return NextInBundle != nullptr || FirstInBundle != this;
641 }
642
643 /// Returns true if it is ready for scheduling, i.e. it has no more
644 /// unscheduled depending instructions/bundles.
645 bool isReady() const {
646 assert(isSchedulingEntity() &&((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 647, __PRETTY_FUNCTION__))
647 "can't consider non-scheduling entity for ready list")((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 647, __PRETTY_FUNCTION__))
;
648 return UnscheduledDepsInBundle == 0 && !IsScheduled;
649 }
650
651 /// Modifies the number of unscheduled dependencies, also updating it for
652 /// the whole bundle.
653 int incrementUnscheduledDeps(int Incr) {
654 UnscheduledDeps += Incr;
655 return FirstInBundle->UnscheduledDepsInBundle += Incr;
656 }
657
658 /// Sets the number of unscheduled dependencies to the number of
659 /// dependencies.
660 void resetUnscheduledDeps() {
661 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
662 }
663
664 /// Clears all dependency information.
665 void clearDependencies() {
666 Dependencies = InvalidDeps;
667 resetUnscheduledDeps();
668 MemoryDependencies.clear();
669 }
670
671 void dump(raw_ostream &os) const {
672 if (!isSchedulingEntity()) {
673 os << "/ " << *Inst;
674 } else if (NextInBundle) {
675 os << '[' << *Inst;
676 ScheduleData *SD = NextInBundle;
677 while (SD) {
678 os << ';' << *SD->Inst;
679 SD = SD->NextInBundle;
680 }
681 os << ']';
682 } else {
683 os << *Inst;
684 }
685 }
686
687 Instruction *Inst;
688
689 /// Points to the head in an instruction bundle (and always to this for
690 /// single instructions).
691 ScheduleData *FirstInBundle;
692
693 /// Single linked list of all instructions in a bundle. Null if it is a
694 /// single instruction.
695 ScheduleData *NextInBundle;
696
697 /// Single linked list of all memory instructions (e.g. load, store, call)
698 /// in the block - until the end of the scheduling region.
699 ScheduleData *NextLoadStore;
700
701 /// The dependent memory instructions.
702 /// This list is derived on demand in calculateDependencies().
703 SmallVector<ScheduleData *, 4> MemoryDependencies;
704
705 /// This ScheduleData is in the current scheduling region if this matches
706 /// the current SchedulingRegionID of BlockScheduling.
707 int SchedulingRegionID;
708
709 /// Used for getting a "good" final ordering of instructions.
710 int SchedulingPriority;
711
712 /// The number of dependencies. Constitutes of the number of users of the
713 /// instruction plus the number of dependent memory instructions (if any).
714 /// This value is calculated on demand.
715 /// If InvalidDeps, the number of dependencies is not calculated yet.
716 ///
717 int Dependencies;
718
719 /// The number of dependencies minus the number of dependencies of scheduled
720 /// instructions. As soon as this is zero, the instruction/bundle gets ready
721 /// for scheduling.
722 /// Note that this is negative as long as Dependencies is not calculated.
723 int UnscheduledDeps;
724
725 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
726 /// single instructions.
727 int UnscheduledDepsInBundle;
728
729 /// True if this instruction is scheduled (or considered as scheduled in the
730 /// dry-run).
731 bool IsScheduled;
732 };
733
734#ifndef NDEBUG
735 friend inline raw_ostream &operator<<(raw_ostream &os,
736 const BoUpSLP::ScheduleData &SD) {
737 SD.dump(os);
738 return os;
739 }
740#endif
741
742 /// Contains all scheduling data for a basic block.
743 ///
744 struct BlockScheduling {
745
746 BlockScheduling(BasicBlock *BB)
747 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize),
748 ScheduleStart(nullptr), ScheduleEnd(nullptr),
749 FirstLoadStoreInRegion(nullptr), LastLoadStoreInRegion(nullptr),
750 ScheduleRegionSize(0),
751 ScheduleRegionSizeLimit(ScheduleRegionSizeBudget),
752 // Make sure that the initial SchedulingRegionID is greater than the
753 // initial SchedulingRegionID in ScheduleData (which is 0).
754 SchedulingRegionID(1) {}
755
756 void clear() {
757 ReadyInsts.clear();
758 ScheduleStart = nullptr;
759 ScheduleEnd = nullptr;
760 FirstLoadStoreInRegion = nullptr;
761 LastLoadStoreInRegion = nullptr;
762
763 // Reduce the maximum schedule region size by the size of the
764 // previous scheduling run.
765 ScheduleRegionSizeLimit -= ScheduleRegionSize;
766 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
767 ScheduleRegionSizeLimit = MinScheduleRegionSize;
768 ScheduleRegionSize = 0;
769
770 // Make a new scheduling region, i.e. all existing ScheduleData is not
771 // in the new region yet.
772 ++SchedulingRegionID;
773 }
774
775 ScheduleData *getScheduleData(Value *V) {
776 ScheduleData *SD = ScheduleDataMap[V];
777 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
778 return SD;
779 return nullptr;
780 }
781
782 bool isInSchedulingRegion(ScheduleData *SD) {
783 return SD->SchedulingRegionID == SchedulingRegionID;
784 }
785
786 /// Marks an instruction as scheduled and puts all dependent ready
787 /// instructions into the ready-list.
788 template <typename ReadyListType>
789 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
790 SD->IsScheduled = true;
791 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
792
793 ScheduleData *BundleMember = SD;
794 while (BundleMember) {
795 // Handle the def-use chain dependencies.
796 for (Use &U : BundleMember->Inst->operands()) {
797 ScheduleData *OpDef = getScheduleData(U.get());
798 if (OpDef && OpDef->hasValidDependencies() &&
799 OpDef->incrementUnscheduledDeps(-1) == 0) {
800 // There are no more unscheduled dependencies after decrementing,
801 // so we can put the dependent instruction into the ready list.
802 ScheduleData *DepBundle = OpDef->FirstInBundle;
803 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 804, __PRETTY_FUNCTION__))
804 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 804, __PRETTY_FUNCTION__))
;
805 ReadyList.insert(DepBundle);
806 DEBUG(dbgs() << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
807 }
808 }
809 // Handle the memory dependencies.
810 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
811 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
812 // There are no more unscheduled dependencies after decrementing,
813 // so we can put the dependent instruction into the ready list.
814 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
815 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 816, __PRETTY_FUNCTION__))
816 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 816, __PRETTY_FUNCTION__))
;
817 ReadyList.insert(DepBundle);
818 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
819 }
820 }
821 BundleMember = BundleMember->NextInBundle;
822 }
823 }
824
825 /// Put all instructions into the ReadyList which are ready for scheduling.
826 template <typename ReadyListType>
827 void initialFillReadyList(ReadyListType &ReadyList) {
828 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
829 ScheduleData *SD = getScheduleData(I);
830 if (SD->isSchedulingEntity() && SD->isReady()) {
831 ReadyList.insert(SD);
832 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
833 }
834 }
835 }
836
837 /// Checks if a bundle of instructions can be scheduled, i.e. has no
838 /// cyclic dependencies. This is only a dry-run, no instructions are
839 /// actually moved at this stage.
840 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP);
841
842 /// Un-bundles a group of instructions.
843 void cancelScheduling(ArrayRef<Value *> VL);
844
845 /// Extends the scheduling region so that V is inside the region.
846 /// \returns true if the region size is within the limit.
847 bool extendSchedulingRegion(Value *V);
848
849 /// Initialize the ScheduleData structures for new instructions in the
850 /// scheduling region.
851 void initScheduleData(Instruction *FromI, Instruction *ToI,
852 ScheduleData *PrevLoadStore,
853 ScheduleData *NextLoadStore);
854
855 /// Updates the dependency information of a bundle and of all instructions/
856 /// bundles which depend on the original bundle.
857 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
858 BoUpSLP *SLP);
859
860 /// Sets all instruction in the scheduling region to un-scheduled.
861 void resetSchedule();
862
863 BasicBlock *BB;
864
865 /// Simple memory allocation for ScheduleData.
866 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
867
868 /// The size of a ScheduleData array in ScheduleDataChunks.
869 int ChunkSize;
870
871 /// The allocator position in the current chunk, which is the last entry
872 /// of ScheduleDataChunks.
873 int ChunkPos;
874
875 /// Attaches ScheduleData to Instruction.
876 /// Note that the mapping survives during all vectorization iterations, i.e.
877 /// ScheduleData structures are recycled.
878 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
879
880 struct ReadyList : SmallVector<ScheduleData *, 8> {
881 void insert(ScheduleData *SD) { push_back(SD); }
882 };
883
884 /// The ready-list for scheduling (only used for the dry-run).
885 ReadyList ReadyInsts;
886
887 /// The first instruction of the scheduling region.
888 Instruction *ScheduleStart;
889
890 /// The first instruction _after_ the scheduling region.
891 Instruction *ScheduleEnd;
892
893 /// The first memory accessing instruction in the scheduling region
894 /// (can be null).
895 ScheduleData *FirstLoadStoreInRegion;
896
897 /// The last memory accessing instruction in the scheduling region
898 /// (can be null).
899 ScheduleData *LastLoadStoreInRegion;
900
901 /// The current size of the scheduling region.
902 int ScheduleRegionSize;
903
904 /// The maximum size allowed for the scheduling region.
905 int ScheduleRegionSizeLimit;
906
907 /// The ID of the scheduling region. For a new vectorization iteration this
908 /// is incremented which "removes" all ScheduleData from the region.
909 int SchedulingRegionID;
910 };
911
912 /// Attaches the BlockScheduling structures to basic blocks.
913 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
914
915 /// Performs the "real" scheduling. Done before vectorization is actually
916 /// performed in a basic block.
917 void scheduleBlock(BlockScheduling *BS);
918
919 /// List of users to ignore during scheduling and that don't need extracting.
920 ArrayRef<Value *> UserIgnoreList;
921
922 // Number of load bundles that contain consecutive loads.
923 int NumLoadsWantToKeepOrder;
924
925 // Number of load bundles that contain consecutive loads in reversed order.
926 int NumLoadsWantToChangeOrder;
927
928 // Analysis and block reference.
929 Function *F;
930 ScalarEvolution *SE;
931 TargetTransformInfo *TTI;
932 TargetLibraryInfo *TLI;
933 AliasAnalysis *AA;
934 LoopInfo *LI;
935 DominatorTree *DT;
936 AssumptionCache *AC;
937 DemandedBits *DB;
938 const DataLayout *DL;
939 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
940 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
941 /// Instruction builder to construct the vectorized tree.
942 IRBuilder<> Builder;
943
944 /// A map of scalar integer values to the smallest bit width with which they
945 /// can legally be represented. The values map to (width, signed) pairs,
946 /// where "width" indicates the minimum bit width and "signed" is True if the
947 /// value must be signed-extended, rather than zero-extended, back to its
948 /// original width.
949 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
950};
951
952} // end namespace llvm
953} // end namespace slpvectorizer
954
955void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
956 ArrayRef<Value *> UserIgnoreLst) {
957 ExtraValueToDebugLocsMap ExternallyUsedValues;
958 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
959}
960void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
961 ExtraValueToDebugLocsMap &ExternallyUsedValues,
962 ArrayRef<Value *> UserIgnoreLst) {
963 deleteTree();
964 UserIgnoreList = UserIgnoreLst;
965 if (!allSameType(Roots))
966 return;
967 buildTree_rec(Roots, 0);
968
969 // Collect the values that we need to extract from the tree.
970 for (TreeEntry &EIdx : VectorizableTree) {
971 TreeEntry *Entry = &EIdx;
972
973 // For each lane:
974 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
975 Value *Scalar = Entry->Scalars[Lane];
976
977 // No need to handle users of gathered values.
978 if (Entry->NeedToGather)
979 continue;
980
981 // Check if the scalar is externally used as an extra arg.
982 auto ExtI = ExternallyUsedValues.find(Scalar);
983 if (ExtI != ExternallyUsedValues.end()) {
984 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
985 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
986 ExternalUses.emplace_back(Scalar, nullptr, Lane);
987 continue;
988 }
989 for (User *U : Scalar->users()) {
990 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
991
992 Instruction *UserInst = dyn_cast<Instruction>(U);
993 if (!UserInst)
994 continue;
995
996 // Skip in-tree scalars that become vectors
997 if (ScalarToTreeEntry.count(U)) {
998 int Idx = ScalarToTreeEntry[U];
999 TreeEntry *UseEntry = &VectorizableTree[Idx];
1000 Value *UseScalar = UseEntry->Scalars[0];
1001 // Some in-tree scalars will remain as scalar in vectorized
1002 // instructions. If that is the case, the one in Lane 0 will
1003 // be used.
1004 if (UseScalar != U ||
1005 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1006 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1007 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
1008 assert(!VectorizableTree[Idx].NeedToGather && "Bad state")((!VectorizableTree[Idx].NeedToGather && "Bad state")
? static_cast<void> (0) : __assert_fail ("!VectorizableTree[Idx].NeedToGather && \"Bad state\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1008, __PRETTY_FUNCTION__))
;
1009 continue;
1010 }
1011 }
1012
1013 // Ignore users in the user ignore list.
1014 if (is_contained(UserIgnoreList, UserInst))
1015 continue;
1016
1017 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1018 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
1019 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1020 }
1021 }
1022 }
1023}
1024
1025
1026void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth) {
1027 bool isAltShuffle = false;
1028 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(((allConstant(VL) || allSameType(VL)) && "Invalid types!"
) ? static_cast<void> (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1028, __PRETTY_FUNCTION__))
;
1029
1030 if (Depth == RecursionMaxDepth) {
1031 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
1032 newTreeEntry(VL, false, false);
1033 return;
1034 }
1035
1036 // Don't handle vectors.
1037 if (VL[0]->getType()->isVectorTy()) {
1038 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
1039 newTreeEntry(VL, false, false);
1040 return;
1041 }
1042
1043 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1044 if (SI->getValueOperand()->getType()->isVectorTy()) {
1045 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1046 newTreeEntry(VL, false, false);
1047 return;
1048 }
1049 unsigned Opcode = getSameOpcode(VL);
1050
1051 // Check that this shuffle vector refers to the alternate
1052 // sequence of opcodes.
1053 if (Opcode == Instruction::ShuffleVector) {
1054 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
1055 unsigned Op = I0->getOpcode();
1056 if (Op != Instruction::ShuffleVector)
1057 isAltShuffle = true;
1058 }
1059
1060 // If all of the operands are identical or constant we have a simple solution.
1061 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !Opcode) {
1062 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1063 newTreeEntry(VL, false, false);
1064 return;
1065 }
1066
1067 // We now know that this is a vector of instructions of the same type from
1068 // the same block.
1069
1070 // Don't vectorize ephemeral values.
1071 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1072 if (EphValues.count(VL[i])) {
1073 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1074 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1075 newTreeEntry(VL, false, false);
1076 return;
1077 }
1078 }
1079
1080 // Check if this is a duplicate of another entry.
1081 if (ScalarToTreeEntry.count(VL[0])) {
1082 int Idx = ScalarToTreeEntry[VL[0]];
1083 TreeEntry *E = &VectorizableTree[Idx];
1084 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1085 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*VL[i] << ".\n"; } } while (false)
;
1086 if (E->Scalars[i] != VL[i]) {
1087 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1088 newTreeEntry(VL, false, false);
1089 return;
1090 }
1091 }
1092 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *VL[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*VL[0] << ".\n"; } } while (false)
;
1093 return;
1094 }
1095
1096 // Check that none of the instructions in the bundle are already in the tree.
1097 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1098 if (ScalarToTreeEntry.count(VL[i])) {
1099 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1100 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1101 newTreeEntry(VL, false, false);
1102 return;
1103 }
1104 }
1105
1106 // If any of the scalars is marked as a value that needs to stay scalar then
1107 // we need to gather the scalars.
1108 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1109 if (MustGather.count(VL[i])) {
1110 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1111 newTreeEntry(VL, false, false);
1112 return;
1113 }
1114 }
1115
1116 // Check that all of the users of the scalars that we want to vectorize are
1117 // schedulable.
1118 Instruction *VL0 = cast<Instruction>(VL[0]);
1119 BasicBlock *BB = cast<Instruction>(VL0)->getParent();
1120
1121 if (!DT->isReachableFromEntry(BB)) {
1122 // Don't go into unreachable blocks. They may contain instructions with
1123 // dependency cycles which confuse the final scheduling.
1124 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1125 newTreeEntry(VL, false, false);
1126 return;
1127 }
1128
1129 // Check that every instructions appears once in this bundle.
1130 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1131 for (unsigned j = i+1; j < e; ++j)
1132 if (VL[i] == VL[j]) {
1133 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1134 newTreeEntry(VL, false, false);
1135 return;
1136 }
1137
1138 auto &BSRef = BlocksSchedules[BB];
1139 if (!BSRef) {
1140 BSRef = llvm::make_unique<BlockScheduling>(BB);
1141 }
1142 BlockScheduling &BS = *BSRef.get();
1143
1144 if (!BS.tryScheduleBundle(VL, this)) {
1145 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1146 assert((!BS.getScheduleData(VL[0]) ||(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1148, __PRETTY_FUNCTION__))
1147 !BS.getScheduleData(VL[0])->isPartOfBundle()) &&(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1148, __PRETTY_FUNCTION__))
1148 "tryScheduleBundle should cancelScheduling on failure")(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1148, __PRETTY_FUNCTION__))
;
1149 newTreeEntry(VL, false, false);
1150 return;
1151 }
1152 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1153
1154 switch (Opcode) {
1155 case Instruction::PHI: {
1156 PHINode *PH = dyn_cast<PHINode>(VL0);
1157
1158 // Check for terminator values (e.g. invoke).
1159 for (unsigned j = 0; j < VL.size(); ++j)
1160 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1161 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1162 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1163 if (Term) {
1164 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false)
;
1165 BS.cancelScheduling(VL);
1166 newTreeEntry(VL, false, false);
1167 return;
1168 }
1169 }
1170
1171 newTreeEntry(VL, true, false);
1172 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1173
1174 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1175 ValueList Operands;
1176 // Prepare the operand vector.
1177 for (Value *j : VL)
1178 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1179 PH->getIncomingBlock(i)));
1180
1181 buildTree_rec(Operands, Depth + 1);
1182 }
1183 return;
1184 }
1185 case Instruction::ExtractValue:
1186 case Instruction::ExtractElement: {
1187 bool Reuse = canReuseExtract(VL, Opcode);
1188 if (Reuse) {
1189 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing extract sequence.\n"
; } } while (false)
;
1190 } else {
1191 BS.cancelScheduling(VL);
1192 }
1193 newTreeEntry(VL, Reuse, false);
1194 return;
1195 }
1196 case Instruction::Load: {
1197 // Check that a vectorized load would load the same memory as a scalar
1198 // load.
1199 // For example we don't want vectorize loads that are smaller than 8 bit.
1200 // Even though we have a packed struct {<i2, i2, i2, i2>} LLVM treats
1201 // loading/storing it as an i8 struct. If we vectorize loads/stores from
1202 // such a struct we read/write packed bits disagreeing with the
1203 // unvectorized version.
1204 Type *ScalarTy = VL[0]->getType();
1205
1206 if (DL->getTypeSizeInBits(ScalarTy) !=
1207 DL->getTypeAllocSizeInBits(ScalarTy)) {
1208 BS.cancelScheduling(VL);
1209 newTreeEntry(VL, false, false);
1210 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1211 return;
1212 }
1213
1214 // Make sure all loads in the bundle are simple - we can't vectorize
1215 // atomic or volatile loads.
1216 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1217 LoadInst *L = cast<LoadInst>(VL[i]);
1218 if (!L->isSimple()) {
1219 BS.cancelScheduling(VL);
1220 newTreeEntry(VL, false, false);
1221 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1222 return;
1223 }
1224 }
1225
1226 // Check if the loads are consecutive, reversed, or neither.
1227 bool Consecutive = true;
1228 bool ReverseConsecutive = true;
1229 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1230 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1231 Consecutive = false;
1232 break;
1233 } else {
1234 ReverseConsecutive = false;
1235 }
1236 }
1237
1238 if (Consecutive) {
1239 ++NumLoadsWantToKeepOrder;
1240 newTreeEntry(VL, true, false);
1241 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1242 return;
1243 }
1244
1245 // If none of the load pairs were consecutive when checked in order,
1246 // check the reverse order.
1247 if (ReverseConsecutive)
1248 for (unsigned i = VL.size() - 1; i > 0; --i)
1249 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1250 ReverseConsecutive = false;
1251 break;
1252 }
1253
1254 if (VL.size() > 2 && !ReverseConsecutive) {
1255 bool ShuffledLoads = true;
1256 SmallVector<Value *, 8> Sorted;
1257 if (sortMemAccesses(VL, *DL, *SE, Sorted)) {
1258 auto NewVL = makeArrayRef(Sorted.begin(), Sorted.end());
1259 for (unsigned i = 0, e = NewVL.size() - 1; i < e; ++i) {
1260 if (!isConsecutiveAccess(NewVL[i], NewVL[i + 1], *DL, *SE)) {
1261 ShuffledLoads = false;
1262 break;
1263 }
1264 }
1265 if (ShuffledLoads) {
1266 newTreeEntry(NewVL, true, true);
1267 return;
1268 }
1269 }
1270 }
1271
1272 BS.cancelScheduling(VL);
1273 newTreeEntry(VL, false, false);
1274
1275 if (ReverseConsecutive) {
1276 ++NumLoadsWantToChangeOrder;
1277 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering reversed loads.\n"
; } } while (false)
;
1278 } else {
1279 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1280 }
1281 return;
1282 }
1283 case Instruction::ZExt:
1284 case Instruction::SExt:
1285 case Instruction::FPToUI:
1286 case Instruction::FPToSI:
1287 case Instruction::FPExt:
1288 case Instruction::PtrToInt:
1289 case Instruction::IntToPtr:
1290 case Instruction::SIToFP:
1291 case Instruction::UIToFP:
1292 case Instruction::Trunc:
1293 case Instruction::FPTrunc:
1294 case Instruction::BitCast: {
1295 Type *SrcTy = VL0->getOperand(0)->getType();
1296 for (Value *Val : VL) {
1297 Type *Ty = cast<Instruction>(Val)->getOperand(0)->getType();
1298 if (Ty != SrcTy || !isValidElementType(Ty)) {
1299 BS.cancelScheduling(VL);
1300 newTreeEntry(VL, false, false);
1301 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1302 return;
1303 }
1304 }
1305 newTreeEntry(VL, true, false);
1306 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1307
1308 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1309 ValueList Operands;
1310 // Prepare the operand vector.
1311 for (Value *j : VL)
1312 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1313
1314 buildTree_rec(Operands, Depth+1);
1315 }
1316 return;
1317 }
1318 case Instruction::ICmp:
1319 case Instruction::FCmp: {
1320 // Check that all of the compares have the same predicate.
1321 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1322 Type *ComparedTy = cast<Instruction>(VL[0])->getOperand(0)->getType();
1323 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1324 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1325 if (Cmp->getPredicate() != P0 ||
1326 Cmp->getOperand(0)->getType() != ComparedTy) {
1327 BS.cancelScheduling(VL);
1328 newTreeEntry(VL, false, false);
1329 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1330 return;
1331 }
1332 }
1333
1334 newTreeEntry(VL, true, false);
1335 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1336
1337 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1338 ValueList Operands;
1339 // Prepare the operand vector.
1340 for (Value *j : VL)
1341 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1342
1343 buildTree_rec(Operands, Depth+1);
1344 }
1345 return;
1346 }
1347 case Instruction::Select:
1348 case Instruction::Add:
1349 case Instruction::FAdd:
1350 case Instruction::Sub:
1351 case Instruction::FSub:
1352 case Instruction::Mul:
1353 case Instruction::FMul:
1354 case Instruction::UDiv:
1355 case Instruction::SDiv:
1356 case Instruction::FDiv:
1357 case Instruction::URem:
1358 case Instruction::SRem:
1359 case Instruction::FRem:
1360 case Instruction::Shl:
1361 case Instruction::LShr:
1362 case Instruction::AShr:
1363 case Instruction::And:
1364 case Instruction::Or:
1365 case Instruction::Xor: {
1366 newTreeEntry(VL, true, false);
1367 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1368
1369 // Sort operands of the instructions so that each side is more likely to
1370 // have the same opcode.
1371 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1372 ValueList Left, Right;
1373 reorderInputsAccordingToOpcode(VL, Left, Right);
1374 buildTree_rec(Left, Depth + 1);
1375 buildTree_rec(Right, Depth + 1);
1376 return;
1377 }
1378
1379 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1380 ValueList Operands;
1381 // Prepare the operand vector.
1382 for (Value *j : VL)
1383 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1384
1385 buildTree_rec(Operands, Depth+1);
1386 }
1387 return;
1388 }
1389 case Instruction::GetElementPtr: {
1390 // We don't combine GEPs with complicated (nested) indexing.
1391 for (Value *Val : VL) {
1392 if (cast<Instruction>(Val)->getNumOperands() != 2) {
1393 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1394 BS.cancelScheduling(VL);
1395 newTreeEntry(VL, false, false);
1396 return;
1397 }
1398 }
1399
1400 // We can't combine several GEPs into one vector if they operate on
1401 // different types.
1402 Type *Ty0 = cast<Instruction>(VL0)->getOperand(0)->getType();
1403 for (Value *Val : VL) {
1404 Type *CurTy = cast<Instruction>(Val)->getOperand(0)->getType();
1405 if (Ty0 != CurTy) {
1406 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1407 BS.cancelScheduling(VL);
1408 newTreeEntry(VL, false, false);
1409 return;
1410 }
1411 }
1412
1413 // We don't combine GEPs with non-constant indexes.
1414 for (Value *Val : VL) {
1415 auto Op = cast<Instruction>(Val)->getOperand(1);
1416 if (!isa<ConstantInt>(Op)) {
1417 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1418 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1419 BS.cancelScheduling(VL);
1420 newTreeEntry(VL, false, false);
1421 return;
1422 }
1423 }
1424
1425 newTreeEntry(VL, true, false);
1426 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1427 for (unsigned i = 0, e = 2; i < e; ++i) {
1428 ValueList Operands;
1429 // Prepare the operand vector.
1430 for (Value *j : VL)
1431 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1432
1433 buildTree_rec(Operands, Depth + 1);
1434 }
1435 return;
1436 }
1437 case Instruction::Store: {
1438 // Check if the stores are consecutive or of we need to swizzle them.
1439 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1440 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1441 BS.cancelScheduling(VL);
1442 newTreeEntry(VL, false, false);
1443 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1444 return;
1445 }
1446
1447 newTreeEntry(VL, true, false);
1448 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1449
1450 ValueList Operands;
1451 for (Value *j : VL)
1452 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1453
1454 buildTree_rec(Operands, Depth + 1);
1455 return;
1456 }
1457 case Instruction::Call: {
1458 // Check if the calls are all to the same vectorizable intrinsic.
1459 CallInst *CI = cast<CallInst>(VL[0]);
1460 // Check if this is an Intrinsic call or something that can be
1461 // represented by an intrinsic call
1462 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1463 if (!isTriviallyVectorizable(ID)) {
1464 BS.cancelScheduling(VL);
1465 newTreeEntry(VL, false, false);
1466 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1467 return;
1468 }
1469 Function *Int = CI->getCalledFunction();
1470 Value *A1I = nullptr;
1471 if (hasVectorInstrinsicScalarOpd(ID, 1))
1472 A1I = CI->getArgOperand(1);
1473 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1474 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1475 if (!CI2 || CI2->getCalledFunction() != Int ||
1476 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1477 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1478 BS.cancelScheduling(VL);
1479 newTreeEntry(VL, false, false);
1480 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1481 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1482 return;
1483 }
1484 // ctlz,cttz and powi are special intrinsics whose second argument
1485 // should be same in order for them to be vectorized.
1486 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1487 Value *A1J = CI2->getArgOperand(1);
1488 if (A1I != A1J) {
1489 BS.cancelScheduling(VL);
1490 newTreeEntry(VL, false, false);
1491 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1492 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1493 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
;
1494 return;
1495 }
1496 }
1497 // Verify that the bundle operands are identical between the two calls.
1498 if (CI->hasOperandBundles() &&
1499 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1500 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1501 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1502 BS.cancelScheduling(VL);
1503 newTreeEntry(VL, false, false);
1504 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1505 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1506 return;
1507 }
1508 }
1509
1510 newTreeEntry(VL, true, false);
1511 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1512 ValueList Operands;
1513 // Prepare the operand vector.
1514 for (Value *j : VL) {
1515 CallInst *CI2 = dyn_cast<CallInst>(j);
1516 Operands.push_back(CI2->getArgOperand(i));
1517 }
1518 buildTree_rec(Operands, Depth + 1);
1519 }
1520 return;
1521 }
1522 case Instruction::ShuffleVector: {
1523 // If this is not an alternate sequence of opcode like add-sub
1524 // then do not vectorize this instruction.
1525 if (!isAltShuffle) {
1526 BS.cancelScheduling(VL);
1527 newTreeEntry(VL, false, false);
1528 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
1529 return;
1530 }
1531 newTreeEntry(VL, true, false);
1532 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
1533
1534 // Reorder operands if reordering would enable vectorization.
1535 if (isa<BinaryOperator>(VL0)) {
1536 ValueList Left, Right;
1537 reorderAltShuffleOperands(VL, Left, Right);
1538 buildTree_rec(Left, Depth + 1);
1539 buildTree_rec(Right, Depth + 1);
1540 return;
1541 }
1542
1543 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1544 ValueList Operands;
1545 // Prepare the operand vector.
1546 for (Value *j : VL)
1547 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1548
1549 buildTree_rec(Operands, Depth + 1);
1550 }
1551 return;
1552 }
1553 default:
1554 BS.cancelScheduling(VL);
1555 newTreeEntry(VL, false, false);
1556 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
1557 return;
1558 }
1559}
1560
1561unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1562 unsigned N;
1563 Type *EltTy;
1564 auto *ST = dyn_cast<StructType>(T);
1565 if (ST) {
1566 N = ST->getNumElements();
1567 EltTy = *ST->element_begin();
1568 } else {
1569 N = cast<ArrayType>(T)->getNumElements();
1570 EltTy = cast<ArrayType>(T)->getElementType();
1571 }
1572 if (!isValidElementType(EltTy))
1573 return 0;
1574 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1575 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1576 return 0;
1577 if (ST) {
1578 // Check that struct is homogeneous.
1579 for (const auto *Ty : ST->elements())
1580 if (Ty != EltTy)
1581 return 0;
1582 }
1583 return N;
1584}
1585
1586bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const {
1587 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1588, __PRETTY_FUNCTION__))
1588 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1588, __PRETTY_FUNCTION__))
;
1589 assert(Opcode == getSameOpcode(VL) && "Invalid opcode")((Opcode == getSameOpcode(VL) && "Invalid opcode") ? static_cast
<void> (0) : __assert_fail ("Opcode == getSameOpcode(VL) && \"Invalid opcode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1589, __PRETTY_FUNCTION__))
;
1590 // Check if all of the extracts come from the same vector and from the
1591 // correct offset.
1592 Value *VL0 = VL[0];
1593 Instruction *E0 = cast<Instruction>(VL0);
1594 Value *Vec = E0->getOperand(0);
1595
1596 // We have to extract from a vector/aggregate with the same number of elements.
1597 unsigned NElts;
1598 if (Opcode == Instruction::ExtractValue) {
1599 const DataLayout &DL = E0->getModule()->getDataLayout();
1600 NElts = canMapToVector(Vec->getType(), DL);
1601 if (!NElts)
1602 return false;
1603 // Check if load can be rewritten as load of vector.
1604 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1605 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1606 return false;
1607 } else {
1608 NElts = Vec->getType()->getVectorNumElements();
1609 }
1610
1611 if (NElts != VL.size())
1612 return false;
1613
1614 // Check that all of the indices extract from the correct offset.
1615 if (!matchExtractIndex(E0, 0, Opcode))
1616 return false;
1617
1618 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1619 Instruction *E = cast<Instruction>(VL[i]);
1620 if (!matchExtractIndex(E, i, Opcode))
1621 return false;
1622 if (E->getOperand(0) != Vec)
1623 return false;
1624 }
1625
1626 return true;
1627}
1628
1629int BoUpSLP::getEntryCost(TreeEntry *E) {
1630 ArrayRef<Value*> VL = E->Scalars;
1631
1632 Type *ScalarTy = VL[0]->getType();
1633 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1634 ScalarTy = SI->getValueOperand()->getType();
1635 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1636
1637 // If we have computed a smaller type for the expression, update VecTy so
1638 // that the costs will be accurate.
1639 if (MinBWs.count(VL[0]))
1640 VecTy = VectorType::get(
1641 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1642
1643 if (E->NeedToGather) {
1644 if (allConstant(VL))
1645 return 0;
1646 if (isSplat(VL)) {
1647 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1648 }
1649 return getGatherCost(E->Scalars);
1650 }
1651 unsigned Opcode = getSameOpcode(VL);
1652 assert(Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")((Opcode && allSameType(VL) && allSameBlock(VL
) && "Invalid VL") ? static_cast<void> (0) : __assert_fail
("Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1652, __PRETTY_FUNCTION__))
;
1653 Instruction *VL0 = cast<Instruction>(VL[0]);
1654 switch (Opcode) {
1655 case Instruction::PHI: {
1656 return 0;
1657 }
1658 case Instruction::ExtractValue:
1659 case Instruction::ExtractElement: {
1660 if (canReuseExtract(VL, Opcode)) {
1661 int DeadCost = 0;
1662 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
1663 Instruction *E = cast<Instruction>(VL[i]);
1664 // If all users are going to be vectorized, instruction can be
1665 // considered as dead.
1666 // The same, if have only one user, it will be vectorized for sure.
1667 if (E->hasOneUse() ||
1668 std::all_of(E->user_begin(), E->user_end(), [this](User *U) {
1669 return ScalarToTreeEntry.count(U) > 0;
1670 }))
1671 // Take credit for instruction that will become dead.
1672 DeadCost +=
1673 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
1674 }
1675 return -DeadCost;
1676 }
1677 return getGatherCost(VecTy);
1678 }
1679 case Instruction::ZExt:
1680 case Instruction::SExt:
1681 case Instruction::FPToUI:
1682 case Instruction::FPToSI:
1683 case Instruction::FPExt:
1684 case Instruction::PtrToInt:
1685 case Instruction::IntToPtr:
1686 case Instruction::SIToFP:
1687 case Instruction::UIToFP:
1688 case Instruction::Trunc:
1689 case Instruction::FPTrunc:
1690 case Instruction::BitCast: {
1691 Type *SrcTy = VL0->getOperand(0)->getType();
1692
1693 // Calculate the cost of this instruction.
1694 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
1695 VL0->getType(), SrcTy);
1696
1697 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
1698 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy);
1699 return VecCost - ScalarCost;
1700 }
1701 case Instruction::FCmp:
1702 case Instruction::ICmp:
1703 case Instruction::Select: {
1704 // Calculate the cost of this instruction.
1705 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
1706 int ScalarCost = VecTy->getNumElements() *
1707 TTI->getCmpSelInstrCost(Opcode, ScalarTy, Builder.getInt1Ty());
1708 int VecCost = TTI->getCmpSelInstrCost(Opcode, VecTy, MaskTy);
1709 return VecCost - ScalarCost;
1710 }
1711 case Instruction::Add:
1712 case Instruction::FAdd:
1713 case Instruction::Sub:
1714 case Instruction::FSub:
1715 case Instruction::Mul:
1716 case Instruction::FMul:
1717 case Instruction::UDiv:
1718 case Instruction::SDiv:
1719 case Instruction::FDiv:
1720 case Instruction::URem:
1721 case Instruction::SRem:
1722 case Instruction::FRem:
1723 case Instruction::Shl:
1724 case Instruction::LShr:
1725 case Instruction::AShr:
1726 case Instruction::And:
1727 case Instruction::Or:
1728 case Instruction::Xor: {
1729 // Certain instructions can be cheaper to vectorize if they have a
1730 // constant second vector operand.
1731 TargetTransformInfo::OperandValueKind Op1VK =
1732 TargetTransformInfo::OK_AnyValue;
1733 TargetTransformInfo::OperandValueKind Op2VK =
1734 TargetTransformInfo::OK_UniformConstantValue;
1735 TargetTransformInfo::OperandValueProperties Op1VP =
1736 TargetTransformInfo::OP_None;
1737 TargetTransformInfo::OperandValueProperties Op2VP =
1738 TargetTransformInfo::OP_None;
1739
1740 // If all operands are exactly the same ConstantInt then set the
1741 // operand kind to OK_UniformConstantValue.
1742 // If instead not all operands are constants, then set the operand kind
1743 // to OK_AnyValue. If all operands are constants but not the same,
1744 // then set the operand kind to OK_NonUniformConstantValue.
1745 ConstantInt *CInt = nullptr;
1746 for (unsigned i = 0; i < VL.size(); ++i) {
1747 const Instruction *I = cast<Instruction>(VL[i]);
1748 if (!isa<ConstantInt>(I->getOperand(1))) {
1749 Op2VK = TargetTransformInfo::OK_AnyValue;
1750 break;
1751 }
1752 if (i == 0) {
1753 CInt = cast<ConstantInt>(I->getOperand(1));
1754 continue;
1755 }
1756 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
1757 CInt != cast<ConstantInt>(I->getOperand(1)))
1758 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
1759 }
1760 // FIXME: Currently cost of model modification for division by power of
1761 // 2 is handled for X86 and AArch64. Add support for other targets.
1762 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
1763 CInt->getValue().isPowerOf2())
1764 Op2VP = TargetTransformInfo::OP_PowerOf2;
1765
1766 int ScalarCost = VecTy->getNumElements() *
1767 TTI->getArithmeticInstrCost(Opcode, ScalarTy, Op1VK,
1768 Op2VK, Op1VP, Op2VP);
1769 int VecCost = TTI->getArithmeticInstrCost(Opcode, VecTy, Op1VK, Op2VK,
1770 Op1VP, Op2VP);
1771 return VecCost - ScalarCost;
1772 }
1773 case Instruction::GetElementPtr: {
1774 TargetTransformInfo::OperandValueKind Op1VK =
1775 TargetTransformInfo::OK_AnyValue;
1776 TargetTransformInfo::OperandValueKind Op2VK =
1777 TargetTransformInfo::OK_UniformConstantValue;
1778
1779 int ScalarCost =
1780 VecTy->getNumElements() *
1781 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
1782 int VecCost =
1783 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
1784
1785 return VecCost - ScalarCost;
1786 }
1787 case Instruction::Load: {
1788 // Cost of wide load - cost of scalar loads.
1789 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
1790 int ScalarLdCost = VecTy->getNumElements() *
1791 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0);
1792 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
1793 VecTy, alignment, 0);
1794 if (E->NeedToShuffle) {
1795 VecLdCost += TTI->getShuffleCost(
1796 TargetTransformInfo::SK_PermuteSingleSrc, VecTy, 0);
1797 }
1798 return VecLdCost - ScalarLdCost;
1799 }
1800 case Instruction::Store: {
1801 // We know that we can merge the stores. Calculate the cost.
1802 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
1803 int ScalarStCost = VecTy->getNumElements() *
1804 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0);
1805 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
1806 VecTy, alignment, 0);
1807 return VecStCost - ScalarStCost;
1808 }
1809 case Instruction::Call: {
1810 CallInst *CI = cast<CallInst>(VL0);
1811 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1812
1813 // Calculate the cost of the scalar and vector calls.
1814 SmallVector<Type*, 4> ScalarTys, VecTys;
1815 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op) {
1816 ScalarTys.push_back(CI->getArgOperand(op)->getType());
1817 VecTys.push_back(VectorType::get(CI->getArgOperand(op)->getType(),
1818 VecTy->getNumElements()));
1819 }
1820
1821 FastMathFlags FMF;
1822 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
1823 FMF = FPMO->getFastMathFlags();
1824
1825 int ScalarCallCost = VecTy->getNumElements() *
1826 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
1827
1828 int VecCallCost = TTI->getIntrinsicInstrCost(ID, VecTy, VecTys, FMF);
1829
1830 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
1831 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
1832 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
1833
1834 return VecCallCost - ScalarCallCost;
1835 }
1836 case Instruction::ShuffleVector: {
1837 TargetTransformInfo::OperandValueKind Op1VK =
1838 TargetTransformInfo::OK_AnyValue;
1839 TargetTransformInfo::OperandValueKind Op2VK =
1840 TargetTransformInfo::OK_AnyValue;
1841 int ScalarCost = 0;
1842 int VecCost = 0;
1843 for (Value *i : VL) {
1844 Instruction *I = cast<Instruction>(i);
1845 if (!I)
1846 break;
1847 ScalarCost +=
1848 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
1849 }
1850 // VecCost is equal to sum of the cost of creating 2 vectors
1851 // and the cost of creating shuffle.
1852 Instruction *I0 = cast<Instruction>(VL[0]);
1853 VecCost =
1854 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
1855 Instruction *I1 = cast<Instruction>(VL[1]);
1856 VecCost +=
1857 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
1858 VecCost +=
1859 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
1860 return VecCost - ScalarCost;
1861 }
1862 default:
1863 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1863)
;
1864 }
1865}
1866
1867bool BoUpSLP::isFullyVectorizableTinyTree() {
1868 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
1869 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
1870
1871 // We only handle trees of heights 1 and 2.
1872 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
1873 return true;
1874
1875 if (VectorizableTree.size() != 2)
1876 return false;
1877
1878 // Handle splat and all-constants stores.
1879 if (!VectorizableTree[0].NeedToGather &&
1880 (allConstant(VectorizableTree[1].Scalars) ||
1881 isSplat(VectorizableTree[1].Scalars)))
1882 return true;
1883
1884 // Gathering cost would be too much for tiny trees.
1885 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
1886 return false;
1887
1888 return true;
1889}
1890
1891bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
1892
1893 // We can vectorize the tree if its size is greater than or equal to the
1894 // minimum size specified by the MinTreeSize command line option.
1895 if (VectorizableTree.size() >= MinTreeSize)
1896 return false;
1897
1898 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
1899 // can vectorize it if we can prove it fully vectorizable.
1900 if (isFullyVectorizableTinyTree())
1901 return false;
1902
1903 assert(VectorizableTree.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1905, __PRETTY_FUNCTION__))
1904 ? ExternalUses.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1905, __PRETTY_FUNCTION__))
1905 : true && "We shouldn't have any external users")((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1905, __PRETTY_FUNCTION__))
;
1906
1907 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
1908 // vectorizable.
1909 return true;
1910}
1911
1912int BoUpSLP::getSpillCost() {
1913 // Walk from the bottom of the tree to the top, tracking which values are
1914 // live. When we see a call instruction that is not part of our tree,
1915 // query TTI to see if there is a cost to keeping values live over it
1916 // (for example, if spills and fills are required).
1917 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
1918 int Cost = 0;
1919
1920 SmallPtrSet<Instruction*, 4> LiveValues;
1921 Instruction *PrevInst = nullptr;
1922
1923 for (const auto &N : VectorizableTree) {
1924 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
1925 if (!Inst)
1926 continue;
1927
1928 if (!PrevInst) {
1929 PrevInst = Inst;
1930 continue;
1931 }
1932
1933 // Update LiveValues.
1934 LiveValues.erase(PrevInst);
1935 for (auto &J : PrevInst->operands()) {
1936 if (isa<Instruction>(&*J) && ScalarToTreeEntry.count(&*J))
1937 LiveValues.insert(cast<Instruction>(&*J));
1938 }
1939
1940 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1941 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1942 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1943 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1944 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1945 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1946 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
;
1947
1948 // Now find the sequence of instructions between PrevInst and Inst.
1949 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
1950 PrevInstIt =
1951 PrevInst->getIterator().getReverse();
1952 while (InstIt != PrevInstIt) {
1953 if (PrevInstIt == PrevInst->getParent()->rend()) {
1954 PrevInstIt = Inst->getParent()->rbegin();
1955 continue;
1956 }
1957
1958 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
1959 SmallVector<Type*, 4> V;
1960 for (auto *II : LiveValues)
1961 V.push_back(VectorType::get(II->getType(), BundleWidth));
1962 Cost += TTI->getCostOfKeepingLiveOverCall(V);
1963 }
1964
1965 ++PrevInstIt;
1966 }
1967
1968 PrevInst = Inst;
1969 }
1970
1971 return Cost;
1972}
1973
1974int BoUpSLP::getTreeCost() {
1975 int Cost = 0;
1976 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
1977 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
1978
1979 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
1980
1981 for (TreeEntry &TE : VectorizableTree) {
1982 int C = getEntryCost(&TE);
1983 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
1984 << *TE.Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
1985 Cost += C;
1986 }
1987
1988 SmallSet<Value *, 16> ExtractCostCalculated;
1989 int ExtractCost = 0;
1990 for (ExternalUser &EU : ExternalUses) {
1991 // We only add extract cost once for the same scalar.
1992 if (!ExtractCostCalculated.insert(EU.Scalar).second)
1993 continue;
1994
1995 // Uses by ephemeral values are free (because the ephemeral value will be
1996 // removed prior to code generation, and so the extraction will be
1997 // removed as well).
1998 if (EphValues.count(EU.User))
1999 continue;
2000
2001 // If we plan to rewrite the tree in a smaller type, we will need to sign
2002 // extend the extracted value back to the original type. Here, we account
2003 // for the extract and the added cost of the sign extend if needed.
2004 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2005 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2006 if (MinBWs.count(ScalarRoot)) {
2007 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2008 auto Extend =
2009 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2010 VecTy = VectorType::get(MinTy, BundleWidth);
2011 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2012 VecTy, EU.Lane);
2013 } else {
2014 ExtractCost +=
2015 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2016 }
2017 }
2018
2019 int SpillCost = getSpillCost();
2020 Cost += SpillCost + ExtractCost;
2021
2022 DEBUG(dbgs() << "SLP: Spill Cost = " << SpillCost << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Spill Cost = " << SpillCost
<< ".\n" << "SLP: Extract Cost = " << ExtractCost
<< ".\n" << "SLP: Total Cost = " << Cost <<
".\n"; } } while (false)
2023 << "SLP: Extract Cost = " << ExtractCost << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Spill Cost = " << SpillCost
<< ".\n" << "SLP: Extract Cost = " << ExtractCost
<< ".\n" << "SLP: Total Cost = " << Cost <<
".\n"; } } while (false)
2024 << "SLP: Total Cost = " << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Spill Cost = " << SpillCost
<< ".\n" << "SLP: Extract Cost = " << ExtractCost
<< ".\n" << "SLP: Total Cost = " << Cost <<
".\n"; } } while (false)
;
2025 return Cost;
2026}
2027
2028int BoUpSLP::getGatherCost(Type *Ty) {
2029 int Cost = 0;
2030 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2031 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2032 return Cost;
2033}
2034
2035int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2036 // Find the type of the operands in VL.
2037 Type *ScalarTy = VL[0]->getType();
2038 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2039 ScalarTy = SI->getValueOperand()->getType();
2040 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2041 // Find the cost of inserting/extracting values from the vector.
2042 return getGatherCost(VecTy);
2043}
2044
2045// Reorder commutative operations in alternate shuffle if the resulting vectors
2046// are consecutive loads. This would allow us to vectorize the tree.
2047// If we have something like-
2048// load a[0] - load b[0]
2049// load b[1] + load a[1]
2050// load a[2] - load b[2]
2051// load a[3] + load b[3]
2052// Reordering the second load b[1] load a[1] would allow us to vectorize this
2053// code.
2054void BoUpSLP::reorderAltShuffleOperands(ArrayRef<Value *> VL,
2055 SmallVectorImpl<Value *> &Left,
2056 SmallVectorImpl<Value *> &Right) {
2057 // Push left and right operands of binary operation into Left and Right
2058 for (Value *i : VL) {
2059 Left.push_back(cast<Instruction>(i)->getOperand(0));
2060 Right.push_back(cast<Instruction>(i)->getOperand(1));
2061 }
2062
2063 // Reorder if we have a commutative operation and consecutive access
2064 // are on either side of the alternate instructions.
2065 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2066 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2067 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2068 Instruction *VL1 = cast<Instruction>(VL[j]);
2069 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2070 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2071 std::swap(Left[j], Right[j]);
2072 continue;
2073 } else if (VL2->isCommutative() &&
2074 isConsecutiveAccess(L, L1, *DL, *SE)) {
2075 std::swap(Left[j + 1], Right[j + 1]);
2076 continue;
2077 }
2078 // else unchanged
2079 }
2080 }
2081 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2082 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2083 Instruction *VL1 = cast<Instruction>(VL[j]);
2084 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2085 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2086 std::swap(Left[j], Right[j]);
2087 continue;
2088 } else if (VL2->isCommutative() &&
2089 isConsecutiveAccess(L, L1, *DL, *SE)) {
2090 std::swap(Left[j + 1], Right[j + 1]);
2091 continue;
2092 }
2093 // else unchanged
2094 }
2095 }
2096 }
2097}
2098
2099// Return true if I should be commuted before adding it's left and right
2100// operands to the arrays Left and Right.
2101//
2102// The vectorizer is trying to either have all elements one side being
2103// instruction with the same opcode to enable further vectorization, or having
2104// a splat to lower the vectorizing cost.
2105static bool shouldReorderOperands(int i, Instruction &I,
2106 SmallVectorImpl<Value *> &Left,
2107 SmallVectorImpl<Value *> &Right,
2108 bool AllSameOpcodeLeft,
2109 bool AllSameOpcodeRight, bool SplatLeft,
2110 bool SplatRight) {
2111 Value *VLeft = I.getOperand(0);
2112 Value *VRight = I.getOperand(1);
2113 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2114 if (SplatRight) {
2115 if (VRight == Right[i - 1])
2116 // Preserve SplatRight
2117 return false;
2118 if (VLeft == Right[i - 1]) {
2119 // Commuting would preserve SplatRight, but we don't want to break
2120 // SplatLeft either, i.e. preserve the original order if possible.
2121 // (FIXME: why do we care?)
2122 if (SplatLeft && VLeft == Left[i - 1])
2123 return false;
2124 return true;
2125 }
2126 }
2127 // Symmetrically handle Right side.
2128 if (SplatLeft) {
2129 if (VLeft == Left[i - 1])
2130 // Preserve SplatLeft
2131 return false;
2132 if (VRight == Left[i - 1])
2133 return true;
2134 }
2135
2136 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2137 Instruction *IRight = dyn_cast<Instruction>(VRight);
2138
2139 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2140 // it and not the right, in this case we want to commute.
2141 if (AllSameOpcodeRight) {
2142 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2143 if (IRight && RightPrevOpcode == IRight->getOpcode())
2144 // Do not commute, a match on the right preserves AllSameOpcodeRight
2145 return false;
2146 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2147 // We have a match and may want to commute, but first check if there is
2148 // not also a match on the existing operands on the Left to preserve
2149 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2150 // (FIXME: why do we care?)
2151 if (AllSameOpcodeLeft && ILeft &&
2152 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2153 return false;
2154 return true;
2155 }
2156 }
2157 // Symmetrically handle Left side.
2158 if (AllSameOpcodeLeft) {
2159 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2160 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2161 return false;
2162 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2163 return true;
2164 }
2165 return false;
2166}
2167
2168void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
2169 SmallVectorImpl<Value *> &Left,
2170 SmallVectorImpl<Value *> &Right) {
2171
2172 if (VL.size()) {
2173 // Peel the first iteration out of the loop since there's nothing
2174 // interesting to do anyway and it simplifies the checks in the loop.
2175 auto VLeft = cast<Instruction>(VL[0])->getOperand(0);
2176 auto VRight = cast<Instruction>(VL[0])->getOperand(1);
2177 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2178 // Favor having instruction to the right. FIXME: why?
2179 std::swap(VLeft, VRight);
2180 Left.push_back(VLeft);
2181 Right.push_back(VRight);
2182 }
2183
2184 // Keep track if we have instructions with all the same opcode on one side.
2185 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2186 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2187 // Keep track if we have one side with all the same value (broadcast).
2188 bool SplatLeft = true;
2189 bool SplatRight = true;
2190
2191 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2192 Instruction *I = cast<Instruction>(VL[i]);
2193 assert(I->isCommutative() && "Can only process commutative instruction")((I->isCommutative() && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("I->isCommutative() && \"Can only process commutative instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2193, __PRETTY_FUNCTION__))
;
2194 // Commute to favor either a splat or maximizing having the same opcodes on
2195 // one side.
2196 if (shouldReorderOperands(i, *I, Left, Right, AllSameOpcodeLeft,
2197 AllSameOpcodeRight, SplatLeft, SplatRight)) {
2198 Left.push_back(I->getOperand(1));
2199 Right.push_back(I->getOperand(0));
2200 } else {
2201 Left.push_back(I->getOperand(0));
2202 Right.push_back(I->getOperand(1));
2203 }
2204 // Update Splat* and AllSameOpcode* after the insertion.
2205 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2206 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2207 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2208 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2209 cast<Instruction>(Left[i])->getOpcode());
2210 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2211 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2212 cast<Instruction>(Right[i])->getOpcode());
2213 }
2214
2215 // If one operand end up being broadcast, return this operand order.
2216 if (SplatRight || SplatLeft)
2217 return;
2218
2219 // Finally check if we can get longer vectorizable chain by reordering
2220 // without breaking the good operand order detected above.
2221 // E.g. If we have something like-
2222 // load a[0] load b[0]
2223 // load b[1] load a[1]
2224 // load a[2] load b[2]
2225 // load a[3] load b[3]
2226 // Reordering the second load b[1] load a[1] would allow us to vectorize
2227 // this code and we still retain AllSameOpcode property.
2228 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2229 // such as-
2230 // add a[0],c[0] load b[0]
2231 // add a[1],c[2] load b[1]
2232 // b[2] load b[2]
2233 // add a[3],c[3] load b[3]
2234 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2235 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2236 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2237 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2238 std::swap(Left[j + 1], Right[j + 1]);
2239 continue;
2240 }
2241 }
2242 }
2243 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2244 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2245 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2246 std::swap(Left[j + 1], Right[j + 1]);
2247 continue;
2248 }
2249 }
2250 }
2251 // else unchanged
2252 }
2253}
2254
2255void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL) {
2256
2257 // Get the basic block this bundle is in. All instructions in the bundle
2258 // should be in this block.
2259 auto *Front = cast<Instruction>(VL.front());
2260 auto *BB = Front->getParent();
2261 assert(all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool {((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2263, __PRETTY_FUNCTION__))
2262 return cast<Instruction>(V)->getParent() == BB;((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2263, __PRETTY_FUNCTION__))
2263 }))((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2263, __PRETTY_FUNCTION__))
;
2264
2265 // The last instruction in the bundle in program order.
2266 Instruction *LastInst = nullptr;
2267
2268 // Find the last instruction. The common case should be that BB has been
2269 // scheduled, and the last instruction is VL.back(). So we start with
2270 // VL.back() and iterate over schedule data until we reach the end of the
2271 // bundle. The end of the bundle is marked by null ScheduleData.
2272 if (BlocksSchedules.count(BB)) {
2273 auto *Bundle = BlocksSchedules[BB]->getScheduleData(VL.back());
2274 if (Bundle && Bundle->isPartOfBundle())
2275 for (; Bundle; Bundle = Bundle->NextInBundle)
2276 LastInst = Bundle->Inst;
2277 }
2278
2279 // LastInst can still be null at this point if there's either not an entry
2280 // for BB in BlocksSchedules or there's no ScheduleData available for
2281 // VL.back(). This can be the case if buildTree_rec aborts for various
2282 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2283 // size is reached, etc.). ScheduleData is initialized in the scheduling
2284 // "dry-run".
2285 //
2286 // If this happens, we can still find the last instruction by brute force. We
2287 // iterate forwards from Front (inclusive) until we either see all
2288 // instructions in the bundle or reach the end of the block. If Front is the
2289 // last instruction in program order, LastInst will be set to Front, and we
2290 // will visit all the remaining instructions in the block.
2291 //
2292 // One of the reasons we exit early from buildTree_rec is to place an upper
2293 // bound on compile-time. Thus, taking an additional compile-time hit here is
2294 // not ideal. However, this should be exceedingly rare since it requires that
2295 // we both exit early from buildTree_rec and that the bundle be out-of-order
2296 // (causing us to iterate all the way to the end of the block).
2297 if (!LastInst) {
2298 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2299 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2300 if (Bundle.erase(&I))
2301 LastInst = &I;
2302 if (Bundle.empty())
2303 break;
2304 }
2305 }
2306
2307 // Set the insertion point after the last instruction in the bundle. Set the
2308 // debug location to Front.
2309 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2310 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2311}
2312
2313Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2314 Value *Vec = UndefValue::get(Ty);
2315 // Generate the 'InsertElement' instruction.
2316 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2317 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2318 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2319 GatherSeq.insert(Insrt);
2320 CSEBlocks.insert(Insrt->getParent());
2321
2322 // Add to our 'need-to-extract' list.
2323 if (ScalarToTreeEntry.count(VL[i])) {
2324 int Idx = ScalarToTreeEntry[VL[i]];
2325 TreeEntry *E = &VectorizableTree[Idx];
2326 // Find which lane we need to extract.
2327 int FoundLane = -1;
2328 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2329 // Is this the lane of the scalar that we are looking for ?
2330 if (E->Scalars[Lane] == VL[i]) {
2331 FoundLane = Lane;
2332 break;
2333 }
2334 }
2335 assert(FoundLane >= 0 && "Could not find the correct lane")((FoundLane >= 0 && "Could not find the correct lane"
) ? static_cast<void> (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2335, __PRETTY_FUNCTION__))
;
2336 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2337 }
2338 }
2339 }
2340
2341 return Vec;
2342}
2343
2344Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL) const {
2345 SmallDenseMap<Value*, int>::const_iterator Entry
2346 = ScalarToTreeEntry.find(VL[0]);
2347 if (Entry != ScalarToTreeEntry.end()) {
2348 int Idx = Entry->second;
2349 const TreeEntry *En = &VectorizableTree[Idx];
2350 if (En->isSame(VL) && En->VectorizedValue)
2351 return En->VectorizedValue;
2352 }
2353 return nullptr;
2354}
2355
2356Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2357 if (ScalarToTreeEntry.count(VL[0])) {
2358 int Idx = ScalarToTreeEntry[VL[0]];
2359 TreeEntry *E = &VectorizableTree[Idx];
2360 if (E->isSame(VL) || (E->NeedToShuffle && E->isFoundJumbled(VL, *DL, *SE)))
2361 return vectorizeTree(VL, E);
2362 }
2363
2364 Type *ScalarTy = VL[0]->getType();
2365 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2366 ScalarTy = SI->getValueOperand()->getType();
2367 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2368
2369 return Gather(VL, VecTy);
2370}
2371
2372Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL, TreeEntry *E) {
2373 IRBuilder<>::InsertPointGuard Guard(Builder);
2374
2375 if (E->VectorizedValue && !E->NeedToShuffle) {
2376 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
2377 return E->VectorizedValue;
2378 }
2379
2380 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2381 Type *ScalarTy = VL0->getType();
2382 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2383 ScalarTy = SI->getValueOperand()->getType();
2384 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2385
2386 if (E->NeedToGather) {
2387 setInsertPointAfterBundle(E->Scalars);
2388 auto *V = Gather(E->Scalars, VecTy);
2389 E->VectorizedValue = V;
2390 return V;
2391 }
2392
2393 unsigned Opcode = getSameOpcode(E->Scalars);
2394
2395 switch (Opcode) {
2396 case Instruction::PHI: {
2397 PHINode *PH = dyn_cast<PHINode>(VL0);
2398 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2399 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2400 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2401 E->VectorizedValue = NewPhi;
2402
2403 // PHINodes may have multiple entries from the same block. We want to
2404 // visit every block once.
2405 SmallSet<BasicBlock*, 4> VisitedBBs;
2406
2407 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2408 ValueList Operands;
2409 BasicBlock *IBB = PH->getIncomingBlock(i);
2410
2411 if (!VisitedBBs.insert(IBB).second) {
2412 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2413 continue;
2414 }
2415
2416 // Prepare the operand vector.
2417 for (Value *V : E->Scalars)
2418 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2419
2420 Builder.SetInsertPoint(IBB->getTerminator());
2421 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2422 Value *Vec = vectorizeTree(Operands);
2423 NewPhi->addIncoming(Vec, IBB);
2424 }
2425
2426 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2427, __PRETTY_FUNCTION__))
2427 "Invalid number of incoming values")((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2427, __PRETTY_FUNCTION__))
;
2428 return NewPhi;
2429 }
2430
2431 case Instruction::ExtractElement: {
2432 if (canReuseExtract(E->Scalars, Instruction::ExtractElement)) {
2433 Value *V = VL0->getOperand(0);
2434 E->VectorizedValue = V;
2435 return V;
2436 }
2437 setInsertPointAfterBundle(E->Scalars);
2438 auto *V = Gather(E->Scalars, VecTy);
2439 E->VectorizedValue = V;
2440 return V;
2441 }
2442 case Instruction::ExtractValue: {
2443 if (canReuseExtract(E->Scalars, Instruction::ExtractValue)) {
2444 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2445 Builder.SetInsertPoint(LI);
2446 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2447 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2448 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2449 E->VectorizedValue = V;
2450 return propagateMetadata(V, E->Scalars);
2451 }
2452 setInsertPointAfterBundle(E->Scalars);
2453 auto *V = Gather(E->Scalars, VecTy);
2454 E->VectorizedValue = V;
2455 return V;
2456 }
2457 case Instruction::ZExt:
2458 case Instruction::SExt:
2459 case Instruction::FPToUI:
2460 case Instruction::FPToSI:
2461 case Instruction::FPExt:
2462 case Instruction::PtrToInt:
2463 case Instruction::IntToPtr:
2464 case Instruction::SIToFP:
2465 case Instruction::UIToFP:
2466 case Instruction::Trunc:
2467 case Instruction::FPTrunc:
2468 case Instruction::BitCast: {
2469 ValueList INVL;
2470 for (Value *V : E->Scalars)
2471 INVL.push_back(cast<Instruction>(V)->getOperand(0));
2472
2473 setInsertPointAfterBundle(E->Scalars);
2474
2475 Value *InVec = vectorizeTree(INVL);
2476
2477 if (Value *V = alreadyVectorized(E->Scalars))
2478 return V;
2479
2480 CastInst *CI = dyn_cast<CastInst>(VL0);
2481 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2482 E->VectorizedValue = V;
2483 ++NumVectorInstructions;
2484 return V;
2485 }
2486 case Instruction::FCmp:
2487 case Instruction::ICmp: {
2488 ValueList LHSV, RHSV;
2489 for (Value *V : E->Scalars) {
2490 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2491 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2492 }
2493
2494 setInsertPointAfterBundle(E->Scalars);
2495
2496 Value *L = vectorizeTree(LHSV);
2497 Value *R = vectorizeTree(RHSV);
2498
2499 if (Value *V = alreadyVectorized(E->Scalars))
2500 return V;
2501
2502 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2503 Value *V;
2504 if (Opcode == Instruction::FCmp)
2505 V = Builder.CreateFCmp(P0, L, R);
2506 else
2507 V = Builder.CreateICmp(P0, L, R);
2508
2509 E->VectorizedValue = V;
2510 propagateIRFlags(E->VectorizedValue, E->Scalars);
2511 ++NumVectorInstructions;
2512 return V;
2513 }
2514 case Instruction::Select: {
2515 ValueList TrueVec, FalseVec, CondVec;
2516 for (Value *V : E->Scalars) {
2517 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2518 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2519 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2520 }
2521
2522 setInsertPointAfterBundle(E->Scalars);
2523
2524 Value *Cond = vectorizeTree(CondVec);
2525 Value *True = vectorizeTree(TrueVec);
2526 Value *False = vectorizeTree(FalseVec);
2527
2528 if (Value *V = alreadyVectorized(E->Scalars))
2529 return V;
2530
2531 Value *V = Builder.CreateSelect(Cond, True, False);
2532 E->VectorizedValue = V;
2533 ++NumVectorInstructions;
2534 return V;
2535 }
2536 case Instruction::Add:
2537 case Instruction::FAdd:
2538 case Instruction::Sub:
2539 case Instruction::FSub:
2540 case Instruction::Mul:
2541 case Instruction::FMul:
2542 case Instruction::UDiv:
2543 case Instruction::SDiv:
2544 case Instruction::FDiv:
2545 case Instruction::URem:
2546 case Instruction::SRem:
2547 case Instruction::FRem:
2548 case Instruction::Shl:
2549 case Instruction::LShr:
2550 case Instruction::AShr:
2551 case Instruction::And:
2552 case Instruction::Or:
2553 case Instruction::Xor: {
2554 ValueList LHSVL, RHSVL;
2555 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2556 reorderInputsAccordingToOpcode(E->Scalars, LHSVL, RHSVL);
2557 else
2558 for (Value *V : E->Scalars) {
2559 LHSVL.push_back(cast<Instruction>(V)->getOperand(0));
2560 RHSVL.push_back(cast<Instruction>(V)->getOperand(1));
2561 }
2562
2563 setInsertPointAfterBundle(E->Scalars);
2564
2565 Value *LHS = vectorizeTree(LHSVL);
2566 Value *RHS = vectorizeTree(RHSVL);
2567
2568 if (Value *V = alreadyVectorized(E->Scalars))
2569 return V;
2570
2571 BinaryOperator *BinOp = cast<BinaryOperator>(VL0);
2572 Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS);
2573 E->VectorizedValue = V;
2574 propagateIRFlags(E->VectorizedValue, E->Scalars);
2575 ++NumVectorInstructions;
2576
2577 if (Instruction *I = dyn_cast<Instruction>(V))
2578 return propagateMetadata(I, E->Scalars);
2579
2580 return V;
2581 }
2582 case Instruction::Load: {
2583 // Loads are inserted at the head of the tree because we don't want to
2584 // sink them all the way down past store instructions.
2585 setInsertPointAfterBundle(E->Scalars);
2586
2587 LoadInst *LI = cast<LoadInst>(VL0);
2588 Type *ScalarLoadTy = LI->getType();
2589 unsigned AS = LI->getPointerAddressSpace();
2590
2591 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2592 VecTy->getPointerTo(AS));
2593
2594 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2595 // ExternalUses list to make sure that an extract will be generated in the
2596 // future.
2597 if (ScalarToTreeEntry.count(LI->getPointerOperand()))
2598 ExternalUses.push_back(
2599 ExternalUser(LI->getPointerOperand(), cast<User>(VecPtr), 0));
2600
2601 unsigned Alignment = LI->getAlignment();
2602 LI = Builder.CreateLoad(VecPtr);
2603 if (!Alignment) {
2604 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2605 }
2606 LI->setAlignment(Alignment);
2607 E->VectorizedValue = LI;
2608 ++NumVectorInstructions;
2609 propagateMetadata(LI, E->Scalars);
2610
2611 // As program order of scalar loads are jumbled, the vectorized 'load'
2612 // must be followed by a 'shuffle' with the required jumbled mask.
2613 if (!VL.empty() && (E->NeedToShuffle)) {
2614 assert(VL.size() == E->Scalars.size() &&((VL.size() == E->Scalars.size() && "Equal number of scalars expected"
) ? static_cast<void> (0) : __assert_fail ("VL.size() == E->Scalars.size() && \"Equal number of scalars expected\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2615, __PRETTY_FUNCTION__))
2615 "Equal number of scalars expected")((VL.size() == E->Scalars.size() && "Equal number of scalars expected"
) ? static_cast<void> (0) : __assert_fail ("VL.size() == E->Scalars.size() && \"Equal number of scalars expected\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2615, __PRETTY_FUNCTION__))
;
2616 SmallVector<Constant *, 8> Mask;
2617 for (Value *Val : VL) {
2618 if (ScalarToTreeEntry.count(Val)) {
2619 int Idx = ScalarToTreeEntry[Val];
2620 TreeEntry *E = &VectorizableTree[Idx];
2621 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2622 if (E->Scalars[Lane] == Val) {
2623 Mask.push_back(Builder.getInt32(Lane));
2624 break;
2625 }
2626 }
2627 }
2628 }
2629
2630 // Generate shuffle for jumbled memory access
2631 Value *Undef = UndefValue::get(VecTy);
2632 Value *Shuf = Builder.CreateShuffleVector((Value *)LI, Undef,
2633 ConstantVector::get(Mask));
2634 return Shuf;
2635 }
2636
2637 return LI;
2638 }
2639 case Instruction::Store: {
2640 StoreInst *SI = cast<StoreInst>(VL0);
2641 unsigned Alignment = SI->getAlignment();
2642 unsigned AS = SI->getPointerAddressSpace();
2643
2644 ValueList ValueOp;
2645 for (Value *V : E->Scalars)
2646 ValueOp.push_back(cast<StoreInst>(V)->getValueOperand());
2647
2648 setInsertPointAfterBundle(E->Scalars);
2649
2650 Value *VecValue = vectorizeTree(ValueOp);
2651 Value *VecPtr = Builder.CreateBitCast(SI->getPointerOperand(),
2652 VecTy->getPointerTo(AS));
2653 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
2654
2655 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2656 // ExternalUses list to make sure that an extract will be generated in the
2657 // future.
2658 if (ScalarToTreeEntry.count(SI->getPointerOperand()))
2659 ExternalUses.push_back(
2660 ExternalUser(SI->getPointerOperand(), cast<User>(VecPtr), 0));
2661
2662 if (!Alignment) {
2663 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
2664 }
2665 S->setAlignment(Alignment);
2666 E->VectorizedValue = S;
2667 ++NumVectorInstructions;
2668 return propagateMetadata(S, E->Scalars);
2669 }
2670 case Instruction::GetElementPtr: {
2671 setInsertPointAfterBundle(E->Scalars);
2672
2673 ValueList Op0VL;
2674 for (Value *V : E->Scalars)
2675 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
2676
2677 Value *Op0 = vectorizeTree(Op0VL);
2678
2679 std::vector<Value *> OpVecs;
2680 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
2681 ++j) {
2682 ValueList OpVL;
2683 for (Value *V : E->Scalars)
2684 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
2685
2686 Value *OpVec = vectorizeTree(OpVL);
2687 OpVecs.push_back(OpVec);
2688 }
2689
2690 Value *V = Builder.CreateGEP(
2691 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
2692 E->VectorizedValue = V;
2693 ++NumVectorInstructions;
2694
2695 if (Instruction *I = dyn_cast<Instruction>(V))
2696 return propagateMetadata(I, E->Scalars);
2697
2698 return V;
2699 }
2700 case Instruction::Call: {
2701 CallInst *CI = cast<CallInst>(VL0);
2702 setInsertPointAfterBundle(E->Scalars);
2703 Function *FI;
2704 Intrinsic::ID IID = Intrinsic::not_intrinsic;
2705 Value *ScalarArg = nullptr;
2706 if (CI && (FI = CI->getCalledFunction())) {
2707 IID = FI->getIntrinsicID();
2708 }
2709 std::vector<Value *> OpVecs;
2710 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
2711 ValueList OpVL;
2712 // ctlz,cttz and powi are special intrinsics whose second argument is
2713 // a scalar. This argument should not be vectorized.
2714 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
2715 CallInst *CEI = cast<CallInst>(E->Scalars[0]);
2716 ScalarArg = CEI->getArgOperand(j);
2717 OpVecs.push_back(CEI->getArgOperand(j));
2718 continue;
2719 }
2720 for (Value *V : E->Scalars) {
2721 CallInst *CEI = cast<CallInst>(V);
2722 OpVL.push_back(CEI->getArgOperand(j));
2723 }
2724
2725 Value *OpVec = vectorizeTree(OpVL);
2726 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
2727 OpVecs.push_back(OpVec);
2728 }
2729
2730 Module *M = F->getParent();
2731 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2732 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
2733 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
2734 SmallVector<OperandBundleDef, 1> OpBundles;
2735 CI->getOperandBundlesAsDefs(OpBundles);
2736 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
2737
2738 // The scalar argument uses an in-tree scalar so we add the new vectorized
2739 // call to ExternalUses list to make sure that an extract will be
2740 // generated in the future.
2741 if (ScalarArg && ScalarToTreeEntry.count(ScalarArg))
2742 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
2743
2744 E->VectorizedValue = V;
2745 propagateIRFlags(E->VectorizedValue, E->Scalars);
2746 ++NumVectorInstructions;
2747 return V;
2748 }
2749 case Instruction::ShuffleVector: {
2750 ValueList LHSVL, RHSVL;
2751 assert(isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand")((isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(VL0) && \"Invalid Shuffle Vector Operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2751, __PRETTY_FUNCTION__))
;
2752 reorderAltShuffleOperands(E->Scalars, LHSVL, RHSVL);
2753 setInsertPointAfterBundle(E->Scalars);
2754
2755 Value *LHS = vectorizeTree(LHSVL);
2756 Value *RHS = vectorizeTree(RHSVL);
2757
2758 if (Value *V = alreadyVectorized(E->Scalars))
2759 return V;
2760
2761 // Create a vector of LHS op1 RHS
2762 BinaryOperator *BinOp0 = cast<BinaryOperator>(VL0);
2763 Value *V0 = Builder.CreateBinOp(BinOp0->getOpcode(), LHS, RHS);
2764
2765 // Create a vector of LHS op2 RHS
2766 Instruction *VL1 = cast<Instruction>(E->Scalars[1]);
2767 BinaryOperator *BinOp1 = cast<BinaryOperator>(VL1);
2768 Value *V1 = Builder.CreateBinOp(BinOp1->getOpcode(), LHS, RHS);
2769
2770 // Create shuffle to take alternate operations from the vector.
2771 // Also, gather up odd and even scalar ops to propagate IR flags to
2772 // each vector operation.
2773 ValueList OddScalars, EvenScalars;
2774 unsigned e = E->Scalars.size();
2775 SmallVector<Constant *, 8> Mask(e);
2776 for (unsigned i = 0; i < e; ++i) {
2777 if (i & 1) {
2778 Mask[i] = Builder.getInt32(e + i);
2779 OddScalars.push_back(E->Scalars[i]);
2780 } else {
2781 Mask[i] = Builder.getInt32(i);
2782 EvenScalars.push_back(E->Scalars[i]);
2783 }
2784 }
2785
2786 Value *ShuffleMask = ConstantVector::get(Mask);
2787 propagateIRFlags(V0, EvenScalars);
2788 propagateIRFlags(V1, OddScalars);
2789
2790 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2791 E->VectorizedValue = V;
2792 ++NumVectorInstructions;
2793 if (Instruction *I = dyn_cast<Instruction>(V))
2794 return propagateMetadata(I, E->Scalars);
2795
2796 return V;
2797 }
2798 default:
2799 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2799)
;
2800 }
2801 return nullptr;
2802}
2803
2804Value *BoUpSLP::vectorizeTree() {
2805 ExtraValueToDebugLocsMap ExternallyUsedValues;
2806 return vectorizeTree(ExternallyUsedValues);
2807}
2808
2809Value *
2810BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
2811
2812 // All blocks must be scheduled before any instructions are inserted.
2813 for (auto &BSIter : BlocksSchedules) {
2814 scheduleBlock(BSIter.second.get());
2815 }
2816
2817 Builder.SetInsertPoint(&F->getEntryBlock().front());
2818 auto *VectorRoot = vectorizeTree(ArrayRef<Value *>(), &VectorizableTree[0]);
2819
2820 // If the vectorized tree can be rewritten in a smaller type, we truncate the
2821 // vectorized root. InstCombine will then rewrite the entire expression. We
2822 // sign extend the extracted values below.
2823 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2824 if (MinBWs.count(ScalarRoot)) {
2825 if (auto *I = dyn_cast<Instruction>(VectorRoot))
2826 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
2827 auto BundleWidth = VectorizableTree[0].Scalars.size();
2828 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2829 auto *VecTy = VectorType::get(MinTy, BundleWidth);
2830 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
2831 VectorizableTree[0].VectorizedValue = Trunc;
2832 }
2833
2834 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
2835
2836 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
2837 // specified by ScalarType.
2838 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
2839 if (!MinBWs.count(ScalarRoot))
2840 return Ex;
2841 if (MinBWs[ScalarRoot].second)
2842 return Builder.CreateSExt(Ex, ScalarType);
2843 return Builder.CreateZExt(Ex, ScalarType);
2844 };
2845
2846 // Extract all of the elements with the external uses.
2847 for (const auto &ExternalUse : ExternalUses) {
2848 Value *Scalar = ExternalUse.Scalar;
2849 llvm::User *User = ExternalUse.User;
2850
2851 // Skip users that we already RAUW. This happens when one instruction
2852 // has multiple uses of the same value.
2853 if (User && !is_contained(Scalar->users(), User))
2854 continue;
2855 assert(ScalarToTreeEntry.count(Scalar) && "Invalid scalar")((ScalarToTreeEntry.count(Scalar) && "Invalid scalar"
) ? static_cast<void> (0) : __assert_fail ("ScalarToTreeEntry.count(Scalar) && \"Invalid scalar\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2855, __PRETTY_FUNCTION__))
;
2856
2857 int Idx = ScalarToTreeEntry[Scalar];
2858 TreeEntry *E = &VectorizableTree[Idx];
2859 assert(!E->NeedToGather && "Extracting from a gather list")((!E->NeedToGather && "Extracting from a gather list"
) ? static_cast<void> (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2859, __PRETTY_FUNCTION__))
;
2860
2861 Value *Vec = E->VectorizedValue;
2862 assert(Vec && "Can't find vectorizable value")((Vec && "Can't find vectorizable value") ? static_cast
<void> (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2862, __PRETTY_FUNCTION__))
;
2863
2864 Value *Lane = Builder.getInt32(ExternalUse.Lane);
2865 // If User == nullptr, the Scalar is used as extra arg. Generate
2866 // ExtractElement instruction and update the record for this scalar in
2867 // ExternallyUsedValues.
2868 if (!User) {
2869 assert(ExternallyUsedValues.count(Scalar) &&((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2871, __PRETTY_FUNCTION__))
2870 "Scalar with nullptr as an external user must be registered in "((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2871, __PRETTY_FUNCTION__))
2871 "ExternallyUsedValues map")((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2871, __PRETTY_FUNCTION__))
;
2872 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
2873 Builder.SetInsertPoint(VecI->getParent(),
2874 std::next(VecI->getIterator()));
2875 } else {
2876 Builder.SetInsertPoint(&F->getEntryBlock().front());
2877 }
2878 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2879 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2880 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
2881 auto &Locs = ExternallyUsedValues[Scalar];
2882 ExternallyUsedValues.insert({Ex, Locs});
2883 ExternallyUsedValues.erase(Scalar);
2884 continue;
2885 }
2886
2887 // Generate extracts for out-of-tree users.
2888 // Find the insertion point for the extractelement lane.
2889 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
2890 if (PHINode *PH = dyn_cast<PHINode>(User)) {
2891 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
2892 if (PH->getIncomingValue(i) == Scalar) {
2893 TerminatorInst *IncomingTerminator =
2894 PH->getIncomingBlock(i)->getTerminator();
2895 if (isa<CatchSwitchInst>(IncomingTerminator)) {
2896 Builder.SetInsertPoint(VecI->getParent(),
2897 std::next(VecI->getIterator()));
2898 } else {
2899 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
2900 }
2901 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2902 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2903 CSEBlocks.insert(PH->getIncomingBlock(i));
2904 PH->setOperand(i, Ex);
2905 }
2906 }
2907 } else {
2908 Builder.SetInsertPoint(cast<Instruction>(User));
2909 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2910 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2911 CSEBlocks.insert(cast<Instruction>(User)->getParent());
2912 User->replaceUsesOfWith(Scalar, Ex);
2913 }
2914 } else {
2915 Builder.SetInsertPoint(&F->getEntryBlock().front());
2916 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2917 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2918 CSEBlocks.insert(&F->getEntryBlock());
2919 User->replaceUsesOfWith(Scalar, Ex);
2920 }
2921
2922 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
2923 }
2924
2925 // For each vectorized value:
2926 for (TreeEntry &EIdx : VectorizableTree) {
2927 TreeEntry *Entry = &EIdx;
2928
2929 // For each lane:
2930 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
2931 Value *Scalar = Entry->Scalars[Lane];
2932 // No need to handle users of gathered values.
2933 if (Entry->NeedToGather)
2934 continue;
2935
2936 assert(Entry->VectorizedValue && "Can't find vectorizable value")((Entry->VectorizedValue && "Can't find vectorizable value"
) ? static_cast<void> (0) : __assert_fail ("Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2936, __PRETTY_FUNCTION__))
;
2937
2938 Type *Ty = Scalar->getType();
2939 if (!Ty->isVoidTy()) {
2940#ifndef NDEBUG
2941 for (User *U : Scalar->users()) {
2942 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
2943
2944 assert((ScalarToTreeEntry.count(U) ||(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2947, __PRETTY_FUNCTION__))
2945 // It is legal to replace users in the ignorelist by undef.(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2947, __PRETTY_FUNCTION__))
2946 is_contained(UserIgnoreList, U)) &&(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2947, __PRETTY_FUNCTION__))
2947 "Replacing out-of-tree value with undef")(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2947, __PRETTY_FUNCTION__))
;
2948 }
2949#endif
2950 Value *Undef = UndefValue::get(Ty);
2951 Scalar->replaceAllUsesWith(Undef);
2952 }
2953 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
2954 eraseInstruction(cast<Instruction>(Scalar));
2955 }
2956 }
2957
2958 Builder.ClearInsertionPoint();
2959
2960 return VectorizableTree[0].VectorizedValue;
2961}
2962
2963void BoUpSLP::optimizeGatherSequence() {
2964 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
2965 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
2966 // LICM InsertElementInst sequences.
2967 for (Instruction *it : GatherSeq) {
2968 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
2969
2970 if (!Insert)
2971 continue;
2972
2973 // Check if this block is inside a loop.
2974 Loop *L = LI->getLoopFor(Insert->getParent());
2975 if (!L)
2976 continue;
2977
2978 // Check if it has a preheader.
2979 BasicBlock *PreHeader = L->getLoopPreheader();
2980 if (!PreHeader)
2981 continue;
2982
2983 // If the vector or the element that we insert into it are
2984 // instructions that are defined in this basic block then we can't
2985 // hoist this instruction.
2986 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
2987 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
2988 if (CurrVec && L->contains(CurrVec))
2989 continue;
2990 if (NewElem && L->contains(NewElem))
2991 continue;
2992
2993 // We can hoist this instruction. Move it to the pre-header.
2994 Insert->moveBefore(PreHeader->getTerminator());
2995 }
2996
2997 // Make a list of all reachable blocks in our CSE queue.
2998 SmallVector<const DomTreeNode *, 8> CSEWorkList;
2999 CSEWorkList.reserve(CSEBlocks.size());
3000 for (BasicBlock *BB : CSEBlocks)
3001 if (DomTreeNode *N = DT->getNode(BB)) {
3002 assert(DT->isReachableFromEntry(N))((DT->isReachableFromEntry(N)) ? static_cast<void> (
0) : __assert_fail ("DT->isReachableFromEntry(N)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3002, __PRETTY_FUNCTION__))
;
3003 CSEWorkList.push_back(N);
3004 }
3005
3006 // Sort blocks by domination. This ensures we visit a block after all blocks
3007 // dominating it are visited.
3008 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3009 [this](const DomTreeNode *A, const DomTreeNode *B) {
3010 return DT->properlyDominates(A, B);
3011 });
3012
3013 // Perform O(N^2) search over the gather sequences and merge identical
3014 // instructions. TODO: We can further optimize this scan if we split the
3015 // instructions into different buckets based on the insert lane.
3016 SmallVector<Instruction *, 16> Visited;
3017 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
1
Assuming 'I' is not equal to 'E'
2
Loop condition is true. Entering loop body
4
Loop condition is true. Entering loop body
6
Loop condition is true. Entering loop body
8
Loop condition is true. Entering loop body
3018 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3019, __PRETTY_FUNCTION__))
3019 "Worklist not sorted properly!")(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3019, __PRETTY_FUNCTION__))
;
3020 BasicBlock *BB = (*I)->getBlock();
9
Called C++ object pointer is null
3021 // For all instructions in blocks containing gather sequences:
3022 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3
Loop condition is false. Execution continues on line 3017
5
Loop condition is false. Execution continues on line 3017
7
Loop condition is false. Execution continues on line 3017
3023 Instruction *In = &*it++;
3024 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3025 continue;
3026
3027 // Check if we can replace this instruction with any of the
3028 // visited instructions.
3029 for (Instruction *v : Visited) {
3030 if (In->isIdenticalTo(v) &&
3031 DT->dominates(v->getParent(), In->getParent())) {
3032 In->replaceAllUsesWith(v);
3033 eraseInstruction(In);
3034 In = nullptr;
3035 break;
3036 }
3037 }
3038 if (In) {
3039 assert(!is_contained(Visited, In))((!is_contained(Visited, In)) ? static_cast<void> (0) :
__assert_fail ("!is_contained(Visited, In)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3039, __PRETTY_FUNCTION__))
;
3040 Visited.push_back(In);
3041 }
3042 }
3043 }
3044 CSEBlocks.clear();
3045 GatherSeq.clear();
3046}
3047
3048// Groups the instructions to a bundle (which is then a single scheduling entity)
3049// and schedules instructions until the bundle gets ready.
3050bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3051 BoUpSLP *SLP) {
3052 if (isa<PHINode>(VL[0]))
3053 return true;
3054
3055 // Initialize the instruction bundle.
3056 Instruction *OldScheduleEnd = ScheduleEnd;
3057 ScheduleData *PrevInBundle = nullptr;
3058 ScheduleData *Bundle = nullptr;
3059 bool ReSchedule = false;
3060 DEBUG(dbgs() << "SLP: bundle: " << *VL[0] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *VL[0] <<
"\n"; } } while (false)
;
3061
3062 // Make sure that the scheduling region contains all
3063 // instructions of the bundle.
3064 for (Value *V : VL) {
3065 if (!extendSchedulingRegion(V))
3066 return false;
3067 }
3068
3069 for (Value *V : VL) {
3070 ScheduleData *BundleMember = getScheduleData(V);
3071 assert(BundleMember &&((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3072, __PRETTY_FUNCTION__))
3072 "no ScheduleData for bundle member (maybe not in same basic block)")((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3072, __PRETTY_FUNCTION__))
;
3073 if (BundleMember->IsScheduled) {
3074 // A bundle member was scheduled as single instruction before and now
3075 // needs to be scheduled as part of the bundle. We just get rid of the
3076 // existing schedule.
3077 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3078 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
3079 ReSchedule = true;
3080 }
3081 assert(BundleMember->isSchedulingEntity() &&((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3082, __PRETTY_FUNCTION__))
3082 "bundle member already part of other bundle")((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3082, __PRETTY_FUNCTION__))
;
3083 if (PrevInBundle) {
3084 PrevInBundle->NextInBundle = BundleMember;
3085 } else {
3086 Bundle = BundleMember;
3087 }
3088 BundleMember->UnscheduledDepsInBundle = 0;
3089 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3090
3091 // Group the instructions to a bundle.
3092 BundleMember->FirstInBundle = Bundle;
3093 PrevInBundle = BundleMember;
3094 }
3095 if (ScheduleEnd != OldScheduleEnd) {
3096 // The scheduling region got new instructions at the lower end (or it is a
3097 // new region for the first bundle). This makes it necessary to
3098 // recalculate all dependencies.
3099 // It is seldom that this needs to be done a second time after adding the
3100 // initial bundle to the region.
3101 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3102 ScheduleData *SD = getScheduleData(I);
3103 SD->clearDependencies();
3104 }
3105 ReSchedule = true;
3106 }
3107 if (ReSchedule) {
3108 resetSchedule();
3109 initialFillReadyList(ReadyInsts);
3110 }
3111
3112 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
3113 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
3114
3115 calculateDependencies(Bundle, true, SLP);
3116
3117 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3118 // means that there are no cyclic dependencies and we can schedule it.
3119 // Note that's important that we don't "schedule" the bundle yet (see
3120 // cancelScheduling).
3121 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3122
3123 ScheduleData *pickedSD = ReadyInsts.back();
3124 ReadyInsts.pop_back();
3125
3126 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3127 schedule(pickedSD, ReadyInsts);
3128 }
3129 }
3130 if (!Bundle->isReady()) {
3131 cancelScheduling(VL);
3132 return false;
3133 }
3134 return true;
3135}
3136
3137void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL) {
3138 if (isa<PHINode>(VL[0]))
3139 return;
3140
3141 ScheduleData *Bundle = getScheduleData(VL[0]);
3142 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3143 assert(!Bundle->IsScheduled &&((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3144, __PRETTY_FUNCTION__))
3144 "Can't cancel bundle which is already scheduled")((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3144, __PRETTY_FUNCTION__))
;
3145 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3146, __PRETTY_FUNCTION__))
3146 "tried to unbundle something which is not a bundle")((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3146, __PRETTY_FUNCTION__))
;
3147
3148 // Un-bundle: make single instructions out of the bundle.
3149 ScheduleData *BundleMember = Bundle;
3150 while (BundleMember) {
3151 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")((BundleMember->FirstInBundle == Bundle && "corrupt bundle links"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3151, __PRETTY_FUNCTION__))
;
3152 BundleMember->FirstInBundle = BundleMember;
3153 ScheduleData *Next = BundleMember->NextInBundle;
3154 BundleMember->NextInBundle = nullptr;
3155 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3156 if (BundleMember->UnscheduledDepsInBundle == 0) {
3157 ReadyInsts.insert(BundleMember);
3158 }
3159 BundleMember = Next;
3160 }
3161}
3162
3163bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V) {
3164 if (getScheduleData(V))
3165 return true;
3166 Instruction *I = dyn_cast<Instruction>(V);
3167 assert(I && "bundle member must be an instruction")((I && "bundle member must be an instruction") ? static_cast
<void> (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3167, __PRETTY_FUNCTION__))
;
3168 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")((!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3168, __PRETTY_FUNCTION__))
;
3169 if (!ScheduleStart) {
3170 // It's the first instruction in the new region.
3171 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3172 ScheduleStart = I;
3173 ScheduleEnd = I->getNextNode();
3174 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3174, __PRETTY_FUNCTION__))
;
3175 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3176 return true;
3177 }
3178 // Search up and down at the same time, because we don't know if the new
3179 // instruction is above or below the existing scheduling region.
3180 BasicBlock::reverse_iterator UpIter =
3181 ++ScheduleStart->getIterator().getReverse();
3182 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3183 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3184 BasicBlock::iterator LowerEnd = BB->end();
3185 for (;;) {
3186 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3187 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3188 return false;
3189 }
3190
3191 if (UpIter != UpperEnd) {
3192 if (&*UpIter == I) {
3193 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3194 ScheduleStart = I;
3195 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3196 return true;
3197 }
3198 UpIter++;
3199 }
3200 if (DownIter != LowerEnd) {
3201 if (&*DownIter == I) {
3202 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3203 nullptr);
3204 ScheduleEnd = I->getNextNode();
3205 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3205, __PRETTY_FUNCTION__))
;
3206 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
3207 return true;
3208 }
3209 DownIter++;
3210 }
3211 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3212, __PRETTY_FUNCTION__))
3212 "instruction not found in block")(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3212, __PRETTY_FUNCTION__))
;
3213 }
3214 return true;
3215}
3216
3217void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3218 Instruction *ToI,
3219 ScheduleData *PrevLoadStore,
3220 ScheduleData *NextLoadStore) {
3221 ScheduleData *CurrentLoadStore = PrevLoadStore;
3222 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3223 ScheduleData *SD = ScheduleDataMap[I];
3224 if (!SD) {
3225 // Allocate a new ScheduleData for the instruction.
3226 if (ChunkPos >= ChunkSize) {
3227 ScheduleDataChunks.push_back(
3228 llvm::make_unique<ScheduleData[]>(ChunkSize));
3229 ChunkPos = 0;
3230 }
3231 SD = &(ScheduleDataChunks.back()[ChunkPos++]);
3232 ScheduleDataMap[I] = SD;
3233 SD->Inst = I;
3234 }
3235 assert(!isInSchedulingRegion(SD) &&((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3236, __PRETTY_FUNCTION__))
3236 "new ScheduleData already in scheduling region")((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3236, __PRETTY_FUNCTION__))
;
3237 SD->init(SchedulingRegionID);
3238
3239 if (I->mayReadOrWriteMemory()) {
3240 // Update the linked list of memory accessing instructions.
3241 if (CurrentLoadStore) {
3242 CurrentLoadStore->NextLoadStore = SD;
3243 } else {
3244 FirstLoadStoreInRegion = SD;
3245 }
3246 CurrentLoadStore = SD;
3247 }
3248 }
3249 if (NextLoadStore) {
3250 if (CurrentLoadStore)
3251 CurrentLoadStore->NextLoadStore = NextLoadStore;
3252 } else {
3253 LastLoadStoreInRegion = CurrentLoadStore;
3254 }
3255}
3256
3257void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3258 bool InsertInReadyList,
3259 BoUpSLP *SLP) {
3260 assert(SD->isSchedulingEntity())((SD->isSchedulingEntity()) ? static_cast<void> (0) :
__assert_fail ("SD->isSchedulingEntity()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3260, __PRETTY_FUNCTION__))
;
3261
3262 SmallVector<ScheduleData *, 10> WorkList;
3263 WorkList.push_back(SD);
3264
3265 while (!WorkList.empty()) {
3266 ScheduleData *SD = WorkList.back();
3267 WorkList.pop_back();
3268
3269 ScheduleData *BundleMember = SD;
3270 while (BundleMember) {
3271 assert(isInSchedulingRegion(BundleMember))((isInSchedulingRegion(BundleMember)) ? static_cast<void>
(0) : __assert_fail ("isInSchedulingRegion(BundleMember)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3271, __PRETTY_FUNCTION__))
;
3272 if (!BundleMember->hasValidDependencies()) {
3273
3274 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
3275 BundleMember->Dependencies = 0;
3276 BundleMember->resetUnscheduledDeps();
3277
3278 // Handle def-use chain dependencies.
3279 for (User *U : BundleMember->Inst->users()) {
3280 if (isa<Instruction>(U)) {
3281 ScheduleData *UseSD = getScheduleData(U);
3282 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3283 BundleMember->Dependencies++;
3284 ScheduleData *DestBundle = UseSD->FirstInBundle;
3285 if (!DestBundle->IsScheduled) {
3286 BundleMember->incrementUnscheduledDeps(1);
3287 }
3288 if (!DestBundle->hasValidDependencies()) {
3289 WorkList.push_back(DestBundle);
3290 }
3291 }
3292 } else {
3293 // I'm not sure if this can ever happen. But we need to be safe.
3294 // This lets the instruction/bundle never be scheduled and
3295 // eventually disable vectorization.
3296 BundleMember->Dependencies++;
3297 BundleMember->incrementUnscheduledDeps(1);
3298 }
3299 }
3300
3301 // Handle the memory dependencies.
3302 ScheduleData *DepDest = BundleMember->NextLoadStore;
3303 if (DepDest) {
3304 Instruction *SrcInst = BundleMember->Inst;
3305 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3306 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3307 unsigned numAliased = 0;
3308 unsigned DistToSrc = 1;
3309
3310 while (DepDest) {
3311 assert(isInSchedulingRegion(DepDest))((isInSchedulingRegion(DepDest)) ? static_cast<void> (0
) : __assert_fail ("isInSchedulingRegion(DepDest)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3311, __PRETTY_FUNCTION__))
;
3312
3313 // We have two limits to reduce the complexity:
3314 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3315 // SLP->isAliased (which is the expensive part in this loop).
3316 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3317 // the whole loop (even if the loop is fast, it's quadratic).
3318 // It's important for the loop break condition (see below) to
3319 // check this limit even between two read-only instructions.
3320 if (DistToSrc >= MaxMemDepDistance ||
3321 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3322 (numAliased >= AliasedCheckLimit ||
3323 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3324
3325 // We increment the counter only if the locations are aliased
3326 // (instead of counting all alias checks). This gives a better
3327 // balance between reduced runtime and accurate dependencies.
3328 numAliased++;
3329
3330 DepDest->MemoryDependencies.push_back(BundleMember);
3331 BundleMember->Dependencies++;
3332 ScheduleData *DestBundle = DepDest->FirstInBundle;
3333 if (!DestBundle->IsScheduled) {
3334 BundleMember->incrementUnscheduledDeps(1);
3335 }
3336 if (!DestBundle->hasValidDependencies()) {
3337 WorkList.push_back(DestBundle);
3338 }
3339 }
3340 DepDest = DepDest->NextLoadStore;
3341
3342 // Example, explaining the loop break condition: Let's assume our
3343 // starting instruction is i0 and MaxMemDepDistance = 3.
3344 //
3345 // +--------v--v--v
3346 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3347 // +--------^--^--^
3348 //
3349 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3350 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3351 // Previously we already added dependencies from i3 to i6,i7,i8
3352 // (because of MaxMemDepDistance). As we added a dependency from
3353 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3354 // and we can abort this loop at i6.
3355 if (DistToSrc >= 2 * MaxMemDepDistance)
3356 break;
3357 DistToSrc++;
3358 }
3359 }
3360 }
3361 BundleMember = BundleMember->NextInBundle;
3362 }
3363 if (InsertInReadyList && SD->isReady()) {
3364 ReadyInsts.push_back(SD);
3365 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
3366 }
3367 }
3368}
3369
3370void BoUpSLP::BlockScheduling::resetSchedule() {
3371 assert(ScheduleStart &&((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3372, __PRETTY_FUNCTION__))
3372 "tried to reset schedule on block which has not been scheduled")((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3372, __PRETTY_FUNCTION__))
;
3373 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3374 ScheduleData *SD = getScheduleData(I);
3375 assert(isInSchedulingRegion(SD))((isInSchedulingRegion(SD)) ? static_cast<void> (0) : __assert_fail
("isInSchedulingRegion(SD)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3375, __PRETTY_FUNCTION__))
;
3376 SD->IsScheduled = false;
3377 SD->resetUnscheduledDeps();
3378 }
3379 ReadyInsts.clear();
3380}
3381
3382void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3383
3384 if (!BS->ScheduleStart)
3385 return;
3386
3387 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
3388
3389 BS->resetSchedule();
3390
3391 // For the real scheduling we use a more sophisticated ready-list: it is
3392 // sorted by the original instruction location. This lets the final schedule
3393 // be as close as possible to the original instruction order.
3394 struct ScheduleDataCompare {
3395 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3396 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3397 }
3398 };
3399 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3400
3401 // Ensure that all dependency data is updated and fill the ready-list with
3402 // initial instructions.
3403 int Idx = 0;
3404 int NumToSchedule = 0;
3405 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3406 I = I->getNextNode()) {
3407 ScheduleData *SD = BS->getScheduleData(I);
3408 assert(((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3410, __PRETTY_FUNCTION__))
3409 SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) &&((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3410, __PRETTY_FUNCTION__))
3410 "scheduler and vectorizer have different opinion on what is a bundle")((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3410, __PRETTY_FUNCTION__))
;
3411 SD->FirstInBundle->SchedulingPriority = Idx++;
3412 if (SD->isSchedulingEntity()) {
3413 BS->calculateDependencies(SD, false, this);
3414 NumToSchedule++;
3415 }
3416 }
3417 BS->initialFillReadyList(ReadyInsts);
3418
3419 Instruction *LastScheduledInst = BS->ScheduleEnd;
3420
3421 // Do the "real" scheduling.
3422 while (!ReadyInsts.empty()) {
3423 ScheduleData *picked = *ReadyInsts.begin();
3424 ReadyInsts.erase(ReadyInsts.begin());
3425
3426 // Move the scheduled instruction(s) to their dedicated places, if not
3427 // there yet.
3428 ScheduleData *BundleMember = picked;
3429 while (BundleMember) {
3430 Instruction *pickedInst = BundleMember->Inst;
3431 if (LastScheduledInst->getNextNode() != pickedInst) {
3432 BS->BB->getInstList().remove(pickedInst);
3433 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3434 pickedInst);
3435 }
3436 LastScheduledInst = pickedInst;
3437 BundleMember = BundleMember->NextInBundle;
3438 }
3439
3440 BS->schedule(picked, ReadyInsts);
3441 NumToSchedule--;
3442 }
3443 assert(NumToSchedule == 0 && "could not schedule all instructions")((NumToSchedule == 0 && "could not schedule all instructions"
) ? static_cast<void> (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3443, __PRETTY_FUNCTION__))
;
3444
3445 // Avoid duplicate scheduling of the block.
3446 BS->ScheduleStart = nullptr;
3447}
3448
3449unsigned BoUpSLP::getVectorElementSize(Value *V) {
3450 // If V is a store, just return the width of the stored value without
3451 // traversing the expression tree. This is the common case.
3452 if (auto *Store = dyn_cast<StoreInst>(V))
3453 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3454
3455 // If V is not a store, we can traverse the expression tree to find loads
3456 // that feed it. The type of the loaded value may indicate a more suitable
3457 // width than V's type. We want to base the vector element size on the width
3458 // of memory operations where possible.
3459 SmallVector<Instruction *, 16> Worklist;
3460 SmallPtrSet<Instruction *, 16> Visited;
3461 if (auto *I = dyn_cast<Instruction>(V))
3462 Worklist.push_back(I);
3463
3464 // Traverse the expression tree in bottom-up order looking for loads. If we
3465 // encounter an instruciton we don't yet handle, we give up.
3466 auto MaxWidth = 0u;
3467 auto FoundUnknownInst = false;
3468 while (!Worklist.empty() && !FoundUnknownInst) {
3469 auto *I = Worklist.pop_back_val();
3470 Visited.insert(I);
3471
3472 // We should only be looking at scalar instructions here. If the current
3473 // instruction has a vector type, give up.
3474 auto *Ty = I->getType();
3475 if (isa<VectorType>(Ty))
3476 FoundUnknownInst = true;
3477
3478 // If the current instruction is a load, update MaxWidth to reflect the
3479 // width of the loaded value.
3480 else if (isa<LoadInst>(I))
3481 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3482
3483 // Otherwise, we need to visit the operands of the instruction. We only
3484 // handle the interesting cases from buildTree here. If an operand is an
3485 // instruction we haven't yet visited, we add it to the worklist.
3486 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3487 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3488 for (Use &U : I->operands())
3489 if (auto *J = dyn_cast<Instruction>(U.get()))
3490 if (!Visited.count(J))
3491 Worklist.push_back(J);
3492 }
3493
3494 // If we don't yet handle the instruction, give up.
3495 else
3496 FoundUnknownInst = true;
3497 }
3498
3499 // If we didn't encounter a memory access in the expression tree, or if we
3500 // gave up for some reason, just return the width of V.
3501 if (!MaxWidth || FoundUnknownInst)
3502 return DL->getTypeSizeInBits(V->getType());
3503
3504 // Otherwise, return the maximum width we found.
3505 return MaxWidth;
3506}
3507
3508// Determine if a value V in a vectorizable expression Expr can be demoted to a
3509// smaller type with a truncation. We collect the values that will be demoted
3510// in ToDemote and additional roots that require investigating in Roots.
3511static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
3512 SmallVectorImpl<Value *> &ToDemote,
3513 SmallVectorImpl<Value *> &Roots) {
3514
3515 // We can always demote constants.
3516 if (isa<Constant>(V)) {
3517 ToDemote.push_back(V);
3518 return true;
3519 }
3520
3521 // If the value is not an instruction in the expression with only one use, it
3522 // cannot be demoted.
3523 auto *I = dyn_cast<Instruction>(V);
3524 if (!I || !I->hasOneUse() || !Expr.count(I))
3525 return false;
3526
3527 switch (I->getOpcode()) {
3528
3529 // We can always demote truncations and extensions. Since truncations can
3530 // seed additional demotion, we save the truncated value.
3531 case Instruction::Trunc:
3532 Roots.push_back(I->getOperand(0));
3533 case Instruction::ZExt:
3534 case Instruction::SExt:
3535 break;
3536
3537 // We can demote certain binary operations if we can demote both of their
3538 // operands.
3539 case Instruction::Add:
3540 case Instruction::Sub:
3541 case Instruction::Mul:
3542 case Instruction::And:
3543 case Instruction::Or:
3544 case Instruction::Xor:
3545 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3546 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3547 return false;
3548 break;
3549
3550 // We can demote selects if we can demote their true and false values.
3551 case Instruction::Select: {
3552 SelectInst *SI = cast<SelectInst>(I);
3553 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3554 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3555 return false;
3556 break;
3557 }
3558
3559 // We can demote phis if we can demote all their incoming operands. Note that
3560 // we don't need to worry about cycles since we ensure single use above.
3561 case Instruction::PHI: {
3562 PHINode *PN = cast<PHINode>(I);
3563 for (Value *IncValue : PN->incoming_values())
3564 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3565 return false;
3566 break;
3567 }
3568
3569 // Otherwise, conservatively give up.
3570 default:
3571 return false;
3572 }
3573
3574 // Record the value that we can demote.
3575 ToDemote.push_back(V);
3576 return true;
3577}
3578
3579void BoUpSLP::computeMinimumValueSizes() {
3580 // If there are no external uses, the expression tree must be rooted by a
3581 // store. We can't demote in-memory values, so there is nothing to do here.
3582 if (ExternalUses.empty())
3583 return;
3584
3585 // We only attempt to truncate integer expressions.
3586 auto &TreeRoot = VectorizableTree[0].Scalars;
3587 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3588 if (!TreeRootIT)
3589 return;
3590
3591 // If the expression is not rooted by a store, these roots should have
3592 // external uses. We will rely on InstCombine to rewrite the expression in
3593 // the narrower type. However, InstCombine only rewrites single-use values.
3594 // This means that if a tree entry other than a root is used externally, it
3595 // must have multiple uses and InstCombine will not rewrite it. The code
3596 // below ensures that only the roots are used externally.
3597 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3598 for (auto &EU : ExternalUses)
3599 if (!Expr.erase(EU.Scalar))
3600 return;
3601 if (!Expr.empty())
3602 return;
3603
3604 // Collect the scalar values of the vectorizable expression. We will use this
3605 // context to determine which values can be demoted. If we see a truncation,
3606 // we mark it as seeding another demotion.
3607 for (auto &Entry : VectorizableTree)
3608 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3609
3610 // Ensure the roots of the vectorizable tree don't form a cycle. They must
3611 // have a single external user that is not in the vectorizable tree.
3612 for (auto *Root : TreeRoot)
3613 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
3614 return;
3615
3616 // Conservatively determine if we can actually truncate the roots of the
3617 // expression. Collect the values that can be demoted in ToDemote and
3618 // additional roots that require investigating in Roots.
3619 SmallVector<Value *, 32> ToDemote;
3620 SmallVector<Value *, 4> Roots;
3621 for (auto *Root : TreeRoot)
3622 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
3623 return;
3624
3625 // The maximum bit width required to represent all the values that can be
3626 // demoted without loss of precision. It would be safe to truncate the roots
3627 // of the expression to this width.
3628 auto MaxBitWidth = 8u;
3629
3630 // We first check if all the bits of the roots are demanded. If they're not,
3631 // we can truncate the roots to this narrower type.
3632 for (auto *Root : TreeRoot) {
3633 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
3634 MaxBitWidth = std::max<unsigned>(
3635 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
3636 }
3637
3638 // True if the roots can be zero-extended back to their original type, rather
3639 // than sign-extended. We know that if the leading bits are not demanded, we
3640 // can safely zero-extend. So we initialize IsKnownPositive to True.
3641 bool IsKnownPositive = true;
3642
3643 // If all the bits of the roots are demanded, we can try a little harder to
3644 // compute a narrower type. This can happen, for example, if the roots are
3645 // getelementptr indices. InstCombine promotes these indices to the pointer
3646 // width. Thus, all their bits are technically demanded even though the
3647 // address computation might be vectorized in a smaller type.
3648 //
3649 // We start by looking at each entry that can be demoted. We compute the
3650 // maximum bit width required to store the scalar by using ValueTracking to
3651 // compute the number of high-order bits we can truncate.
3652 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
3653 MaxBitWidth = 8u;
3654
3655 // Determine if the sign bit of all the roots is known to be zero. If not,
3656 // IsKnownPositive is set to False.
3657 IsKnownPositive = all_of(TreeRoot, [&](Value *R) {
3658 bool KnownZero = false;
3659 bool KnownOne = false;
3660 ComputeSignBit(R, KnownZero, KnownOne, *DL);
3661 return KnownZero;
3662 });
3663
3664 // Determine the maximum number of bits required to store the scalar
3665 // values.
3666 for (auto *Scalar : ToDemote) {
3667 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, 0, DT);
3668 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
3669 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
3670 }
3671
3672 // If we can't prove that the sign bit is zero, we must add one to the
3673 // maximum bit width to account for the unknown sign bit. This preserves
3674 // the existing sign bit so we can safely sign-extend the root back to the
3675 // original type. Otherwise, if we know the sign bit is zero, we will
3676 // zero-extend the root instead.
3677 //
3678 // FIXME: This is somewhat suboptimal, as there will be cases where adding
3679 // one to the maximum bit width will yield a larger-than-necessary
3680 // type. In general, we need to add an extra bit only if we can't
3681 // prove that the upper bit of the original type is equal to the
3682 // upper bit of the proposed smaller type. If these two bits are the
3683 // same (either zero or one) we know that sign-extending from the
3684 // smaller type will result in the same value. Here, since we can't
3685 // yet prove this, we are just making the proposed smaller type
3686 // larger to ensure correctness.
3687 if (!IsKnownPositive)
3688 ++MaxBitWidth;
3689 }
3690
3691 // Round MaxBitWidth up to the next power-of-two.
3692 if (!isPowerOf2_64(MaxBitWidth))
3693 MaxBitWidth = NextPowerOf2(MaxBitWidth);
3694
3695 // If the maximum bit width we compute is less than the with of the roots'
3696 // type, we can proceed with the narrowing. Otherwise, do nothing.
3697 if (MaxBitWidth >= TreeRootIT->getBitWidth())
3698 return;
3699
3700 // If we can truncate the root, we must collect additional values that might
3701 // be demoted as a result. That is, those seeded by truncations we will
3702 // modify.
3703 while (!Roots.empty())
3704 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
3705
3706 // Finally, map the values we can demote to the maximum bit with we computed.
3707 for (auto *Scalar : ToDemote)
3708 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
3709}
3710
3711namespace {
3712/// The SLPVectorizer Pass.
3713struct SLPVectorizer : public FunctionPass {
3714 SLPVectorizerPass Impl;
3715
3716 /// Pass identification, replacement for typeid
3717 static char ID;
3718
3719 explicit SLPVectorizer() : FunctionPass(ID) {
3720 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
3721 }
3722
3723
3724 bool doInitialization(Module &M) override {
3725 return false;
3726 }
3727
3728 bool runOnFunction(Function &F) override {
3729 if (skipFunction(F))
3730 return false;
3731
3732 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
3733 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
3734 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
3735 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
3736 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3737 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
3738 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
3739 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
3740 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
3741
3742 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB);
3743 }
3744
3745 void getAnalysisUsage(AnalysisUsage &AU) const override {
3746 FunctionPass::getAnalysisUsage(AU);
3747 AU.addRequired<AssumptionCacheTracker>();
3748 AU.addRequired<ScalarEvolutionWrapperPass>();
3749 AU.addRequired<AAResultsWrapperPass>();
3750 AU.addRequired<TargetTransformInfoWrapperPass>();
3751 AU.addRequired<LoopInfoWrapperPass>();
3752 AU.addRequired<DominatorTreeWrapperPass>();
3753 AU.addRequired<DemandedBitsWrapperPass>();
3754 AU.addPreserved<LoopInfoWrapperPass>();
3755 AU.addPreserved<DominatorTreeWrapperPass>();
3756 AU.addPreserved<AAResultsWrapperPass>();
3757 AU.addPreserved<GlobalsAAWrapperPass>();
3758 AU.setPreservesCFG();
3759 }
3760};
3761} // end anonymous namespace
3762
3763PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
3764 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
3765 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
3766 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
3767 auto *AA = &AM.getResult<AAManager>(F);
3768 auto *LI = &AM.getResult<LoopAnalysis>(F);
3769 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
3770 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
3771 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
3772
3773 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB);
3774 if (!Changed)
3775 return PreservedAnalyses::all();
3776
3777 PreservedAnalyses PA;
3778 PA.preserveSet<CFGAnalyses>();
3779 PA.preserve<AAManager>();
3780 PA.preserve<GlobalsAA>();
3781 return PA;
3782}
3783
3784bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
3785 TargetTransformInfo *TTI_,
3786 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
3787 LoopInfo *LI_, DominatorTree *DT_,
3788 AssumptionCache *AC_, DemandedBits *DB_) {
3789 SE = SE_;
3790 TTI = TTI_;
3791 TLI = TLI_;
3792 AA = AA_;
3793 LI = LI_;
3794 DT = DT_;
3795 AC = AC_;
3796 DB = DB_;
3797 DL = &F.getParent()->getDataLayout();
3798
3799 Stores.clear();
3800 GEPs.clear();
3801 bool Changed = false;
3802
3803 // If the target claims to have no vector registers don't attempt
3804 // vectorization.
3805 if (!TTI->getNumberOfRegisters(true))
3806 return false;
3807
3808 // Don't vectorize when the attribute NoImplicitFloat is used.
3809 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
3810 return false;
3811
3812 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
3813
3814 // Use the bottom up slp vectorizer to construct chains that start with
3815 // store instructions.
3816 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL);
3817
3818 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
3819 // delete instructions.
3820
3821 // Scan the blocks in the function in post order.
3822 for (auto BB : post_order(&F.getEntryBlock())) {
3823 collectSeedInstructions(BB);
3824
3825 // Vectorize trees that end at stores.
3826 if (!Stores.empty()) {
3827 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
3828 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
3829 Changed |= vectorizeStoreChains(R);
3830 }
3831
3832 // Vectorize trees that end at reductions.
3833 Changed |= vectorizeChainsInBlock(BB, R);
3834
3835 // Vectorize the index computations of getelementptr instructions. This
3836 // is primarily intended to catch gather-like idioms ending at
3837 // non-consecutive loads.
3838 if (!GEPs.empty()) {
3839 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
3840 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
3841 Changed |= vectorizeGEPIndices(BB, R);
3842 }
3843 }
3844
3845 if (Changed) {
3846 R.optimizeGatherSequence();
3847 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
3848 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
3849 }
3850 return Changed;
3851}
3852
3853/// \brief Check that the Values in the slice in VL array are still existent in
3854/// the WeakVH array.
3855/// Vectorization of part of the VL array may cause later values in the VL array
3856/// to become invalid. We track when this has happened in the WeakVH array.
3857static bool hasValueBeenRAUWed(ArrayRef<Value *> VL, ArrayRef<WeakVH> VH,
3858 unsigned SliceBegin, unsigned SliceSize) {
3859 VL = VL.slice(SliceBegin, SliceSize);
3860 VH = VH.slice(SliceBegin, SliceSize);
3861 return !std::equal(VL.begin(), VL.end(), VH.begin());
3862}
3863
3864bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
3865 unsigned VecRegSize) {
3866 unsigned ChainLen = Chain.size();
3867 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
3868 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
3869 unsigned Sz = R.getVectorElementSize(Chain[0]);
3870 unsigned VF = VecRegSize / Sz;
3871
3872 if (!isPowerOf2_32(Sz) || VF < 2)
3873 return false;
3874
3875 // Keep track of values that were deleted by vectorizing in the loop below.
3876 SmallVector<WeakVH, 8> TrackValues(Chain.begin(), Chain.end());
3877
3878 bool Changed = false;
3879 // Look for profitable vectorizable trees at all offsets, starting at zero.
3880 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
3881 if (i + VF > e)
3882 break;
3883
3884 // Check that a previous iteration of this loop did not delete the Value.
3885 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
3886 continue;
3887
3888 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
3889 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
3890 ArrayRef<Value *> Operands = Chain.slice(i, VF);
3891
3892 R.buildTree(Operands);
3893 if (R.isTreeTinyAndNotFullyVectorizable())
3894 continue;
3895
3896 R.computeMinimumValueSizes();
3897
3898 int Cost = R.getTreeCost();
3899
3900 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
3901 if (Cost < -SLPCostThreshold) {
3902 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
3903 R.vectorizeTree();
3904
3905 // Move to the next bundle.
3906 i += VF - 1;
3907 Changed = true;
3908 }
3909 }
3910
3911 return Changed;
3912}
3913
3914bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
3915 BoUpSLP &R) {
3916 SetVector<StoreInst *> Heads, Tails;
3917 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
3918
3919 // We may run into multiple chains that merge into a single chain. We mark the
3920 // stores that we vectorized so that we don't visit the same store twice.
3921 BoUpSLP::ValueSet VectorizedStores;
3922 bool Changed = false;
3923
3924 // Do a quadratic search on all of the given stores and find
3925 // all of the pairs of stores that follow each other.
3926 SmallVector<unsigned, 16> IndexQueue;
3927 for (unsigned i = 0, e = Stores.size(); i < e; ++i) {
3928 IndexQueue.clear();
3929 // If a store has multiple consecutive store candidates, search Stores
3930 // array according to the sequence: from i+1 to e, then from i-1 to 0.
3931 // This is because usually pairing with immediate succeeding or preceding
3932 // candidate create the best chance to find slp vectorization opportunity.
3933 unsigned j = 0;
3934 for (j = i + 1; j < e; ++j)
3935 IndexQueue.push_back(j);
3936 for (j = i; j > 0; --j)
3937 IndexQueue.push_back(j - 1);
3938
3939 for (auto &k : IndexQueue) {
3940 if (isConsecutiveAccess(Stores[i], Stores[k], *DL, *SE)) {
3941 Tails.insert(Stores[k]);
3942 Heads.insert(Stores[i]);
3943 ConsecutiveChain[Stores[i]] = Stores[k];
3944 break;
3945 }
3946 }
3947 }
3948
3949 // For stores that start but don't end a link in the chain:
3950 for (SetVector<StoreInst *>::iterator it = Heads.begin(), e = Heads.end();
3951 it != e; ++it) {
3952 if (Tails.count(*it))
3953 continue;
3954
3955 // We found a store instr that starts a chain. Now follow the chain and try
3956 // to vectorize it.
3957 BoUpSLP::ValueList Operands;
3958 StoreInst *I = *it;
3959 // Collect the chain into a list.
3960 while (Tails.count(I) || Heads.count(I)) {
3961 if (VectorizedStores.count(I))
3962 break;
3963 Operands.push_back(I);
3964 // Move to the next value in the chain.
3965 I = ConsecutiveChain[I];
3966 }
3967
3968 // FIXME: Is division-by-2 the correct step? Should we assert that the
3969 // register size is a power-of-2?
3970 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
3971 Size /= 2) {
3972 if (vectorizeStoreChain(Operands, R, Size)) {
3973 // Mark the vectorized stores so that we don't vectorize them again.
3974 VectorizedStores.insert(Operands.begin(), Operands.end());
3975 Changed = true;
3976 break;
3977 }
3978 }
3979 }
3980
3981 return Changed;
3982}
3983
3984void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
3985
3986 // Initialize the collections. We will make a single pass over the block.
3987 Stores.clear();
3988 GEPs.clear();
3989
3990 // Visit the store and getelementptr instructions in BB and organize them in
3991 // Stores and GEPs according to the underlying objects of their pointer
3992 // operands.
3993 for (Instruction &I : *BB) {
3994
3995 // Ignore store instructions that are volatile or have a pointer operand
3996 // that doesn't point to a scalar type.
3997 if (auto *SI = dyn_cast<StoreInst>(&I)) {
3998 if (!SI->isSimple())
3999 continue;
4000 if (!isValidElementType(SI->getValueOperand()->getType()))
4001 continue;
4002 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4003 }
4004
4005 // Ignore getelementptr instructions that have more than one index, a
4006 // constant index, or a pointer operand that doesn't point to a scalar
4007 // type.
4008 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4009 auto Idx = GEP->idx_begin()->get();
4010 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4011 continue;
4012 if (!isValidElementType(Idx->getType()))
4013 continue;
4014 if (GEP->getType()->isVectorTy())
4015 continue;
4016 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4017 }
4018 }
4019}
4020
4021bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4022 if (!A || !B)
4023 return false;
4024 Value *VL[] = { A, B };
4025 return tryToVectorizeList(VL, R, None, true);
4026}
4027
4028bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4029 ArrayRef<Value *> BuildVector,
4030 bool AllowReorder) {
4031 if (VL.size() < 2)
4032 return false;
4033
4034 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4035 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
4036
4037 // Check that all of the parts are scalar instructions of the same type.
4038 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4039 if (!I0)
4040 return false;
4041
4042 unsigned Opcode0 = I0->getOpcode();
4043
4044 unsigned Sz = R.getVectorElementSize(I0);
4045 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4046 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4047 if (MaxVF < 2)
4048 return false;
4049
4050 for (Value *V : VL) {
4051 Type *Ty = V->getType();
4052 if (!isValidElementType(Ty))
4053 return false;
4054 Instruction *Inst = dyn_cast<Instruction>(V);
4055 if (!Inst || Inst->getOpcode() != Opcode0)
4056 return false;
4057 }
4058
4059 bool Changed = false;
4060
4061 // Keep track of values that were deleted by vectorizing in the loop below.
4062 SmallVector<WeakVH, 8> TrackValues(VL.begin(), VL.end());
4063
4064 unsigned NextInst = 0, MaxInst = VL.size();
4065 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4066 VF /= 2) {
4067 // No actual vectorization should happen, if number of parts is the same as
4068 // provided vectorization factor (i.e. the scalar type is used for vector
4069 // code during codegen).
4070 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4071 if (TTI->getNumberOfParts(VecTy) == VF)
4072 continue;
4073 for (unsigned I = NextInst; I < MaxInst; ++I) {
4074 unsigned OpsWidth = 0;
4075
4076 if (I + VF > MaxInst)
4077 OpsWidth = MaxInst - I;
4078 else
4079 OpsWidth = VF;
4080
4081 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4082 break;
4083
4084 // Check that a previous iteration of this loop did not delete the Value.
4085 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4086 continue;
4087
4088 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4089 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
4090 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4091
4092 ArrayRef<Value *> BuildVectorSlice;
4093 if (!BuildVector.empty())
4094 BuildVectorSlice = BuildVector.slice(I, OpsWidth);
4095
4096 R.buildTree(Ops, BuildVectorSlice);
4097 // TODO: check if we can allow reordering for more cases.
4098 if (AllowReorder && R.shouldReorder()) {
4099 // Conceptually, there is nothing actually preventing us from trying to
4100 // reorder a larger list. In fact, we do exactly this when vectorizing
4101 // reductions. However, at this point, we only expect to get here from
4102 // tryToVectorizePair().
4103 assert(Ops.size() == 2)((Ops.size() == 2) ? static_cast<void> (0) : __assert_fail
("Ops.size() == 2", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4103, __PRETTY_FUNCTION__))
;
4104 assert(BuildVectorSlice.empty())((BuildVectorSlice.empty()) ? static_cast<void> (0) : __assert_fail
("BuildVectorSlice.empty()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4104, __PRETTY_FUNCTION__))
;
4105 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4106 R.buildTree(ReorderedOps, None);
4107 }
4108 if (R.isTreeTinyAndNotFullyVectorizable())
4109 continue;
4110
4111 R.computeMinimumValueSizes();
4112 int Cost = R.getTreeCost();
4113
4114 if (Cost < -SLPCostThreshold) {
4115 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
4116 Value *VectorizedRoot = R.vectorizeTree();
4117
4118 // Reconstruct the build vector by extracting the vectorized root. This
4119 // way we handle the case where some elements of the vector are
4120 // undefined.
4121 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
4122 if (!BuildVectorSlice.empty()) {
4123 // The insert point is the last build vector instruction. The
4124 // vectorized root will precede it. This guarantees that we get an
4125 // instruction. The vectorized tree could have been constant folded.
4126 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4127 unsigned VecIdx = 0;
4128 for (auto &V : BuildVectorSlice) {
4129 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4130 ++BasicBlock::iterator(InsertAfter));
4131 Instruction *I = cast<Instruction>(V);
4132 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I))((isa<InsertElementInst>(I) || isa<InsertValueInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<InsertElementInst>(I) || isa<InsertValueInst>(I)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4132, __PRETTY_FUNCTION__))
;
4133 Instruction *Extract =
4134 cast<Instruction>(Builder.CreateExtractElement(
4135 VectorizedRoot, Builder.getInt32(VecIdx++)));
4136 I->setOperand(1, Extract);
4137 I->removeFromParent();
4138 I->insertAfter(Extract);
4139 InsertAfter = I;
4140 }
4141 }
4142 // Move to the next bundle.
4143 I += VF - 1;
4144 NextInst = I + 1;
4145 Changed = true;
4146 }
4147 }
4148 }
4149
4150 return Changed;
4151}
4152
4153bool SLPVectorizerPass::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
4154 if (!V)
4155 return false;
4156
4157 Value *P = V->getParent();
4158
4159 // Vectorize in current basic block only.
4160 auto *Op0 = dyn_cast<Instruction>(V->getOperand(0));
4161 auto *Op1 = dyn_cast<Instruction>(V->getOperand(1));
4162 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4163 return false;
4164
4165 // Try to vectorize V.
4166 if (tryToVectorizePair(Op0, Op1, R))
4167 return true;
4168
4169 auto *A = dyn_cast<BinaryOperator>(Op0);
4170 auto *B = dyn_cast<BinaryOperator>(Op1);
4171 // Try to skip B.
4172 if (B && B->hasOneUse()) {
4173 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4174 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4175 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4176 return true;
4177 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4178 return true;
4179 }
4180
4181 // Try to skip A.
4182 if (A && A->hasOneUse()) {
4183 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4184 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4185 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4186 return true;
4187 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4188 return true;
4189 }
4190 return false;
4191}
4192
4193/// \brief Generate a shuffle mask to be used in a reduction tree.
4194///
4195/// \param VecLen The length of the vector to be reduced.
4196/// \param NumEltsToRdx The number of elements that should be reduced in the
4197/// vector.
4198/// \param IsPairwise Whether the reduction is a pairwise or splitting
4199/// reduction. A pairwise reduction will generate a mask of
4200/// <0,2,...> or <1,3,..> while a splitting reduction will generate
4201/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4202/// \param IsLeft True will generate a mask of even elements, odd otherwise.
4203static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4204 bool IsPairwise, bool IsLeft,
4205 IRBuilder<> &Builder) {
4206 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"
) ? static_cast<void> (0) : __assert_fail ("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4206, __PRETTY_FUNCTION__))
;
4207
4208 SmallVector<Constant *, 32> ShuffleMask(
4209 VecLen, UndefValue::get(Builder.getInt32Ty()));
4210
4211 if (IsPairwise)
4212 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4213 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4214 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4215 else
4216 // Move the upper half of the vector to the lower half.
4217 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4218 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4219
4220 return ConstantVector::get(ShuffleMask);
4221}
4222
4223namespace {
4224/// Model horizontal reductions.
4225///
4226/// A horizontal reduction is a tree of reduction operations (currently add and
4227/// fadd) that has operations that can be put into a vector as its leaf.
4228/// For example, this tree:
4229///
4230/// mul mul mul mul
4231/// \ / \ /
4232/// + +
4233/// \ /
4234/// +
4235/// This tree has "mul" as its reduced values and "+" as its reduction
4236/// operations. A reduction might be feeding into a store or a binary operation
4237/// feeding a phi.
4238/// ...
4239/// \ /
4240/// +
4241/// |
4242/// phi +=
4243///
4244/// Or:
4245/// ...
4246/// \ /
4247/// +
4248/// |
4249/// *p =
4250///
4251class HorizontalReduction {
4252 SmallVector<Value *, 16> ReductionOps;
4253 SmallVector<Value *, 32> ReducedVals;
4254 // Use map vector to make stable output.
4255 MapVector<Instruction *, Value *> ExtraArgs;
4256
4257 BinaryOperator *ReductionRoot = nullptr;
4258 // After successfull horizontal reduction vectorization attempt for PHI node
4259 // vectorizer tries to update root binary op by combining vectorized tree and
4260 // the ReductionPHI node. But during vectorization this ReductionPHI can be
4261 // vectorized itself and replaced by the undef value, while the instruction
4262 // itself is marked for deletion. This 'marked for deletion' PHI node then can
4263 // be used in new binary operation, causing "Use still stuck around after Def
4264 // is destroyed" crash upon PHI node deletion.
4265 WeakVH ReductionPHI;
4266
4267 /// The opcode of the reduction.
4268 Instruction::BinaryOps ReductionOpcode = Instruction::BinaryOpsEnd;
4269 /// The opcode of the values we perform a reduction on.
4270 unsigned ReducedValueOpcode = 0;
4271 /// Should we model this reduction as a pairwise reduction tree or a tree that
4272 /// splits the vector in halves and adds those halves.
4273 bool IsPairwiseReduction = false;
4274
4275 /// Checks if the ParentStackElem.first should be marked as a reduction
4276 /// operation with an extra argument or as extra argument itself.
4277 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
4278 Value *ExtraArg) {
4279 if (ExtraArgs.count(ParentStackElem.first)) {
4280 ExtraArgs[ParentStackElem.first] = nullptr;
4281 // We ran into something like:
4282 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
4283 // The whole ParentStackElem.first should be considered as an extra value
4284 // in this case.
4285 // Do not perform analysis of remaining operands of ParentStackElem.first
4286 // instruction, this whole instruction is an extra argument.
4287 ParentStackElem.second = ParentStackElem.first->getNumOperands();
4288 } else {
4289 // We ran into something like:
4290 // ParentStackElem.first += ... + ExtraArg + ...
4291 ExtraArgs[ParentStackElem.first] = ExtraArg;
4292 }
4293 }
4294
4295public:
4296 HorizontalReduction() = default;
4297
4298 /// \brief Try to find a reduction tree.
4299 bool matchAssociativeReduction(PHINode *Phi, BinaryOperator *B) {
4300 assert((!Phi || is_contained(Phi->operands(), B)) &&(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4301, __PRETTY_FUNCTION__))
4301 "Thi phi needs to use the binary operator")(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4301, __PRETTY_FUNCTION__))
;
4302
4303 // We could have a initial reductions that is not an add.
4304 // r *= v1 + v2 + v3 + v4
4305 // In such a case start looking for a tree rooted in the first '+'.
4306 if (Phi) {
4307 if (B->getOperand(0) == Phi) {
4308 Phi = nullptr;
4309 B = dyn_cast<BinaryOperator>(B->getOperand(1));
4310 } else if (B->getOperand(1) == Phi) {
4311 Phi = nullptr;
4312 B = dyn_cast<BinaryOperator>(B->getOperand(0));
4313 }
4314 }
4315
4316 if (!B)
4317 return false;
4318
4319 Type *Ty = B->getType();
4320 if (!isValidElementType(Ty))
4321 return false;
4322
4323 ReductionOpcode = B->getOpcode();
4324 ReducedValueOpcode = 0;
4325 ReductionRoot = B;
4326 ReductionPHI = Phi;
4327
4328 // We currently only support adds.
4329 if ((ReductionOpcode != Instruction::Add &&
4330 ReductionOpcode != Instruction::FAdd) ||
4331 !B->isAssociative())
4332 return false;
4333
4334 // Post order traverse the reduction tree starting at B. We only handle true
4335 // trees containing only binary operators or selects.
4336 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
4337 Stack.push_back(std::make_pair(B, 0));
4338 while (!Stack.empty()) {
4339 Instruction *TreeN = Stack.back().first;
4340 unsigned EdgeToVist = Stack.back().second++;
4341 bool IsReducedValue = TreeN->getOpcode() != ReductionOpcode;
4342
4343 // Postorder vist.
4344 if (EdgeToVist == 2 || IsReducedValue) {
4345 if (IsReducedValue)
4346 ReducedVals.push_back(TreeN);
4347 else {
4348 auto I = ExtraArgs.find(TreeN);
4349 if (I != ExtraArgs.end() && !I->second) {
4350 // Check if TreeN is an extra argument of its parent operation.
4351 if (Stack.size() <= 1) {
4352 // TreeN can't be an extra argument as it is a root reduction
4353 // operation.
4354 return false;
4355 }
4356 // Yes, TreeN is an extra argument, do not add it to a list of
4357 // reduction operations.
4358 // Stack[Stack.size() - 2] always points to the parent operation.
4359 markExtraArg(Stack[Stack.size() - 2], TreeN);
4360 ExtraArgs.erase(TreeN);
4361 } else
4362 ReductionOps.push_back(TreeN);
4363 }
4364 // Retract.
4365 Stack.pop_back();
4366 continue;
4367 }
4368
4369 // Visit left or right.
4370 Value *NextV = TreeN->getOperand(EdgeToVist);
4371 if (NextV != Phi) {
4372 auto *I = dyn_cast<Instruction>(NextV);
4373 // Continue analysis if the next operand is a reduction operation or
4374 // (possibly) a reduced value. If the reduced value opcode is not set,
4375 // the first met operation != reduction operation is considered as the
4376 // reduced value class.
4377 if (I && (!ReducedValueOpcode || I->getOpcode() == ReducedValueOpcode ||
4378 I->getOpcode() == ReductionOpcode)) {
4379 // Only handle trees in the current basic block.
4380 if (I->getParent() != B->getParent()) {
4381 // I is an extra argument for TreeN (its parent operation).
4382 markExtraArg(Stack.back(), I);
4383 continue;
4384 }
4385
4386 // Each tree node needs to have one user except for the ultimate
4387 // reduction.
4388 if (!I->hasOneUse() && I != B) {
4389 // I is an extra argument for TreeN (its parent operation).
4390 markExtraArg(Stack.back(), I);
4391 continue;
4392 }
4393
4394 if (I->getOpcode() == ReductionOpcode) {
4395 // We need to be able to reassociate the reduction operations.
4396 if (!I->isAssociative()) {
4397 // I is an extra argument for TreeN (its parent operation).
4398 markExtraArg(Stack.back(), I);
4399 continue;
4400 }
4401 } else if (ReducedValueOpcode &&
4402 ReducedValueOpcode != I->getOpcode()) {
4403 // Make sure that the opcodes of the operations that we are going to
4404 // reduce match.
4405 // I is an extra argument for TreeN (its parent operation).
4406 markExtraArg(Stack.back(), I);
4407 continue;
4408 } else if (!ReducedValueOpcode)
4409 ReducedValueOpcode = I->getOpcode();
4410
4411 Stack.push_back(std::make_pair(I, 0));
4412 continue;
4413 }
4414 // NextV is an extra argument for TreeN (its parent operation).
4415 markExtraArg(Stack.back(), NextV);
4416 }
4417 }
4418 return true;
4419 }
4420
4421 /// \brief Attempt to vectorize the tree found by
4422 /// matchAssociativeReduction.
4423 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
4424 if (ReducedVals.empty())
4425 return false;
4426
4427 // If there is a sufficient number of reduction values, reduce
4428 // to a nearby power-of-2. Can safely generate oversized
4429 // vectors and rely on the backend to split them to legal sizes.
4430 unsigned NumReducedVals = ReducedVals.size();
4431 if (NumReducedVals < 4)
4432 return false;
4433
4434 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
4435
4436 Value *VectorizedTree = nullptr;
4437 IRBuilder<> Builder(ReductionRoot);
4438 FastMathFlags Unsafe;
4439 Unsafe.setUnsafeAlgebra();
4440 Builder.setFastMathFlags(Unsafe);
4441 unsigned i = 0;
4442
4443 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
4444 // The same extra argument may be used several time, so log each attempt
4445 // to use it.
4446 for (auto &Pair : ExtraArgs)
4447 ExternallyUsedValues[Pair.second].push_back(Pair.first->getDebugLoc());
4448 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
4449 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
4450 V.buildTree(VL, ExternallyUsedValues, ReductionOps);
4451 if (V.shouldReorder()) {
4452 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
4453 V.buildTree(Reversed, ExternallyUsedValues, ReductionOps);
4454 }
4455 if (V.isTreeTinyAndNotFullyVectorizable())
4456 break;
4457
4458 V.computeMinimumValueSizes();
4459
4460 // Estimate cost.
4461 int Cost =
4462 V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
4463 if (Cost >= -SLPCostThreshold)
4464 break;
4465
4466 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
4467 << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
;
4468
4469 // Vectorize a tree.
4470 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
4471 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
4472
4473 // Emit a reduction.
4474 Value *ReducedSubTree =
4475 emitReduction(VectorizedRoot, Builder, ReduxWidth);
4476 if (VectorizedTree) {
4477 Builder.SetCurrentDebugLocation(Loc);
4478 VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree,
4479 ReducedSubTree, "bin.rdx");
4480 } else
4481 VectorizedTree = ReducedSubTree;
4482 i += ReduxWidth;
4483 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
4484 }
4485
4486 if (VectorizedTree) {
4487 // Finish the reduction.
4488 for (; i < NumReducedVals; ++i) {
4489 auto *I = cast<Instruction>(ReducedVals[i]);
4490 Builder.SetCurrentDebugLocation(I->getDebugLoc());
4491 VectorizedTree =
4492 Builder.CreateBinOp(ReductionOpcode, VectorizedTree, I);
4493 }
4494 for (auto &Pair : ExternallyUsedValues) {
4495 assert(!Pair.second.empty() &&((!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? static_cast<void> (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
4496 "At least one DebugLoc must be inserted")((!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? static_cast<void> (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4496, __PRETTY_FUNCTION__))
;
4497 // Add each externally used value to the final reduction.
4498 for (auto &DL : Pair.second) {
4499 Builder.SetCurrentDebugLocation(DL);
4500 VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree,
4501 Pair.first, "bin.extra");
4502 }
4503 }
4504 // Update users.
4505 if (ReductionPHI && !isa<UndefValue>(ReductionPHI)) {
4506 assert(ReductionRoot && "Need a reduction operation")((ReductionRoot && "Need a reduction operation") ? static_cast
<void> (0) : __assert_fail ("ReductionRoot && \"Need a reduction operation\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4506, __PRETTY_FUNCTION__))
;
4507 ReductionRoot->setOperand(0, VectorizedTree);
4508 ReductionRoot->setOperand(1, ReductionPHI);
4509 } else
4510 ReductionRoot->replaceAllUsesWith(VectorizedTree);
4511 }
4512 return VectorizedTree != nullptr;
4513 }
4514
4515 unsigned numReductionValues() const {
4516 return ReducedVals.size();
4517 }
4518
4519private:
4520 /// \brief Calculate the cost of a reduction.
4521 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
4522 unsigned ReduxWidth) {
4523 Type *ScalarTy = FirstReducedVal->getType();
4524 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
4525
4526 int PairwiseRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, true);
4527 int SplittingRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, false);
4528
4529 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
4530 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
4531
4532 int ScalarReduxCost =
4533 (ReduxWidth - 1) *
4534 TTI->getArithmeticInstrCost(ReductionOpcode, ScalarTy);
4535
4536 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4537 << " for reduction that starts with " << *FirstReducedValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4538 << " (It is a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4539 << (IsPairwiseReduction ? "pairwise" : "splitting")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4540 << " reduction)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
;
4541
4542 return VecReduxCost - ScalarReduxCost;
4543 }
4544
4545 /// \brief Emit a horizontal reduction of the vectorized value.
4546 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
4547 unsigned ReduxWidth) {
4548 assert(VectorizedValue && "Need to have a vectorized tree node")((VectorizedValue && "Need to have a vectorized tree node"
) ? static_cast<void> (0) : __assert_fail ("VectorizedValue && \"Need to have a vectorized tree node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4548, __PRETTY_FUNCTION__))
;
4549 assert(isPowerOf2_32(ReduxWidth) &&((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4550, __PRETTY_FUNCTION__))
4550 "We only handle power-of-two reductions for now")((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4550, __PRETTY_FUNCTION__))
;
4551
4552 Value *TmpVec = VectorizedValue;
4553 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
4554 if (IsPairwiseReduction) {
4555 Value *LeftMask =
4556 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
4557 Value *RightMask =
4558 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
4559
4560 Value *LeftShuf = Builder.CreateShuffleVector(
4561 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
4562 Value *RightShuf = Builder.CreateShuffleVector(
4563 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
4564 "rdx.shuf.r");
4565 TmpVec = Builder.CreateBinOp(ReductionOpcode, LeftShuf, RightShuf,
4566 "bin.rdx");
4567 } else {
4568 Value *UpperHalf =
4569 createRdxShuffleMask(ReduxWidth, i, false, false, Builder);
4570 Value *Shuf = Builder.CreateShuffleVector(
4571 TmpVec, UndefValue::get(TmpVec->getType()), UpperHalf, "rdx.shuf");
4572 TmpVec = Builder.CreateBinOp(ReductionOpcode, TmpVec, Shuf, "bin.rdx");
4573 }
4574 }
4575
4576 // The result is in the first element of the vector.
4577 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
4578 }
4579};
4580} // end anonymous namespace
4581
4582/// \brief Recognize construction of vectors like
4583/// %ra = insertelement <4 x float> undef, float %s0, i32 0
4584/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
4585/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
4586/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
4587///
4588/// Returns true if it matches
4589///
4590static bool findBuildVector(InsertElementInst *FirstInsertElem,
4591 SmallVectorImpl<Value *> &BuildVector,
4592 SmallVectorImpl<Value *> &BuildVectorOpds) {
4593 if (!isa<UndefValue>(FirstInsertElem->getOperand(0)))
4594 return false;
4595
4596 InsertElementInst *IE = FirstInsertElem;
4597 while (true) {
4598 BuildVector.push_back(IE);
4599 BuildVectorOpds.push_back(IE->getOperand(1));
4600
4601 if (IE->use_empty())
4602 return false;
4603
4604 InsertElementInst *NextUse = dyn_cast<InsertElementInst>(IE->user_back());
4605 if (!NextUse)
4606 return true;
4607
4608 // If this isn't the final use, make sure the next insertelement is the only
4609 // use. It's OK if the final constructed vector is used multiple times
4610 if (!IE->hasOneUse())
4611 return false;
4612
4613 IE = NextUse;
4614 }
4615
4616 return false;
4617}
4618
4619/// \brief Like findBuildVector, but looks backwards for construction of aggregate.
4620///
4621/// \return true if it matches.
4622static bool findBuildAggregate(InsertValueInst *IV,
4623 SmallVectorImpl<Value *> &BuildVector,
4624 SmallVectorImpl<Value *> &BuildVectorOpds) {
4625 Value *V;
4626 do {
4627 BuildVector.push_back(IV);
4628 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
4629 V = IV->getAggregateOperand();
4630 if (isa<UndefValue>(V))
4631 break;
4632 IV = dyn_cast<InsertValueInst>(V);
4633 if (!IV || !IV->hasOneUse())
4634 return false;
4635 } while (true);
4636 std::reverse(BuildVector.begin(), BuildVector.end());
4637 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
4638 return true;
4639}
4640
4641static bool PhiTypeSorterFunc(Value *V, Value *V2) {
4642 return V->getType() < V2->getType();
4643}
4644
4645/// \brief Try and get a reduction value from a phi node.
4646///
4647/// Given a phi node \p P in a block \p ParentBB, consider possible reductions
4648/// if they come from either \p ParentBB or a containing loop latch.
4649///
4650/// \returns A candidate reduction value if possible, or \code nullptr \endcode
4651/// if not possible.
4652static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
4653 BasicBlock *ParentBB, LoopInfo *LI) {
4654 // There are situations where the reduction value is not dominated by the
4655 // reduction phi. Vectorizing such cases has been reported to cause
4656 // miscompiles. See PR25787.
4657 auto DominatedReduxValue = [&](Value *R) {
4658 return (
4659 dyn_cast<Instruction>(R) &&
4660 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
4661 };
4662
4663 Value *Rdx = nullptr;
4664
4665 // Return the incoming value if it comes from the same BB as the phi node.
4666 if (P->getIncomingBlock(0) == ParentBB) {
4667 Rdx = P->getIncomingValue(0);
4668 } else if (P->getIncomingBlock(1) == ParentBB) {
4669 Rdx = P->getIncomingValue(1);
4670 }
4671
4672 if (Rdx && DominatedReduxValue(Rdx))
4673 return Rdx;
4674
4675 // Otherwise, check whether we have a loop latch to look at.
4676 Loop *BBL = LI->getLoopFor(ParentBB);
4677 if (!BBL)
4678 return nullptr;
4679 BasicBlock *BBLatch = BBL->getLoopLatch();
4680 if (!BBLatch)
4681 return nullptr;
4682
4683 // There is a loop latch, return the incoming value if it comes from
4684 // that. This reduction pattern occasionally turns up.
4685 if (P->getIncomingBlock(0) == BBLatch) {
4686 Rdx = P->getIncomingValue(0);
4687 } else if (P->getIncomingBlock(1) == BBLatch) {
4688 Rdx = P->getIncomingValue(1);
4689 }
4690
4691 if (Rdx && DominatedReduxValue(Rdx))
4692 return Rdx;
4693
4694 return nullptr;
4695}
4696
4697namespace {
4698/// Tracks instructons and its children.
4699class WeakVHWithLevel final : public CallbackVH {
4700 /// Operand index of the instruction currently beeing analized.
4701 unsigned Level = 0;
4702 /// Is this the instruction that should be vectorized, or are we now
4703 /// processing children (i.e. operands of this instruction) for potential
4704 /// vectorization?
4705 bool IsInitial = true;
4706
4707public:
4708 explicit WeakVHWithLevel() = default;
4709 WeakVHWithLevel(Value *V) : CallbackVH(V){};
4710 /// Restart children analysis each time it is repaced by the new instruction.
4711 void allUsesReplacedWith(Value *New) override {
4712 setValPtr(New);
4713 Level = 0;
4714 IsInitial = true;
4715 }
4716 /// Check if the instruction was not deleted during vectorization.
4717 bool isValid() const { return !getValPtr(); }
4718 /// Is the istruction itself must be vectorized?
4719 bool isInitial() const { return IsInitial; }
4720 /// Try to vectorize children.
4721 void clearInitial() { IsInitial = false; }
4722 /// Are all children processed already?
4723 bool isFinal() const {
4724 assert(getValPtr() &&((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4726, __PRETTY_FUNCTION__))
4725 (isa<Instruction>(getValPtr()) &&((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4726, __PRETTY_FUNCTION__))
4726 cast<Instruction>(getValPtr())->getNumOperands() >= Level))((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4726, __PRETTY_FUNCTION__))
;
4727 return getValPtr() &&
4728 cast<Instruction>(getValPtr())->getNumOperands() == Level;
4729 }
4730 /// Get next child operation.
4731 Value *nextOperand() {
4732 assert(getValPtr() && isa<Instruction>(getValPtr()) &&((getValPtr() && isa<Instruction>(getValPtr()) &&
cast<Instruction>(getValPtr())->getNumOperands() >
Level) ? static_cast<void> (0) : __assert_fail ("getValPtr() && isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() > Level"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4733, __PRETTY_FUNCTION__))
4733 cast<Instruction>(getValPtr())->getNumOperands() > Level)((getValPtr() && isa<Instruction>(getValPtr()) &&
cast<Instruction>(getValPtr())->getNumOperands() >
Level) ? static_cast<void> (0) : __assert_fail ("getValPtr() && isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() > Level"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4733, __PRETTY_FUNCTION__))
;
4734 return cast<Instruction>(getValPtr())->getOperand(Level++);
4735 }
4736 virtual ~WeakVHWithLevel() = default;
4737};
4738} // namespace
4739
4740/// \brief Attempt to reduce a horizontal reduction.
4741/// If it is legal to match a horizontal reduction feeding
4742/// the phi node P with reduction operators Root in a basic block BB, then check
4743/// if it can be done.
4744/// \returns true if a horizontal reduction was matched and reduced.
4745/// \returns false if a horizontal reduction was not matched.
4746static bool canBeVectorized(
4747 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
4748 TargetTransformInfo *TTI,
4749 const function_ref<bool(BinaryOperator *, BoUpSLP &)> Vectorize) {
4750 if (!ShouldVectorizeHor)
4751 return false;
4752
4753 if (!Root)
4754 return false;
4755
4756 if (Root->getParent() != BB)
4757 return false;
4758 SmallVector<WeakVHWithLevel, 8> Stack(1, Root);
4759 SmallSet<Value *, 8> VisitedInstrs;
4760 bool Res = false;
4761 while (!Stack.empty()) {
4762 Value *V = Stack.back();
4763 if (!V) {
4764 Stack.pop_back();
4765 continue;
4766 }
4767 auto *Inst = dyn_cast<Instruction>(V);
4768 if (!Inst || isa<PHINode>(Inst)) {
4769 Stack.pop_back();
4770 continue;
4771 }
4772 if (Stack.back().isInitial()) {
4773 Stack.back().clearInitial();
4774 if (auto *BI = dyn_cast<BinaryOperator>(Inst)) {
4775 HorizontalReduction HorRdx;
4776 if (HorRdx.matchAssociativeReduction(P, BI)) {
4777 if (HorRdx.tryToReduce(R, TTI)) {
4778 Res = true;
4779 P = nullptr;
4780 continue;
4781 }
4782 }
4783 if (P) {
4784 Inst = dyn_cast<Instruction>(BI->getOperand(0));
4785 if (Inst == P)
4786 Inst = dyn_cast<Instruction>(BI->getOperand(1));
4787 if (!Inst) {
4788 P = nullptr;
4789 continue;
4790 }
4791 }
4792 }
4793 P = nullptr;
4794 if (Vectorize(dyn_cast<BinaryOperator>(Inst), R)) {
4795 Res = true;
4796 continue;
4797 }
4798 }
4799 if (Stack.back().isFinal()) {
4800 Stack.pop_back();
4801 continue;
4802 }
4803
4804 if (auto *NextV = dyn_cast<Instruction>(Stack.back().nextOperand()))
4805 if (NextV->getParent() == BB && VisitedInstrs.insert(NextV).second &&
4806 Stack.size() < RecursionMaxDepth)
4807 Stack.push_back(NextV);
4808 }
4809 return Res;
4810}
4811
4812bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
4813 BasicBlock *BB, BoUpSLP &R,
4814 TargetTransformInfo *TTI) {
4815 if (!V)
4816 return false;
4817 auto *I = dyn_cast<Instruction>(V);
4818 if (!I)
4819 return false;
4820
4821 if (!isa<BinaryOperator>(I))
4822 P = nullptr;
4823 // Try to match and vectorize a horizontal reduction.
4824 return canBeVectorized(P, I, BB, R, TTI,
4825 [this](BinaryOperator *BI, BoUpSLP &R) -> bool {
4826 return tryToVectorize(BI, R);
4827 });
4828}
4829
4830bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
4831 bool Changed = false;
4832 SmallVector<Value *, 4> Incoming;
4833 SmallSet<Value *, 16> VisitedInstrs;
4834
4835 bool HaveVectorizedPhiNodes = true;
4836 while (HaveVectorizedPhiNodes) {
4837 HaveVectorizedPhiNodes = false;
4838
4839 // Collect the incoming values from the PHIs.
4840 Incoming.clear();
4841 for (Instruction &I : *BB) {
4842 PHINode *P = dyn_cast<PHINode>(&I);
4843 if (!P)
4844 break;
4845
4846 if (!VisitedInstrs.count(P))
4847 Incoming.push_back(P);
4848 }
4849
4850 // Sort by type.
4851 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
4852
4853 // Try to vectorize elements base on their type.
4854 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
4855 E = Incoming.end();
4856 IncIt != E;) {
4857
4858 // Look for the next elements with the same type.
4859 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
4860 while (SameTypeIt != E &&
4861 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
4862 VisitedInstrs.insert(*SameTypeIt);
4863 ++SameTypeIt;
4864 }
4865
4866 // Try to vectorize them.
4867 unsigned NumElts = (SameTypeIt - IncIt);
4868 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { errs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false)
;
4869 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R)) {
4870 // Success start over because instructions might have been changed.
4871 HaveVectorizedPhiNodes = true;
4872 Changed = true;
4873 break;
4874 }
4875
4876 // Start over at the next instruction of a different type (or the end).
4877 IncIt = SameTypeIt;
4878 }
4879 }
4880
4881 VisitedInstrs.clear();
4882
4883 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
4884 // We may go through BB multiple times so skip the one we have checked.
4885 if (!VisitedInstrs.insert(&*it).second)
4886 continue;
4887
4888 if (isa<DbgInfoIntrinsic>(it))
4889 continue;
4890
4891 // Try to vectorize reductions that use PHINodes.
4892 if (PHINode *P = dyn_cast<PHINode>(it)) {
4893 // Check that the PHI is a reduction PHI.
4894 if (P->getNumIncomingValues() != 2)
4895 return Changed;
4896
4897 // Try to match and vectorize a horizontal reduction.
4898 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
4899 TTI)) {
4900 Changed = true;
4901 it = BB->begin();
4902 e = BB->end();
4903 continue;
4904 }
4905 continue;
4906 }
4907
4908 if (ShouldStartVectorizeHorAtStore) {
4909 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
4910 // Try to match and vectorize a horizontal reduction.
4911 if (vectorizeRootInstruction(nullptr, SI->getValueOperand(), BB, R,
4912 TTI)) {
4913 Changed = true;
4914 it = BB->begin();
4915 e = BB->end();
4916 continue;
4917 }
4918 }
4919 }
4920
4921 // Try to vectorize horizontal reductions feeding into a return.
4922 if (ReturnInst *RI = dyn_cast<ReturnInst>(it)) {
4923 if (RI->getNumOperands() != 0) {
4924 // Try to match and vectorize a horizontal reduction.
4925 if (vectorizeRootInstruction(nullptr, RI->getOperand(0), BB, R, TTI)) {
4926 Changed = true;
4927 it = BB->begin();
4928 e = BB->end();
4929 continue;
4930 }
4931 }
4932 }
4933
4934 // Try to vectorize trees that start at compare instructions.
4935 if (CmpInst *CI = dyn_cast<CmpInst>(it)) {
4936 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R)) {
4937 Changed = true;
4938 // We would like to start over since some instructions are deleted
4939 // and the iterator may become invalid value.
4940 it = BB->begin();
4941 e = BB->end();
4942 continue;
4943 }
4944
4945 for (int I = 0; I < 2; ++I) {
4946 if (vectorizeRootInstruction(nullptr, CI->getOperand(I), BB, R, TTI)) {
4947 Changed = true;
4948 // We would like to start over since some instructions are deleted
4949 // and the iterator may become invalid value.
4950 it = BB->begin();
4951 e = BB->end();
4952 break;
4953 }
4954 }
4955 continue;
4956 }
4957
4958 // Try to vectorize trees that start at insertelement instructions.
4959 if (InsertElementInst *FirstInsertElem = dyn_cast<InsertElementInst>(it)) {
4960 SmallVector<Value *, 16> BuildVector;
4961 SmallVector<Value *, 16> BuildVectorOpds;
4962 if (!findBuildVector(FirstInsertElem, BuildVector, BuildVectorOpds))
4963 continue;
4964
4965 // Vectorize starting with the build vector operands ignoring the
4966 // BuildVector instructions for the purpose of scheduling and user
4967 // extraction.
4968 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector)) {
4969 Changed = true;
4970 it = BB->begin();
4971 e = BB->end();
4972 }
4973
4974 continue;
4975 }
4976
4977 // Try to vectorize trees that start at insertvalue instructions feeding into
4978 // a store.
4979 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
4980 if (InsertValueInst *LastInsertValue = dyn_cast<InsertValueInst>(SI->getValueOperand())) {
4981 const DataLayout &DL = BB->getModule()->getDataLayout();
4982 if (R.canMapToVector(SI->getValueOperand()->getType(), DL)) {
4983 SmallVector<Value *, 16> BuildVector;
4984 SmallVector<Value *, 16> BuildVectorOpds;
4985 if (!findBuildAggregate(LastInsertValue, BuildVector, BuildVectorOpds))
4986 continue;
4987
4988 DEBUG(dbgs() << "SLP: store of array mappable to vector: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: store of array mappable to vector: "
<< *SI << "\n"; } } while (false)
;
4989 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector, false)) {
4990 Changed = true;
4991 it = BB->begin();
4992 e = BB->end();
4993 }
4994 continue;
4995 }
4996 }
4997 }
4998 }
4999
5000 return Changed;
5001}
5002
5003bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
5004 auto Changed = false;
5005 for (auto &Entry : GEPs) {
5006
5007 // If the getelementptr list has fewer than two elements, there's nothing
5008 // to do.
5009 if (Entry.second.size() < 2)
5010 continue;
5011
5012 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
5013 << Entry.second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
;
5014
5015 // We process the getelementptr list in chunks of 16 (like we do for
5016 // stores) to minimize compile-time.
5017 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
5018 auto Len = std::min<unsigned>(BE - BI, 16);
5019 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
5020
5021 // Initialize a set a candidate getelementptrs. Note that we use a
5022 // SetVector here to preserve program order. If the index computations
5023 // are vectorizable and begin with loads, we want to minimize the chance
5024 // of having to reorder them later.
5025 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
5026
5027 // Some of the candidates may have already been vectorized after we
5028 // initially collected them. If so, the WeakVHs will have nullified the
5029 // values, so remove them from the set of candidates.
5030 Candidates.remove(nullptr);
5031
5032 // Remove from the set of candidates all pairs of getelementptrs with
5033 // constant differences. Such getelementptrs are likely not good
5034 // candidates for vectorization in a bottom-up phase since one can be
5035 // computed from the other. We also ensure all candidate getelementptr
5036 // indices are unique.
5037 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
5038 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
5039 if (!Candidates.count(GEPI))
5040 continue;
5041 auto *SCEVI = SE->getSCEV(GEPList[I]);
5042 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
5043 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
5044 auto *SCEVJ = SE->getSCEV(GEPList[J]);
5045 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
5046 Candidates.remove(GEPList[I]);
5047 Candidates.remove(GEPList[J]);
5048 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
5049 Candidates.remove(GEPList[J]);
5050 }
5051 }
5052 }
5053
5054 // We break out of the above computation as soon as we know there are
5055 // fewer than two candidates remaining.
5056 if (Candidates.size() < 2)
5057 continue;
5058
5059 // Add the single, non-constant index of each candidate to the bundle. We
5060 // ensured the indices met these constraints when we originally collected
5061 // the getelementptrs.
5062 SmallVector<Value *, 16> Bundle(Candidates.size());
5063 auto BundleIndex = 0u;
5064 for (auto *V : Candidates) {
5065 auto *GEP = cast<GetElementPtrInst>(V);
5066 auto *GEPIdx = GEP->idx_begin()->get();
5067 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx))((GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx
)) ? static_cast<void> (0) : __assert_fail ("GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn296300/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5067, __PRETTY_FUNCTION__))
;
5068 Bundle[BundleIndex++] = GEPIdx;
5069 }
5070
5071 // Try and vectorize the indices. We are currently only interested in
5072 // gather-like cases of the form:
5073 //
5074 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
5075 //
5076 // where the loads of "a", the loads of "b", and the subtractions can be
5077 // performed in parallel. It's likely that detecting this pattern in a
5078 // bottom-up phase will be simpler and less costly than building a
5079 // full-blown top-down phase beginning at the consecutive loads.
5080 Changed |= tryToVectorizeList(Bundle, R);
5081 }
5082 }
5083 return Changed;
5084}
5085
5086bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
5087 bool Changed = false;
5088 // Attempt to sort and vectorize each of the store-groups.
5089 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
5090 ++it) {
5091 if (it->second.size() < 2)
5092 continue;
5093
5094 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
5095 << it->second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
;
5096
5097 // Process the stores in chunks of 16.
5098 // TODO: The limit of 16 inhibits greater vectorization factors.
5099 // For example, AVX2 supports v32i8. Increasing this limit, however,
5100 // may cause a significant compile-time increase.
5101 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
5102 unsigned Len = std::min<unsigned>(CE - CI, 16);
5103 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
5104 }
5105 }
5106 return Changed;
5107}
5108
5109char SLPVectorizer::ID = 0;
5110static const char lv_name[] = "SLP Vectorizer";
5111INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)static void *initializeSLPVectorizerPassOnce(PassRegistry &
Registry) {
5112INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
5113INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
5114INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
5115INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
5116INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry);
5117INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
5118INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "slp-vectorizer", &
SLPVectorizer::ID, PassInfo::NormalCtor_t(callDefaultCtor<
SLPVectorizer>), false, false); Registry.registerPass(*PI,
true); return PI; } static llvm::once_flag InitializeSLPVectorizerPassFlag
; void llvm::initializeSLPVectorizerPass(PassRegistry &Registry
) { llvm::call_once(InitializeSLPVectorizerPassFlag, initializeSLPVectorizerPassOnce
, std::ref(Registry)); }
5119
5120namespace llvm {
5121Pass *createSLPVectorizerPass() { return new SLPVectorizer(); }
5122}