Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 3064, column 22
Called C++ object pointer is null

Annotated Source Code

1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10// stores that can be put together into vector-stores. Next, it attempts to
11// construct vectorizable tree using the use-def chains. If a profitable tree
12// was found, the SLP vectorizer performs vectorization on the tree.
13//
14// The pass is inspired by the work described in the paper:
15// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16//
17//===----------------------------------------------------------------------===//
18#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
19#include "llvm/ADT/Optional.h"
20#include "llvm/ADT/PostOrderIterator.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/CodeMetrics.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/LoopAccessAnalysis.h"
26#include "llvm/Analysis/ScalarEvolutionExpressions.h"
27#include "llvm/Analysis/ValueTracking.h"
28#include "llvm/Analysis/VectorUtils.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/Instructions.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NoFolder.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/Value.h"
38#include "llvm/IR/Verifier.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/CommandLine.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/GraphWriter.h"
43#include "llvm/Support/KnownBits.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Transforms/Utils/LoopUtils.h"
46#include "llvm/Transforms/Vectorize.h"
47#include <algorithm>
48#include <memory>
49
50using namespace llvm;
51using namespace slpvectorizer;
52
53#define SV_NAME"slp-vectorizer" "slp-vectorizer"
54#define DEBUG_TYPE"SLP" "SLP"
55
56STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, false}
;
57
58static cl::opt<int>
59 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
60 cl::desc("Only vectorize if you gain more than this "
61 "number "));
62
63static cl::opt<bool>
64ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
65 cl::desc("Attempt to vectorize horizontal reductions"));
66
67static cl::opt<bool> ShouldStartVectorizeHorAtStore(
68 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
69 cl::desc(
70 "Attempt to vectorize horizontal reductions feeding into a store"));
71
72static cl::opt<int>
73MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
74 cl::desc("Attempt to vectorize for this register size in bits"));
75
76/// Limits the size of scheduling regions in a block.
77/// It avoid long compile times for _very_ large blocks where vector
78/// instructions are spread over a wide range.
79/// This limit is way higher than needed by real-world functions.
80static cl::opt<int>
81ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
82 cl::desc("Limit the size of the SLP scheduling region per block"));
83
84static cl::opt<int> MinVectorRegSizeOption(
85 "slp-min-reg-size", cl::init(128), cl::Hidden,
86 cl::desc("Attempt to vectorize for this register size in bits"));
87
88static cl::opt<unsigned> RecursionMaxDepth(
89 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
90 cl::desc("Limit the recursion depth when building a vectorizable tree"));
91
92static cl::opt<unsigned> MinTreeSize(
93 "slp-min-tree-size", cl::init(3), cl::Hidden,
94 cl::desc("Only vectorize small trees if they are fully vectorizable"));
95
96static cl::opt<bool>
97 ViewSLPTree("view-slp-tree", cl::Hidden,
98 cl::desc("Display the SLP trees with Graphviz"));
99
100// Limit the number of alias checks. The limit is chosen so that
101// it has no negative effect on the llvm benchmarks.
102static const unsigned AliasedCheckLimit = 10;
103
104// Another limit for the alias checks: The maximum distance between load/store
105// instructions where alias checks are done.
106// This limit is useful for very large basic blocks.
107static const unsigned MaxMemDepDistance = 160;
108
109/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
110/// regions to be handled.
111static const int MinScheduleRegionSize = 16;
112
113/// \brief Predicate for the element types that the SLP vectorizer supports.
114///
115/// The most important thing to filter here are types which are invalid in LLVM
116/// vectors. We also filter target specific types which have absolutely no
117/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
118/// avoids spending time checking the cost model and realizing that they will
119/// be inevitably scalarized.
120static bool isValidElementType(Type *Ty) {
121 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
122 !Ty->isPPC_FP128Ty();
123}
124
125/// \returns true if all of the instructions in \p VL are in the same block or
126/// false otherwise.
127static bool allSameBlock(ArrayRef<Value *> VL) {
128 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
129 if (!I0)
130 return false;
131 BasicBlock *BB = I0->getParent();
132 for (int i = 1, e = VL.size(); i < e; i++) {
133 Instruction *I = dyn_cast<Instruction>(VL[i]);
134 if (!I)
135 return false;
136
137 if (BB != I->getParent())
138 return false;
139 }
140 return true;
141}
142
143/// \returns True if all of the values in \p VL are constants.
144static bool allConstant(ArrayRef<Value *> VL) {
145 for (Value *i : VL)
146 if (!isa<Constant>(i))
147 return false;
148 return true;
149}
150
151/// \returns True if all of the values in \p VL are identical.
152static bool isSplat(ArrayRef<Value *> VL) {
153 for (unsigned i = 1, e = VL.size(); i < e; ++i)
154 if (VL[i] != VL[0])
155 return false;
156 return true;
157}
158
159///\returns Opcode that can be clubbed with \p Op to create an alternate
160/// sequence which can later be merged as a ShuffleVector instruction.
161static unsigned getAltOpcode(unsigned Op) {
162 switch (Op) {
163 case Instruction::FAdd:
164 return Instruction::FSub;
165 case Instruction::FSub:
166 return Instruction::FAdd;
167 case Instruction::Add:
168 return Instruction::Sub;
169 case Instruction::Sub:
170 return Instruction::Add;
171 default:
172 return 0;
173 }
174}
175
176///\returns bool representing if Opcode \p Op can be part
177/// of an alternate sequence which can later be merged as
178/// a ShuffleVector instruction.
179static bool canCombineAsAltInst(unsigned Op) {
180 return Op == Instruction::FAdd || Op == Instruction::FSub ||
181 Op == Instruction::Sub || Op == Instruction::Add;
182}
183
184/// \returns ShuffleVector instruction if instructions in \p VL have
185/// alternate fadd,fsub / fsub,fadd/add,sub/sub,add sequence.
186/// (i.e. e.g. opcodes of fadd,fsub,fadd,fsub...)
187static unsigned isAltInst(ArrayRef<Value *> VL) {
188 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
189 unsigned Opcode = I0->getOpcode();
190 unsigned AltOpcode = getAltOpcode(Opcode);
191 for (int i = 1, e = VL.size(); i < e; i++) {
192 Instruction *I = dyn_cast<Instruction>(VL[i]);
193 if (!I || I->getOpcode() != ((i & 1) ? AltOpcode : Opcode))
194 return 0;
195 }
196 return Instruction::ShuffleVector;
197}
198
199/// \returns The opcode if all of the Instructions in \p VL have the same
200/// opcode, or zero.
201static unsigned getSameOpcode(ArrayRef<Value *> VL) {
202 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
203 if (!I0)
204 return 0;
205 unsigned Opcode = I0->getOpcode();
206 for (int i = 1, e = VL.size(); i < e; i++) {
207 Instruction *I = dyn_cast<Instruction>(VL[i]);
208 if (!I || Opcode != I->getOpcode()) {
209 if (canCombineAsAltInst(Opcode) && i == 1)
210 return isAltInst(VL);
211 return 0;
212 }
213 }
214 return Opcode;
215}
216
217/// \returns true if all of the values in \p VL have the same type or false
218/// otherwise.
219static bool allSameType(ArrayRef<Value *> VL) {
220 Type *Ty = VL[0]->getType();
221 for (int i = 1, e = VL.size(); i < e; i++)
222 if (VL[i]->getType() != Ty)
223 return false;
224
225 return true;
226}
227
228/// \returns True if Extract{Value,Element} instruction extracts element Idx.
229static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
230 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 231, __PRETTY_FUNCTION__))
231 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 231, __PRETTY_FUNCTION__))
;
232 if (Opcode == Instruction::ExtractElement) {
233 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
234 return CI && CI->getZExtValue() == Idx;
235 } else {
236 ExtractValueInst *EI = cast<ExtractValueInst>(E);
237 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
238 }
239}
240
241/// \returns True if in-tree use also needs extract. This refers to
242/// possible scalar operand in vectorized instruction.
243static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
244 TargetLibraryInfo *TLI) {
245
246 unsigned Opcode = UserInst->getOpcode();
247 switch (Opcode) {
248 case Instruction::Load: {
249 LoadInst *LI = cast<LoadInst>(UserInst);
250 return (LI->getPointerOperand() == Scalar);
251 }
252 case Instruction::Store: {
253 StoreInst *SI = cast<StoreInst>(UserInst);
254 return (SI->getPointerOperand() == Scalar);
255 }
256 case Instruction::Call: {
257 CallInst *CI = cast<CallInst>(UserInst);
258 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
259 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
260 return (CI->getArgOperand(1) == Scalar);
261 }
262 }
263 default:
264 return false;
265 }
266}
267
268/// \returns the AA location that is being access by the instruction.
269static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
270 if (StoreInst *SI = dyn_cast<StoreInst>(I))
271 return MemoryLocation::get(SI);
272 if (LoadInst *LI = dyn_cast<LoadInst>(I))
273 return MemoryLocation::get(LI);
274 return MemoryLocation();
275}
276
277/// \returns True if the instruction is not a volatile or atomic load/store.
278static bool isSimple(Instruction *I) {
279 if (LoadInst *LI = dyn_cast<LoadInst>(I))
280 return LI->isSimple();
281 if (StoreInst *SI = dyn_cast<StoreInst>(I))
282 return SI->isSimple();
283 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
284 return !MI->isVolatile();
285 return true;
286}
287
288namespace llvm {
289namespace slpvectorizer {
290/// Bottom Up SLP Vectorizer.
291class BoUpSLP {
292public:
293 typedef SmallVector<Value *, 8> ValueList;
294 typedef SmallVector<Instruction *, 16> InstrList;
295 typedef SmallPtrSet<Value *, 16> ValueSet;
296 typedef SmallVector<StoreInst *, 8> StoreList;
297 typedef MapVector<Value *, SmallVector<Instruction *, 2>>
298 ExtraValueToDebugLocsMap;
299
300 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
301 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
302 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
303 const DataLayout *DL, OptimizationRemarkEmitter *ORE)
304 : NumLoadsWantToKeepOrder(0), NumLoadsWantToChangeOrder(0), F(Func),
305 SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC), DB(DB),
306 DL(DL), ORE(ORE), Builder(Se->getContext()) {
307 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
308 // Use the vector register size specified by the target unless overridden
309 // by a command-line option.
310 // TODO: It would be better to limit the vectorization factor based on
311 // data type rather than just register size. For example, x86 AVX has
312 // 256-bit registers, but it does not support integer operations
313 // at that width (that requires AVX2).
314 if (MaxVectorRegSizeOption.getNumOccurrences())
315 MaxVecRegSize = MaxVectorRegSizeOption;
316 else
317 MaxVecRegSize = TTI->getRegisterBitWidth(true);
318
319 if (MinVectorRegSizeOption.getNumOccurrences())
320 MinVecRegSize = MinVectorRegSizeOption;
321 else
322 MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
323 }
324
325 /// \brief Vectorize the tree that starts with the elements in \p VL.
326 /// Returns the vectorized root.
327 Value *vectorizeTree();
328 /// Vectorize the tree but with the list of externally used values \p
329 /// ExternallyUsedValues. Values in this MapVector can be replaced but the
330 /// generated extractvalue instructions.
331 Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
332
333 /// \returns the cost incurred by unwanted spills and fills, caused by
334 /// holding live values over call sites.
335 int getSpillCost();
336
337 /// \returns the vectorization cost of the subtree that starts at \p VL.
338 /// A negative number means that this is profitable.
339 int getTreeCost();
340
341 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
342 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
343 void buildTree(ArrayRef<Value *> Roots,
344 ArrayRef<Value *> UserIgnoreLst = None);
345 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
346 /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
347 /// into account (anf updating it, if required) list of externally used
348 /// values stored in \p ExternallyUsedValues.
349 void buildTree(ArrayRef<Value *> Roots,
350 ExtraValueToDebugLocsMap &ExternallyUsedValues,
351 ArrayRef<Value *> UserIgnoreLst = None);
352
353 /// Clear the internal data structures that are created by 'buildTree'.
354 void deleteTree() {
355 VectorizableTree.clear();
356 ScalarToTreeEntry.clear();
357 MustGather.clear();
358 ExternalUses.clear();
359 NumLoadsWantToKeepOrder = 0;
360 NumLoadsWantToChangeOrder = 0;
361 for (auto &Iter : BlocksSchedules) {
362 BlockScheduling *BS = Iter.second.get();
363 BS->clear();
364 }
365 MinBWs.clear();
366 }
367
368 unsigned getTreeSize() const { return VectorizableTree.size(); }
369
370 /// \brief Perform LICM and CSE on the newly generated gather sequences.
371 void optimizeGatherSequence();
372
373 /// \returns true if it is beneficial to reverse the vector order.
374 bool shouldReorder() const {
375 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
376 }
377
378 /// \return The vector element size in bits to use when vectorizing the
379 /// expression tree ending at \p V. If V is a store, the size is the width of
380 /// the stored value. Otherwise, the size is the width of the largest loaded
381 /// value reaching V. This method is used by the vectorizer to calculate
382 /// vectorization factors.
383 unsigned getVectorElementSize(Value *V);
384
385 /// Compute the minimum type sizes required to represent the entries in a
386 /// vectorizable tree.
387 void computeMinimumValueSizes();
388
389 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
390 unsigned getMaxVecRegSize() const {
391 return MaxVecRegSize;
392 }
393
394 // \returns minimum vector register size as set by cl::opt.
395 unsigned getMinVecRegSize() const {
396 return MinVecRegSize;
397 }
398
399 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
400 ///
401 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
402 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
403
404 /// \returns True if the VectorizableTree is both tiny and not fully
405 /// vectorizable. We do not vectorize such trees.
406 bool isTreeTinyAndNotFullyVectorizable();
407
408 OptimizationRemarkEmitter *getORE() { return ORE; }
409
410private:
411 struct TreeEntry;
412
413 /// \returns the cost of the vectorizable entry.
414 int getEntryCost(TreeEntry *E);
415
416 /// This is the recursive part of buildTree.
417 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
418
419 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
420 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
421 bool canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const;
422
423 /// Vectorize a single entry in the tree.
424 Value *vectorizeTree(TreeEntry *E);
425
426 /// Vectorize a single entry in the tree, starting in \p VL.
427 Value *vectorizeTree(ArrayRef<Value *> VL);
428
429 /// \returns the pointer to the vectorized value if \p VL is already
430 /// vectorized, or NULL. They may happen in cycles.
431 Value *alreadyVectorized(ArrayRef<Value *> VL) const;
432
433 /// \returns the scalarization cost for this type. Scalarization in this
434 /// context means the creation of vectors from a group of scalars.
435 int getGatherCost(Type *Ty);
436
437 /// \returns the scalarization cost for this list of values. Assuming that
438 /// this subtree gets vectorized, we may need to extract the values from the
439 /// roots. This method calculates the cost of extracting the values.
440 int getGatherCost(ArrayRef<Value *> VL);
441
442 /// \brief Set the Builder insert point to one after the last instruction in
443 /// the bundle
444 void setInsertPointAfterBundle(ArrayRef<Value *> VL);
445
446 /// \returns a vector from a collection of scalars in \p VL.
447 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
448
449 /// \returns whether the VectorizableTree is fully vectorizable and will
450 /// be beneficial even the tree height is tiny.
451 bool isFullyVectorizableTinyTree();
452
453 /// \reorder commutative operands in alt shuffle if they result in
454 /// vectorized code.
455 void reorderAltShuffleOperands(ArrayRef<Value *> VL,
456 SmallVectorImpl<Value *> &Left,
457 SmallVectorImpl<Value *> &Right);
458 /// \reorder commutative operands to get better probability of
459 /// generating vectorized code.
460 void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
461 SmallVectorImpl<Value *> &Left,
462 SmallVectorImpl<Value *> &Right);
463 struct TreeEntry {
464 TreeEntry(std::vector<TreeEntry> &Container)
465 : Scalars(), VectorizedValue(nullptr), NeedToGather(0),
466 Container(Container) {}
467
468 /// \returns true if the scalars in VL are equal to this entry.
469 bool isSame(ArrayRef<Value *> VL) const {
470 assert(VL.size() == Scalars.size() && "Invalid size")((VL.size() == Scalars.size() && "Invalid size") ? static_cast
<void> (0) : __assert_fail ("VL.size() == Scalars.size() && \"Invalid size\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 470, __PRETTY_FUNCTION__))
;
471 return std::equal(VL.begin(), VL.end(), Scalars.begin());
472 }
473
474 /// A vector of scalars.
475 ValueList Scalars;
476
477 /// The Scalars are vectorized into this value. It is initialized to Null.
478 Value *VectorizedValue;
479
480 /// Do we need to gather this sequence ?
481 bool NeedToGather;
482
483 /// Points back to the VectorizableTree.
484 ///
485 /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
486 /// to be a pointer and needs to be able to initialize the child iterator.
487 /// Thus we need a reference back to the container to translate the indices
488 /// to entries.
489 std::vector<TreeEntry> &Container;
490
491 /// The TreeEntry index containing the user of this entry. We can actually
492 /// have multiple users so the data structure is not truly a tree.
493 SmallVector<int, 1> UserTreeIndices;
494 };
495
496 /// Create a new VectorizableTree entry.
497 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
498 int &UserTreeIdx) {
499 VectorizableTree.emplace_back(VectorizableTree);
500 int idx = VectorizableTree.size() - 1;
501 TreeEntry *Last = &VectorizableTree[idx];
502 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
503 Last->NeedToGather = !Vectorized;
504 if (Vectorized) {
505 for (int i = 0, e = VL.size(); i != e; ++i) {
506 assert(!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!")((!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!"
) ? static_cast<void> (0) : __assert_fail ("!ScalarToTreeEntry.count(VL[i]) && \"Scalar already in tree!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 506, __PRETTY_FUNCTION__))
;
507 ScalarToTreeEntry[VL[i]] = idx;
508 }
509 } else {
510 MustGather.insert(VL.begin(), VL.end());
511 }
512
513 if (UserTreeIdx >= 0)
514 Last->UserTreeIndices.push_back(UserTreeIdx);
515 UserTreeIdx = idx;
516 return Last;
517 }
518
519 /// -- Vectorization State --
520 /// Holds all of the tree entries.
521 std::vector<TreeEntry> VectorizableTree;
522
523 /// Maps a specific scalar to its tree entry.
524 SmallDenseMap<Value*, int> ScalarToTreeEntry;
525
526 /// A list of scalars that we found that we need to keep as scalars.
527 ValueSet MustGather;
528
529 /// This POD struct describes one external user in the vectorized tree.
530 struct ExternalUser {
531 ExternalUser (Value *S, llvm::User *U, int L) :
532 Scalar(S), User(U), Lane(L){}
533 // Which scalar in our function.
534 Value *Scalar;
535 // Which user that uses the scalar.
536 llvm::User *User;
537 // Which lane does the scalar belong to.
538 int Lane;
539 };
540 typedef SmallVector<ExternalUser, 16> UserList;
541
542 /// Checks if two instructions may access the same memory.
543 ///
544 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
545 /// is invariant in the calling loop.
546 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
547 Instruction *Inst2) {
548
549 // First check if the result is already in the cache.
550 AliasCacheKey key = std::make_pair(Inst1, Inst2);
551 Optional<bool> &result = AliasCache[key];
552 if (result.hasValue()) {
553 return result.getValue();
554 }
555 MemoryLocation Loc2 = getLocation(Inst2, AA);
556 bool aliased = true;
557 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
558 // Do the alias check.
559 aliased = AA->alias(Loc1, Loc2);
560 }
561 // Store the result in the cache.
562 result = aliased;
563 return aliased;
564 }
565
566 typedef std::pair<Instruction *, Instruction *> AliasCacheKey;
567
568 /// Cache for alias results.
569 /// TODO: consider moving this to the AliasAnalysis itself.
570 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
571
572 /// Removes an instruction from its block and eventually deletes it.
573 /// It's like Instruction::eraseFromParent() except that the actual deletion
574 /// is delayed until BoUpSLP is destructed.
575 /// This is required to ensure that there are no incorrect collisions in the
576 /// AliasCache, which can happen if a new instruction is allocated at the
577 /// same address as a previously deleted instruction.
578 void eraseInstruction(Instruction *I) {
579 I->removeFromParent();
580 I->dropAllReferences();
581 DeletedInstructions.emplace_back(I);
582 }
583
584 /// Temporary store for deleted instructions. Instructions will be deleted
585 /// eventually when the BoUpSLP is destructed.
586 SmallVector<unique_value, 8> DeletedInstructions;
587
588 /// A list of values that need to extracted out of the tree.
589 /// This list holds pairs of (Internal Scalar : External User). External User
590 /// can be nullptr, it means that this Internal Scalar will be used later,
591 /// after vectorization.
592 UserList ExternalUses;
593
594 /// Values used only by @llvm.assume calls.
595 SmallPtrSet<const Value *, 32> EphValues;
596
597 /// Holds all of the instructions that we gathered.
598 SetVector<Instruction *> GatherSeq;
599 /// A list of blocks that we are going to CSE.
600 SetVector<BasicBlock *> CSEBlocks;
601
602 /// Contains all scheduling relevant data for an instruction.
603 /// A ScheduleData either represents a single instruction or a member of an
604 /// instruction bundle (= a group of instructions which is combined into a
605 /// vector instruction).
606 struct ScheduleData {
607
608 // The initial value for the dependency counters. It means that the
609 // dependencies are not calculated yet.
610 enum { InvalidDeps = -1 };
611
612 ScheduleData()
613 : Inst(nullptr), FirstInBundle(nullptr), NextInBundle(nullptr),
614 NextLoadStore(nullptr), SchedulingRegionID(0), SchedulingPriority(0),
615 Dependencies(InvalidDeps), UnscheduledDeps(InvalidDeps),
616 UnscheduledDepsInBundle(InvalidDeps), IsScheduled(false) {}
617
618 void init(int BlockSchedulingRegionID) {
619 FirstInBundle = this;
620 NextInBundle = nullptr;
621 NextLoadStore = nullptr;
622 IsScheduled = false;
623 SchedulingRegionID = BlockSchedulingRegionID;
624 UnscheduledDepsInBundle = UnscheduledDeps;
625 clearDependencies();
626 }
627
628 /// Returns true if the dependency information has been calculated.
629 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
630
631 /// Returns true for single instructions and for bundle representatives
632 /// (= the head of a bundle).
633 bool isSchedulingEntity() const { return FirstInBundle == this; }
634
635 /// Returns true if it represents an instruction bundle and not only a
636 /// single instruction.
637 bool isPartOfBundle() const {
638 return NextInBundle != nullptr || FirstInBundle != this;
639 }
640
641 /// Returns true if it is ready for scheduling, i.e. it has no more
642 /// unscheduled depending instructions/bundles.
643 bool isReady() const {
644 assert(isSchedulingEntity() &&((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 645, __PRETTY_FUNCTION__))
645 "can't consider non-scheduling entity for ready list")((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 645, __PRETTY_FUNCTION__))
;
646 return UnscheduledDepsInBundle == 0 && !IsScheduled;
647 }
648
649 /// Modifies the number of unscheduled dependencies, also updating it for
650 /// the whole bundle.
651 int incrementUnscheduledDeps(int Incr) {
652 UnscheduledDeps += Incr;
653 return FirstInBundle->UnscheduledDepsInBundle += Incr;
654 }
655
656 /// Sets the number of unscheduled dependencies to the number of
657 /// dependencies.
658 void resetUnscheduledDeps() {
659 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
660 }
661
662 /// Clears all dependency information.
663 void clearDependencies() {
664 Dependencies = InvalidDeps;
665 resetUnscheduledDeps();
666 MemoryDependencies.clear();
667 }
668
669 void dump(raw_ostream &os) const {
670 if (!isSchedulingEntity()) {
671 os << "/ " << *Inst;
672 } else if (NextInBundle) {
673 os << '[' << *Inst;
674 ScheduleData *SD = NextInBundle;
675 while (SD) {
676 os << ';' << *SD->Inst;
677 SD = SD->NextInBundle;
678 }
679 os << ']';
680 } else {
681 os << *Inst;
682 }
683 }
684
685 Instruction *Inst;
686
687 /// Points to the head in an instruction bundle (and always to this for
688 /// single instructions).
689 ScheduleData *FirstInBundle;
690
691 /// Single linked list of all instructions in a bundle. Null if it is a
692 /// single instruction.
693 ScheduleData *NextInBundle;
694
695 /// Single linked list of all memory instructions (e.g. load, store, call)
696 /// in the block - until the end of the scheduling region.
697 ScheduleData *NextLoadStore;
698
699 /// The dependent memory instructions.
700 /// This list is derived on demand in calculateDependencies().
701 SmallVector<ScheduleData *, 4> MemoryDependencies;
702
703 /// This ScheduleData is in the current scheduling region if this matches
704 /// the current SchedulingRegionID of BlockScheduling.
705 int SchedulingRegionID;
706
707 /// Used for getting a "good" final ordering of instructions.
708 int SchedulingPriority;
709
710 /// The number of dependencies. Constitutes of the number of users of the
711 /// instruction plus the number of dependent memory instructions (if any).
712 /// This value is calculated on demand.
713 /// If InvalidDeps, the number of dependencies is not calculated yet.
714 ///
715 int Dependencies;
716
717 /// The number of dependencies minus the number of dependencies of scheduled
718 /// instructions. As soon as this is zero, the instruction/bundle gets ready
719 /// for scheduling.
720 /// Note that this is negative as long as Dependencies is not calculated.
721 int UnscheduledDeps;
722
723 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
724 /// single instructions.
725 int UnscheduledDepsInBundle;
726
727 /// True if this instruction is scheduled (or considered as scheduled in the
728 /// dry-run).
729 bool IsScheduled;
730 };
731
732#ifndef NDEBUG
733 friend inline raw_ostream &operator<<(raw_ostream &os,
734 const BoUpSLP::ScheduleData &SD) {
735 SD.dump(os);
736 return os;
737 }
738#endif
739 friend struct GraphTraits<BoUpSLP *>;
740 friend struct DOTGraphTraits<BoUpSLP *>;
741
742 /// Contains all scheduling data for a basic block.
743 ///
744 struct BlockScheduling {
745
746 BlockScheduling(BasicBlock *BB)
747 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize),
748 ScheduleStart(nullptr), ScheduleEnd(nullptr),
749 FirstLoadStoreInRegion(nullptr), LastLoadStoreInRegion(nullptr),
750 ScheduleRegionSize(0),
751 ScheduleRegionSizeLimit(ScheduleRegionSizeBudget),
752 // Make sure that the initial SchedulingRegionID is greater than the
753 // initial SchedulingRegionID in ScheduleData (which is 0).
754 SchedulingRegionID(1) {}
755
756 void clear() {
757 ReadyInsts.clear();
758 ScheduleStart = nullptr;
759 ScheduleEnd = nullptr;
760 FirstLoadStoreInRegion = nullptr;
761 LastLoadStoreInRegion = nullptr;
762
763 // Reduce the maximum schedule region size by the size of the
764 // previous scheduling run.
765 ScheduleRegionSizeLimit -= ScheduleRegionSize;
766 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
767 ScheduleRegionSizeLimit = MinScheduleRegionSize;
768 ScheduleRegionSize = 0;
769
770 // Make a new scheduling region, i.e. all existing ScheduleData is not
771 // in the new region yet.
772 ++SchedulingRegionID;
773 }
774
775 ScheduleData *getScheduleData(Value *V) {
776 ScheduleData *SD = ScheduleDataMap[V];
777 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
778 return SD;
779 return nullptr;
780 }
781
782 bool isInSchedulingRegion(ScheduleData *SD) {
783 return SD->SchedulingRegionID == SchedulingRegionID;
784 }
785
786 /// Marks an instruction as scheduled and puts all dependent ready
787 /// instructions into the ready-list.
788 template <typename ReadyListType>
789 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
790 SD->IsScheduled = true;
791 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
792
793 ScheduleData *BundleMember = SD;
794 while (BundleMember) {
795 // Handle the def-use chain dependencies.
796 for (Use &U : BundleMember->Inst->operands()) {
797 ScheduleData *OpDef = getScheduleData(U.get());
798 if (OpDef && OpDef->hasValidDependencies() &&
799 OpDef->incrementUnscheduledDeps(-1) == 0) {
800 // There are no more unscheduled dependencies after decrementing,
801 // so we can put the dependent instruction into the ready list.
802 ScheduleData *DepBundle = OpDef->FirstInBundle;
803 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 804, __PRETTY_FUNCTION__))
804 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 804, __PRETTY_FUNCTION__))
;
805 ReadyList.insert(DepBundle);
806 DEBUG(dbgs() << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
807 }
808 }
809 // Handle the memory dependencies.
810 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
811 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
812 // There are no more unscheduled dependencies after decrementing,
813 // so we can put the dependent instruction into the ready list.
814 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
815 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 816, __PRETTY_FUNCTION__))
816 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 816, __PRETTY_FUNCTION__))
;
817 ReadyList.insert(DepBundle);
818 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
819 }
820 }
821 BundleMember = BundleMember->NextInBundle;
822 }
823 }
824
825 /// Put all instructions into the ReadyList which are ready for scheduling.
826 template <typename ReadyListType>
827 void initialFillReadyList(ReadyListType &ReadyList) {
828 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
829 ScheduleData *SD = getScheduleData(I);
830 if (SD->isSchedulingEntity() && SD->isReady()) {
831 ReadyList.insert(SD);
832 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
833 }
834 }
835 }
836
837 /// Checks if a bundle of instructions can be scheduled, i.e. has no
838 /// cyclic dependencies. This is only a dry-run, no instructions are
839 /// actually moved at this stage.
840 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP);
841
842 /// Un-bundles a group of instructions.
843 void cancelScheduling(ArrayRef<Value *> VL);
844
845 /// Extends the scheduling region so that V is inside the region.
846 /// \returns true if the region size is within the limit.
847 bool extendSchedulingRegion(Value *V);
848
849 /// Initialize the ScheduleData structures for new instructions in the
850 /// scheduling region.
851 void initScheduleData(Instruction *FromI, Instruction *ToI,
852 ScheduleData *PrevLoadStore,
853 ScheduleData *NextLoadStore);
854
855 /// Updates the dependency information of a bundle and of all instructions/
856 /// bundles which depend on the original bundle.
857 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
858 BoUpSLP *SLP);
859
860 /// Sets all instruction in the scheduling region to un-scheduled.
861 void resetSchedule();
862
863 BasicBlock *BB;
864
865 /// Simple memory allocation for ScheduleData.
866 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
867
868 /// The size of a ScheduleData array in ScheduleDataChunks.
869 int ChunkSize;
870
871 /// The allocator position in the current chunk, which is the last entry
872 /// of ScheduleDataChunks.
873 int ChunkPos;
874
875 /// Attaches ScheduleData to Instruction.
876 /// Note that the mapping survives during all vectorization iterations, i.e.
877 /// ScheduleData structures are recycled.
878 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
879
880 struct ReadyList : SmallVector<ScheduleData *, 8> {
881 void insert(ScheduleData *SD) { push_back(SD); }
882 };
883
884 /// The ready-list for scheduling (only used for the dry-run).
885 ReadyList ReadyInsts;
886
887 /// The first instruction of the scheduling region.
888 Instruction *ScheduleStart;
889
890 /// The first instruction _after_ the scheduling region.
891 Instruction *ScheduleEnd;
892
893 /// The first memory accessing instruction in the scheduling region
894 /// (can be null).
895 ScheduleData *FirstLoadStoreInRegion;
896
897 /// The last memory accessing instruction in the scheduling region
898 /// (can be null).
899 ScheduleData *LastLoadStoreInRegion;
900
901 /// The current size of the scheduling region.
902 int ScheduleRegionSize;
903
904 /// The maximum size allowed for the scheduling region.
905 int ScheduleRegionSizeLimit;
906
907 /// The ID of the scheduling region. For a new vectorization iteration this
908 /// is incremented which "removes" all ScheduleData from the region.
909 int SchedulingRegionID;
910 };
911
912 /// Attaches the BlockScheduling structures to basic blocks.
913 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
914
915 /// Performs the "real" scheduling. Done before vectorization is actually
916 /// performed in a basic block.
917 void scheduleBlock(BlockScheduling *BS);
918
919 /// List of users to ignore during scheduling and that don't need extracting.
920 ArrayRef<Value *> UserIgnoreList;
921
922 // Number of load bundles that contain consecutive loads.
923 int NumLoadsWantToKeepOrder;
924
925 // Number of load bundles that contain consecutive loads in reversed order.
926 int NumLoadsWantToChangeOrder;
927
928 // Analysis and block reference.
929 Function *F;
930 ScalarEvolution *SE;
931 TargetTransformInfo *TTI;
932 TargetLibraryInfo *TLI;
933 AliasAnalysis *AA;
934 LoopInfo *LI;
935 DominatorTree *DT;
936 AssumptionCache *AC;
937 DemandedBits *DB;
938 const DataLayout *DL;
939 OptimizationRemarkEmitter *ORE;
940
941 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
942 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
943 /// Instruction builder to construct the vectorized tree.
944 IRBuilder<> Builder;
945
946 /// A map of scalar integer values to the smallest bit width with which they
947 /// can legally be represented. The values map to (width, signed) pairs,
948 /// where "width" indicates the minimum bit width and "signed" is True if the
949 /// value must be signed-extended, rather than zero-extended, back to its
950 /// original width.
951 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
952};
953} // end namespace slpvectorizer
954
955template <> struct GraphTraits<BoUpSLP *> {
956 typedef BoUpSLP::TreeEntry TreeEntry;
957
958 /// NodeRef has to be a pointer per the GraphWriter.
959 typedef TreeEntry *NodeRef;
960
961 /// \brief Add the VectorizableTree to the index iterator to be able to return
962 /// TreeEntry pointers.
963 struct ChildIteratorType
964 : public iterator_adaptor_base<ChildIteratorType,
965 SmallVector<int, 1>::iterator> {
966
967 std::vector<TreeEntry> &VectorizableTree;
968
969 ChildIteratorType(SmallVector<int, 1>::iterator W,
970 std::vector<TreeEntry> &VT)
971 : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
972
973 NodeRef operator*() { return &VectorizableTree[*I]; }
974 };
975
976 static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
977
978 static ChildIteratorType child_begin(NodeRef N) {
979 return {N->UserTreeIndices.begin(), N->Container};
980 }
981 static ChildIteratorType child_end(NodeRef N) {
982 return {N->UserTreeIndices.end(), N->Container};
983 }
984
985 /// For the node iterator we just need to turn the TreeEntry iterator into a
986 /// TreeEntry* iterator so that it dereferences to NodeRef.
987 typedef pointer_iterator<std::vector<TreeEntry>::iterator> nodes_iterator;
988
989 static nodes_iterator nodes_begin(BoUpSLP *R) {
990 return nodes_iterator(R->VectorizableTree.begin());
991 }
992 static nodes_iterator nodes_end(BoUpSLP *R) {
993 return nodes_iterator(R->VectorizableTree.end());
994 }
995
996 static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
997};
998
999template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1000 typedef BoUpSLP::TreeEntry TreeEntry;
1001
1002 DOTGraphTraits(bool isSimple = false) : DefaultDOTGraphTraits(isSimple) {}
1003
1004 std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1005 std::string Str;
1006 raw_string_ostream OS(Str);
1007 if (isSplat(Entry->Scalars)) {
1008 OS << "<splat> " << *Entry->Scalars[0];
1009 return Str;
1010 }
1011 for (auto V : Entry->Scalars) {
1012 OS << *V;
1013 if (std::any_of(
1014 R->ExternalUses.begin(), R->ExternalUses.end(),
1015 [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1016 OS << " <extract>";
1017 OS << "\n";
1018 }
1019 return Str;
1020 }
1021
1022 static std::string getNodeAttributes(const TreeEntry *Entry,
1023 const BoUpSLP *) {
1024 if (Entry->NeedToGather)
1025 return "color=red";
1026 return "";
1027 }
1028};
1029
1030} // end namespace llvm
1031
1032void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1033 ArrayRef<Value *> UserIgnoreLst) {
1034 ExtraValueToDebugLocsMap ExternallyUsedValues;
1035 buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1036}
1037void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1038 ExtraValueToDebugLocsMap &ExternallyUsedValues,
1039 ArrayRef<Value *> UserIgnoreLst) {
1040 deleteTree();
1041 UserIgnoreList = UserIgnoreLst;
1042 if (!allSameType(Roots))
1043 return;
1044 buildTree_rec(Roots, 0, -1);
1045
1046 // Collect the values that we need to extract from the tree.
1047 for (TreeEntry &EIdx : VectorizableTree) {
1048 TreeEntry *Entry = &EIdx;
1049
1050 // For each lane:
1051 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1052 Value *Scalar = Entry->Scalars[Lane];
1053
1054 // No need to handle users of gathered values.
1055 if (Entry->NeedToGather)
1056 continue;
1057
1058 // Check if the scalar is externally used as an extra arg.
1059 auto ExtI = ExternallyUsedValues.find(Scalar);
1060 if (ExtI != ExternallyUsedValues.end()) {
1061 DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
1062 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract: Extra arg from lane "
<< Lane << " from " << *Scalar << ".\n"
; } } while (false)
;
1063 ExternalUses.emplace_back(Scalar, nullptr, Lane);
1064 continue;
1065 }
1066 for (User *U : Scalar->users()) {
1067 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
1068
1069 Instruction *UserInst = dyn_cast<Instruction>(U);
1070 if (!UserInst)
1071 continue;
1072
1073 // Skip in-tree scalars that become vectors
1074 if (ScalarToTreeEntry.count(U)) {
1075 int Idx = ScalarToTreeEntry[U];
1076 TreeEntry *UseEntry = &VectorizableTree[Idx];
1077 Value *UseScalar = UseEntry->Scalars[0];
1078 // Some in-tree scalars will remain as scalar in vectorized
1079 // instructions. If that is the case, the one in Lane 0 will
1080 // be used.
1081 if (UseScalar != U ||
1082 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1083 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
1084 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
1085 assert(!VectorizableTree[Idx].NeedToGather && "Bad state")((!VectorizableTree[Idx].NeedToGather && "Bad state")
? static_cast<void> (0) : __assert_fail ("!VectorizableTree[Idx].NeedToGather && \"Bad state\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1085, __PRETTY_FUNCTION__))
;
1086 continue;
1087 }
1088 }
1089
1090 // Ignore users in the user ignore list.
1091 if (is_contained(UserIgnoreList, UserInst))
1092 continue;
1093
1094 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
1095 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
1096 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1097 }
1098 }
1099 }
1100}
1101
1102void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1103 int UserTreeIdx) {
1104 bool isAltShuffle = false;
1105 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(((allConstant(VL) || allSameType(VL)) && "Invalid types!"
) ? static_cast<void> (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1105, __PRETTY_FUNCTION__))
;
1106
1107 if (Depth == RecursionMaxDepth) {
1108 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
1109 newTreeEntry(VL, false, UserTreeIdx);
1110 return;
1111 }
1112
1113 // Don't handle vectors.
1114 if (VL[0]->getType()->isVectorTy()) {
1115 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
1116 newTreeEntry(VL, false, UserTreeIdx);
1117 return;
1118 }
1119
1120 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1121 if (SI->getValueOperand()->getType()->isVectorTy()) {
1122 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1123 newTreeEntry(VL, false, UserTreeIdx);
1124 return;
1125 }
1126 unsigned Opcode = getSameOpcode(VL);
1127
1128 // Check that this shuffle vector refers to the alternate
1129 // sequence of opcodes.
1130 if (Opcode == Instruction::ShuffleVector) {
1131 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
1132 unsigned Op = I0->getOpcode();
1133 if (Op != Instruction::ShuffleVector)
1134 isAltShuffle = true;
1135 }
1136
1137 // If all of the operands are identical or constant we have a simple solution.
1138 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !Opcode) {
1139 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1140 newTreeEntry(VL, false, UserTreeIdx);
1141 return;
1142 }
1143
1144 // We now know that this is a vector of instructions of the same type from
1145 // the same block.
1146
1147 // Don't vectorize ephemeral values.
1148 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1149 if (EphValues.count(VL[i])) {
1150 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1151 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1152 newTreeEntry(VL, false, UserTreeIdx);
1153 return;
1154 }
1155 }
1156
1157 // Check if this is a duplicate of another entry.
1158 if (ScalarToTreeEntry.count(VL[0])) {
1159 int Idx = ScalarToTreeEntry[VL[0]];
1160 TreeEntry *E = &VectorizableTree[Idx];
1161 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1162 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*VL[i] << ".\n"; } } while (false)
;
1163 if (E->Scalars[i] != VL[i]) {
1164 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1165 newTreeEntry(VL, false, UserTreeIdx);
1166 return;
1167 }
1168 }
1169 // Record the reuse of the tree node. FIXME, currently this is only used to
1170 // properly draw the graph rather than for the actual vectorization.
1171 E->UserTreeIndices.push_back(UserTreeIdx);
1172 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *VL[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*VL[0] << ".\n"; } } while (false)
;
1173 return;
1174 }
1175
1176 // Check that none of the instructions in the bundle are already in the tree.
1177 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1178 if (ScalarToTreeEntry.count(VL[i])) {
1179 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1180 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1181 newTreeEntry(VL, false, UserTreeIdx);
1182 return;
1183 }
1184 }
1185
1186 // If any of the scalars is marked as a value that needs to stay scalar then
1187 // we need to gather the scalars.
1188 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1189 if (MustGather.count(VL[i])) {
1190 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1191 newTreeEntry(VL, false, UserTreeIdx);
1192 return;
1193 }
1194 }
1195
1196 // Check that all of the users of the scalars that we want to vectorize are
1197 // schedulable.
1198 Instruction *VL0 = cast<Instruction>(VL[0]);
1199 BasicBlock *BB = cast<Instruction>(VL0)->getParent();
1200
1201 if (!DT->isReachableFromEntry(BB)) {
1202 // Don't go into unreachable blocks. They may contain instructions with
1203 // dependency cycles which confuse the final scheduling.
1204 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1205 newTreeEntry(VL, false, UserTreeIdx);
1206 return;
1207 }
1208
1209 // Check that every instructions appears once in this bundle.
1210 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1211 for (unsigned j = i+1; j < e; ++j)
1212 if (VL[i] == VL[j]) {
1213 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1214 newTreeEntry(VL, false, UserTreeIdx);
1215 return;
1216 }
1217
1218 auto &BSRef = BlocksSchedules[BB];
1219 if (!BSRef) {
1220 BSRef = llvm::make_unique<BlockScheduling>(BB);
1221 }
1222 BlockScheduling &BS = *BSRef.get();
1223
1224 if (!BS.tryScheduleBundle(VL, this)) {
1225 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1226 assert((!BS.getScheduleData(VL[0]) ||(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1228, __PRETTY_FUNCTION__))
1227 !BS.getScheduleData(VL[0])->isPartOfBundle()) &&(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1228, __PRETTY_FUNCTION__))
1228 "tryScheduleBundle should cancelScheduling on failure")(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1228, __PRETTY_FUNCTION__))
;
1229 newTreeEntry(VL, false, UserTreeIdx);
1230 return;
1231 }
1232 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1233
1234 switch (Opcode) {
1235 case Instruction::PHI: {
1236 PHINode *PH = dyn_cast<PHINode>(VL0);
1237
1238 // Check for terminator values (e.g. invoke).
1239 for (unsigned j = 0; j < VL.size(); ++j)
1240 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1241 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1242 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1243 if (Term) {
1244 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false)
;
1245 BS.cancelScheduling(VL);
1246 newTreeEntry(VL, false, UserTreeIdx);
1247 return;
1248 }
1249 }
1250
1251 newTreeEntry(VL, true, UserTreeIdx);
1252 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1253
1254 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1255 ValueList Operands;
1256 // Prepare the operand vector.
1257 for (Value *j : VL)
1258 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1259 PH->getIncomingBlock(i)));
1260
1261 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1262 }
1263 return;
1264 }
1265 case Instruction::ExtractValue:
1266 case Instruction::ExtractElement: {
1267 bool Reuse = canReuseExtract(VL, Opcode);
1268 if (Reuse) {
1269 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing extract sequence.\n"
; } } while (false)
;
1270 } else {
1271 BS.cancelScheduling(VL);
1272 }
1273 newTreeEntry(VL, Reuse, UserTreeIdx);
1274 return;
1275 }
1276 case Instruction::Load: {
1277 // Check that a vectorized load would load the same memory as a scalar
1278 // load.
1279 // For example we don't want vectorize loads that are smaller than 8 bit.
1280 // Even though we have a packed struct {<i2, i2, i2, i2>} LLVM treats
1281 // loading/storing it as an i8 struct. If we vectorize loads/stores from
1282 // such a struct we read/write packed bits disagreeing with the
1283 // unvectorized version.
1284 Type *ScalarTy = VL[0]->getType();
1285
1286 if (DL->getTypeSizeInBits(ScalarTy) !=
1287 DL->getTypeAllocSizeInBits(ScalarTy)) {
1288 BS.cancelScheduling(VL);
1289 newTreeEntry(VL, false, UserTreeIdx);
1290 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1291 return;
1292 }
1293
1294 // Make sure all loads in the bundle are simple - we can't vectorize
1295 // atomic or volatile loads.
1296 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1297 LoadInst *L = cast<LoadInst>(VL[i]);
1298 if (!L->isSimple()) {
1299 BS.cancelScheduling(VL);
1300 newTreeEntry(VL, false, UserTreeIdx);
1301 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1302 return;
1303 }
1304 }
1305
1306 // Check if the loads are consecutive, reversed, or neither.
1307 // TODO: What we really want is to sort the loads, but for now, check
1308 // the two likely directions.
1309 bool Consecutive = true;
1310 bool ReverseConsecutive = true;
1311 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1312 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1313 Consecutive = false;
1314 break;
1315 } else {
1316 ReverseConsecutive = false;
1317 }
1318 }
1319
1320 if (Consecutive) {
1321 ++NumLoadsWantToKeepOrder;
1322 newTreeEntry(VL, true, UserTreeIdx);
1323 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1324 return;
1325 }
1326
1327 // If none of the load pairs were consecutive when checked in order,
1328 // check the reverse order.
1329 if (ReverseConsecutive)
1330 for (unsigned i = VL.size() - 1; i > 0; --i)
1331 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1332 ReverseConsecutive = false;
1333 break;
1334 }
1335
1336 BS.cancelScheduling(VL);
1337 newTreeEntry(VL, false, UserTreeIdx);
1338
1339 if (ReverseConsecutive) {
1340 ++NumLoadsWantToChangeOrder;
1341 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering reversed loads.\n"
; } } while (false)
;
1342 } else {
1343 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1344 }
1345 return;
1346 }
1347 case Instruction::ZExt:
1348 case Instruction::SExt:
1349 case Instruction::FPToUI:
1350 case Instruction::FPToSI:
1351 case Instruction::FPExt:
1352 case Instruction::PtrToInt:
1353 case Instruction::IntToPtr:
1354 case Instruction::SIToFP:
1355 case Instruction::UIToFP:
1356 case Instruction::Trunc:
1357 case Instruction::FPTrunc:
1358 case Instruction::BitCast: {
1359 Type *SrcTy = VL0->getOperand(0)->getType();
1360 for (unsigned i = 0; i < VL.size(); ++i) {
1361 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1362 if (Ty != SrcTy || !isValidElementType(Ty)) {
1363 BS.cancelScheduling(VL);
1364 newTreeEntry(VL, false, UserTreeIdx);
1365 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1366 return;
1367 }
1368 }
1369 newTreeEntry(VL, true, UserTreeIdx);
1370 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1371
1372 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1373 ValueList Operands;
1374 // Prepare the operand vector.
1375 for (Value *j : VL)
1376 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1377
1378 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1379 }
1380 return;
1381 }
1382 case Instruction::ICmp:
1383 case Instruction::FCmp: {
1384 // Check that all of the compares have the same predicate.
1385 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1386 Type *ComparedTy = cast<Instruction>(VL[0])->getOperand(0)->getType();
1387 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1388 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1389 if (Cmp->getPredicate() != P0 ||
1390 Cmp->getOperand(0)->getType() != ComparedTy) {
1391 BS.cancelScheduling(VL);
1392 newTreeEntry(VL, false, UserTreeIdx);
1393 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1394 return;
1395 }
1396 }
1397
1398 newTreeEntry(VL, true, UserTreeIdx);
1399 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1400
1401 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1402 ValueList Operands;
1403 // Prepare the operand vector.
1404 for (Value *j : VL)
1405 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1406
1407 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1408 }
1409 return;
1410 }
1411 case Instruction::Select:
1412 case Instruction::Add:
1413 case Instruction::FAdd:
1414 case Instruction::Sub:
1415 case Instruction::FSub:
1416 case Instruction::Mul:
1417 case Instruction::FMul:
1418 case Instruction::UDiv:
1419 case Instruction::SDiv:
1420 case Instruction::FDiv:
1421 case Instruction::URem:
1422 case Instruction::SRem:
1423 case Instruction::FRem:
1424 case Instruction::Shl:
1425 case Instruction::LShr:
1426 case Instruction::AShr:
1427 case Instruction::And:
1428 case Instruction::Or:
1429 case Instruction::Xor: {
1430 newTreeEntry(VL, true, UserTreeIdx);
1431 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1432
1433 // Sort operands of the instructions so that each side is more likely to
1434 // have the same opcode.
1435 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1436 ValueList Left, Right;
1437 reorderInputsAccordingToOpcode(VL, Left, Right);
1438 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1439 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1440 return;
1441 }
1442
1443 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1444 ValueList Operands;
1445 // Prepare the operand vector.
1446 for (Value *j : VL)
1447 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1448
1449 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1450 }
1451 return;
1452 }
1453 case Instruction::GetElementPtr: {
1454 // We don't combine GEPs with complicated (nested) indexing.
1455 for (unsigned j = 0; j < VL.size(); ++j) {
1456 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1457 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1458 BS.cancelScheduling(VL);
1459 newTreeEntry(VL, false, UserTreeIdx);
1460 return;
1461 }
1462 }
1463
1464 // We can't combine several GEPs into one vector if they operate on
1465 // different types.
1466 Type *Ty0 = cast<Instruction>(VL0)->getOperand(0)->getType();
1467 for (unsigned j = 0; j < VL.size(); ++j) {
1468 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1469 if (Ty0 != CurTy) {
1470 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1471 BS.cancelScheduling(VL);
1472 newTreeEntry(VL, false, UserTreeIdx);
1473 return;
1474 }
1475 }
1476
1477 // We don't combine GEPs with non-constant indexes.
1478 for (unsigned j = 0; j < VL.size(); ++j) {
1479 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1480 if (!isa<ConstantInt>(Op)) {
1481 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1482 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1483 BS.cancelScheduling(VL);
1484 newTreeEntry(VL, false, UserTreeIdx);
1485 return;
1486 }
1487 }
1488
1489 newTreeEntry(VL, true, UserTreeIdx);
1490 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1491 for (unsigned i = 0, e = 2; i < e; ++i) {
1492 ValueList Operands;
1493 // Prepare the operand vector.
1494 for (Value *j : VL)
1495 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1496
1497 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1498 }
1499 return;
1500 }
1501 case Instruction::Store: {
1502 // Check if the stores are consecutive or of we need to swizzle them.
1503 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1504 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1505 BS.cancelScheduling(VL);
1506 newTreeEntry(VL, false, UserTreeIdx);
1507 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1508 return;
1509 }
1510
1511 newTreeEntry(VL, true, UserTreeIdx);
1512 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1513
1514 ValueList Operands;
1515 for (Value *j : VL)
1516 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1517
1518 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1519 return;
1520 }
1521 case Instruction::Call: {
1522 // Check if the calls are all to the same vectorizable intrinsic.
1523 CallInst *CI = cast<CallInst>(VL[0]);
1524 // Check if this is an Intrinsic call or something that can be
1525 // represented by an intrinsic call
1526 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1527 if (!isTriviallyVectorizable(ID)) {
1528 BS.cancelScheduling(VL);
1529 newTreeEntry(VL, false, UserTreeIdx);
1530 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1531 return;
1532 }
1533 Function *Int = CI->getCalledFunction();
1534 Value *A1I = nullptr;
1535 if (hasVectorInstrinsicScalarOpd(ID, 1))
1536 A1I = CI->getArgOperand(1);
1537 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1538 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1539 if (!CI2 || CI2->getCalledFunction() != Int ||
1540 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1541 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1542 BS.cancelScheduling(VL);
1543 newTreeEntry(VL, false, UserTreeIdx);
1544 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1545 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1546 return;
1547 }
1548 // ctlz,cttz and powi are special intrinsics whose second argument
1549 // should be same in order for them to be vectorized.
1550 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1551 Value *A1J = CI2->getArgOperand(1);
1552 if (A1I != A1J) {
1553 BS.cancelScheduling(VL);
1554 newTreeEntry(VL, false, UserTreeIdx);
1555 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1556 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1557 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
;
1558 return;
1559 }
1560 }
1561 // Verify that the bundle operands are identical between the two calls.
1562 if (CI->hasOperandBundles() &&
1563 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1564 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1565 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1566 BS.cancelScheduling(VL);
1567 newTreeEntry(VL, false, UserTreeIdx);
1568 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1569 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1570 return;
1571 }
1572 }
1573
1574 newTreeEntry(VL, true, UserTreeIdx);
1575 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1576 ValueList Operands;
1577 // Prepare the operand vector.
1578 for (Value *j : VL) {
1579 CallInst *CI2 = dyn_cast<CallInst>(j);
1580 Operands.push_back(CI2->getArgOperand(i));
1581 }
1582 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1583 }
1584 return;
1585 }
1586 case Instruction::ShuffleVector: {
1587 // If this is not an alternate sequence of opcode like add-sub
1588 // then do not vectorize this instruction.
1589 if (!isAltShuffle) {
1590 BS.cancelScheduling(VL);
1591 newTreeEntry(VL, false, UserTreeIdx);
1592 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
1593 return;
1594 }
1595 newTreeEntry(VL, true, UserTreeIdx);
1596 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
1597
1598 // Reorder operands if reordering would enable vectorization.
1599 if (isa<BinaryOperator>(VL0)) {
1600 ValueList Left, Right;
1601 reorderAltShuffleOperands(VL, Left, Right);
1602 buildTree_rec(Left, Depth + 1, UserTreeIdx);
1603 buildTree_rec(Right, Depth + 1, UserTreeIdx);
1604 return;
1605 }
1606
1607 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1608 ValueList Operands;
1609 // Prepare the operand vector.
1610 for (Value *j : VL)
1611 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1612
1613 buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1614 }
1615 return;
1616 }
1617 default:
1618 BS.cancelScheduling(VL);
1619 newTreeEntry(VL, false, UserTreeIdx);
1620 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
1621 return;
1622 }
1623}
1624
1625unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1626 unsigned N;
1627 Type *EltTy;
1628 auto *ST = dyn_cast<StructType>(T);
1629 if (ST) {
1630 N = ST->getNumElements();
1631 EltTy = *ST->element_begin();
1632 } else {
1633 N = cast<ArrayType>(T)->getNumElements();
1634 EltTy = cast<ArrayType>(T)->getElementType();
1635 }
1636 if (!isValidElementType(EltTy))
1637 return 0;
1638 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1639 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1640 return 0;
1641 if (ST) {
1642 // Check that struct is homogeneous.
1643 for (const auto *Ty : ST->elements())
1644 if (Ty != EltTy)
1645 return 0;
1646 }
1647 return N;
1648}
1649
1650bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const {
1651 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1652, __PRETTY_FUNCTION__))
1652 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1652, __PRETTY_FUNCTION__))
;
1653 assert(Opcode == getSameOpcode(VL) && "Invalid opcode")((Opcode == getSameOpcode(VL) && "Invalid opcode") ? static_cast
<void> (0) : __assert_fail ("Opcode == getSameOpcode(VL) && \"Invalid opcode\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1653, __PRETTY_FUNCTION__))
;
1654 // Check if all of the extracts come from the same vector and from the
1655 // correct offset.
1656 Value *VL0 = VL[0];
1657 Instruction *E0 = cast<Instruction>(VL0);
1658 Value *Vec = E0->getOperand(0);
1659
1660 // We have to extract from a vector/aggregate with the same number of elements.
1661 unsigned NElts;
1662 if (Opcode == Instruction::ExtractValue) {
1663 const DataLayout &DL = E0->getModule()->getDataLayout();
1664 NElts = canMapToVector(Vec->getType(), DL);
1665 if (!NElts)
1666 return false;
1667 // Check if load can be rewritten as load of vector.
1668 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1669 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1670 return false;
1671 } else {
1672 NElts = Vec->getType()->getVectorNumElements();
1673 }
1674
1675 if (NElts != VL.size())
1676 return false;
1677
1678 // Check that all of the indices extract from the correct offset.
1679 if (!matchExtractIndex(E0, 0, Opcode))
1680 return false;
1681
1682 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1683 Instruction *E = cast<Instruction>(VL[i]);
1684 if (!matchExtractIndex(E, i, Opcode))
1685 return false;
1686 if (E->getOperand(0) != Vec)
1687 return false;
1688 }
1689
1690 return true;
1691}
1692
1693int BoUpSLP::getEntryCost(TreeEntry *E) {
1694 ArrayRef<Value*> VL = E->Scalars;
1695
1696 Type *ScalarTy = VL[0]->getType();
1697 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1698 ScalarTy = SI->getValueOperand()->getType();
1699 else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
1700 ScalarTy = CI->getOperand(0)->getType();
1701 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1702
1703 // If we have computed a smaller type for the expression, update VecTy so
1704 // that the costs will be accurate.
1705 if (MinBWs.count(VL[0]))
1706 VecTy = VectorType::get(
1707 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1708
1709 if (E->NeedToGather) {
1710 if (allConstant(VL))
1711 return 0;
1712 if (isSplat(VL)) {
1713 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1714 }
1715 return getGatherCost(E->Scalars);
1716 }
1717 unsigned Opcode = getSameOpcode(VL);
1718 assert(Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")((Opcode && allSameType(VL) && allSameBlock(VL
) && "Invalid VL") ? static_cast<void> (0) : __assert_fail
("Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1718, __PRETTY_FUNCTION__))
;
1719 Instruction *VL0 = cast<Instruction>(VL[0]);
1720 switch (Opcode) {
1721 case Instruction::PHI: {
1722 return 0;
1723 }
1724 case Instruction::ExtractValue:
1725 case Instruction::ExtractElement: {
1726 if (canReuseExtract(VL, Opcode)) {
1727 int DeadCost = 0;
1728 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
1729 Instruction *E = cast<Instruction>(VL[i]);
1730 // If all users are going to be vectorized, instruction can be
1731 // considered as dead.
1732 // The same, if have only one user, it will be vectorized for sure.
1733 if (E->hasOneUse() ||
1734 std::all_of(E->user_begin(), E->user_end(), [this](User *U) {
1735 return ScalarToTreeEntry.count(U) > 0;
1736 }))
1737 // Take credit for instruction that will become dead.
1738 DeadCost +=
1739 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
1740 }
1741 return -DeadCost;
1742 }
1743 return getGatherCost(VecTy);
1744 }
1745 case Instruction::ZExt:
1746 case Instruction::SExt:
1747 case Instruction::FPToUI:
1748 case Instruction::FPToSI:
1749 case Instruction::FPExt:
1750 case Instruction::PtrToInt:
1751 case Instruction::IntToPtr:
1752 case Instruction::SIToFP:
1753 case Instruction::UIToFP:
1754 case Instruction::Trunc:
1755 case Instruction::FPTrunc:
1756 case Instruction::BitCast: {
1757 Type *SrcTy = VL0->getOperand(0)->getType();
1758
1759 // Calculate the cost of this instruction.
1760 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
1761 VL0->getType(), SrcTy, VL0);
1762
1763 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
1764 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
1765 return VecCost - ScalarCost;
1766 }
1767 case Instruction::FCmp:
1768 case Instruction::ICmp:
1769 case Instruction::Select: {
1770 // Calculate the cost of this instruction.
1771 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
1772 int ScalarCost = VecTy->getNumElements() *
1773 TTI->getCmpSelInstrCost(Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
1774 int VecCost = TTI->getCmpSelInstrCost(Opcode, VecTy, MaskTy, VL0);
1775 return VecCost - ScalarCost;
1776 }
1777 case Instruction::Add:
1778 case Instruction::FAdd:
1779 case Instruction::Sub:
1780 case Instruction::FSub:
1781 case Instruction::Mul:
1782 case Instruction::FMul:
1783 case Instruction::UDiv:
1784 case Instruction::SDiv:
1785 case Instruction::FDiv:
1786 case Instruction::URem:
1787 case Instruction::SRem:
1788 case Instruction::FRem:
1789 case Instruction::Shl:
1790 case Instruction::LShr:
1791 case Instruction::AShr:
1792 case Instruction::And:
1793 case Instruction::Or:
1794 case Instruction::Xor: {
1795 // Certain instructions can be cheaper to vectorize if they have a
1796 // constant second vector operand.
1797 TargetTransformInfo::OperandValueKind Op1VK =
1798 TargetTransformInfo::OK_AnyValue;
1799 TargetTransformInfo::OperandValueKind Op2VK =
1800 TargetTransformInfo::OK_UniformConstantValue;
1801 TargetTransformInfo::OperandValueProperties Op1VP =
1802 TargetTransformInfo::OP_None;
1803 TargetTransformInfo::OperandValueProperties Op2VP =
1804 TargetTransformInfo::OP_None;
1805
1806 // If all operands are exactly the same ConstantInt then set the
1807 // operand kind to OK_UniformConstantValue.
1808 // If instead not all operands are constants, then set the operand kind
1809 // to OK_AnyValue. If all operands are constants but not the same,
1810 // then set the operand kind to OK_NonUniformConstantValue.
1811 ConstantInt *CInt = nullptr;
1812 for (unsigned i = 0; i < VL.size(); ++i) {
1813 const Instruction *I = cast<Instruction>(VL[i]);
1814 if (!isa<ConstantInt>(I->getOperand(1))) {
1815 Op2VK = TargetTransformInfo::OK_AnyValue;
1816 break;
1817 }
1818 if (i == 0) {
1819 CInt = cast<ConstantInt>(I->getOperand(1));
1820 continue;
1821 }
1822 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
1823 CInt != cast<ConstantInt>(I->getOperand(1)))
1824 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
1825 }
1826 // FIXME: Currently cost of model modification for division by power of
1827 // 2 is handled for X86 and AArch64. Add support for other targets.
1828 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
1829 CInt->getValue().isPowerOf2())
1830 Op2VP = TargetTransformInfo::OP_PowerOf2;
1831
1832 SmallVector<const Value *, 4> Operands(VL0->operand_values());
1833 int ScalarCost =
1834 VecTy->getNumElements() *
1835 TTI->getArithmeticInstrCost(Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
1836 Op2VP, Operands);
1837 int VecCost = TTI->getArithmeticInstrCost(Opcode, VecTy, Op1VK, Op2VK,
1838 Op1VP, Op2VP, Operands);
1839 return VecCost - ScalarCost;
1840 }
1841 case Instruction::GetElementPtr: {
1842 TargetTransformInfo::OperandValueKind Op1VK =
1843 TargetTransformInfo::OK_AnyValue;
1844 TargetTransformInfo::OperandValueKind Op2VK =
1845 TargetTransformInfo::OK_UniformConstantValue;
1846
1847 int ScalarCost =
1848 VecTy->getNumElements() *
1849 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
1850 int VecCost =
1851 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
1852
1853 return VecCost - ScalarCost;
1854 }
1855 case Instruction::Load: {
1856 // Cost of wide load - cost of scalar loads.
1857 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
1858 int ScalarLdCost = VecTy->getNumElements() *
1859 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
1860 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
1861 VecTy, alignment, 0, VL0);
1862 return VecLdCost - ScalarLdCost;
1863 }
1864 case Instruction::Store: {
1865 // We know that we can merge the stores. Calculate the cost.
1866 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
1867 int ScalarStCost = VecTy->getNumElements() *
1868 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
1869 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
1870 VecTy, alignment, 0, VL0);
1871 return VecStCost - ScalarStCost;
1872 }
1873 case Instruction::Call: {
1874 CallInst *CI = cast<CallInst>(VL0);
1875 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1876
1877 // Calculate the cost of the scalar and vector calls.
1878 SmallVector<Type*, 4> ScalarTys;
1879 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
1880 ScalarTys.push_back(CI->getArgOperand(op)->getType());
1881
1882 FastMathFlags FMF;
1883 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
1884 FMF = FPMO->getFastMathFlags();
1885
1886 int ScalarCallCost = VecTy->getNumElements() *
1887 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
1888
1889 SmallVector<Value *, 4> Args(CI->arg_operands());
1890 int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
1891 VecTy->getNumElements());
1892
1893 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
1894 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
1895 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
1896
1897 return VecCallCost - ScalarCallCost;
1898 }
1899 case Instruction::ShuffleVector: {
1900 TargetTransformInfo::OperandValueKind Op1VK =
1901 TargetTransformInfo::OK_AnyValue;
1902 TargetTransformInfo::OperandValueKind Op2VK =
1903 TargetTransformInfo::OK_AnyValue;
1904 int ScalarCost = 0;
1905 int VecCost = 0;
1906 for (Value *i : VL) {
1907 Instruction *I = cast<Instruction>(i);
1908 if (!I)
1909 break;
1910 ScalarCost +=
1911 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
1912 }
1913 // VecCost is equal to sum of the cost of creating 2 vectors
1914 // and the cost of creating shuffle.
1915 Instruction *I0 = cast<Instruction>(VL[0]);
1916 VecCost =
1917 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
1918 Instruction *I1 = cast<Instruction>(VL[1]);
1919 VecCost +=
1920 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
1921 VecCost +=
1922 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
1923 return VecCost - ScalarCost;
1924 }
1925 default:
1926 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1926)
;
1927 }
1928}
1929
1930bool BoUpSLP::isFullyVectorizableTinyTree() {
1931 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
1932 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
1933
1934 // We only handle trees of heights 1 and 2.
1935 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
1936 return true;
1937
1938 if (VectorizableTree.size() != 2)
1939 return false;
1940
1941 // Handle splat and all-constants stores.
1942 if (!VectorizableTree[0].NeedToGather &&
1943 (allConstant(VectorizableTree[1].Scalars) ||
1944 isSplat(VectorizableTree[1].Scalars)))
1945 return true;
1946
1947 // Gathering cost would be too much for tiny trees.
1948 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
1949 return false;
1950
1951 return true;
1952}
1953
1954bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
1955
1956 // We can vectorize the tree if its size is greater than or equal to the
1957 // minimum size specified by the MinTreeSize command line option.
1958 if (VectorizableTree.size() >= MinTreeSize)
1959 return false;
1960
1961 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
1962 // can vectorize it if we can prove it fully vectorizable.
1963 if (isFullyVectorizableTinyTree())
1964 return false;
1965
1966 assert(VectorizableTree.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1968, __PRETTY_FUNCTION__))
1967 ? ExternalUses.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1968, __PRETTY_FUNCTION__))
1968 : true && "We shouldn't have any external users")((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1968, __PRETTY_FUNCTION__))
;
1969
1970 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
1971 // vectorizable.
1972 return true;
1973}
1974
1975int BoUpSLP::getSpillCost() {
1976 // Walk from the bottom of the tree to the top, tracking which values are
1977 // live. When we see a call instruction that is not part of our tree,
1978 // query TTI to see if there is a cost to keeping values live over it
1979 // (for example, if spills and fills are required).
1980 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
1981 int Cost = 0;
1982
1983 SmallPtrSet<Instruction*, 4> LiveValues;
1984 Instruction *PrevInst = nullptr;
1985
1986 for (const auto &N : VectorizableTree) {
1987 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
1988 if (!Inst)
1989 continue;
1990
1991 if (!PrevInst) {
1992 PrevInst = Inst;
1993 continue;
1994 }
1995
1996 // Update LiveValues.
1997 LiveValues.erase(PrevInst);
1998 for (auto &J : PrevInst->operands()) {
1999 if (isa<Instruction>(&*J) && ScalarToTreeEntry.count(&*J))
2000 LiveValues.insert(cast<Instruction>(&*J));
2001 }
2002
2003 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2004 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2005 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2006 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2007 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2008 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
2009 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
;
2010
2011 // Now find the sequence of instructions between PrevInst and Inst.
2012 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
2013 PrevInstIt =
2014 PrevInst->getIterator().getReverse();
2015 while (InstIt != PrevInstIt) {
2016 if (PrevInstIt == PrevInst->getParent()->rend()) {
2017 PrevInstIt = Inst->getParent()->rbegin();
2018 continue;
2019 }
2020
2021 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2022 SmallVector<Type*, 4> V;
2023 for (auto *II : LiveValues)
2024 V.push_back(VectorType::get(II->getType(), BundleWidth));
2025 Cost += TTI->getCostOfKeepingLiveOverCall(V);
2026 }
2027
2028 ++PrevInstIt;
2029 }
2030
2031 PrevInst = Inst;
2032 }
2033
2034 return Cost;
2035}
2036
2037int BoUpSLP::getTreeCost() {
2038 int Cost = 0;
2039 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
2040 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
2041
2042 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2043
2044 for (TreeEntry &TE : VectorizableTree) {
2045 int C = getEntryCost(&TE);
2046 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
2047 << *TE.Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
2048 Cost += C;
2049 }
2050
2051 SmallSet<Value *, 16> ExtractCostCalculated;
2052 int ExtractCost = 0;
2053 for (ExternalUser &EU : ExternalUses) {
2054 // We only add extract cost once for the same scalar.
2055 if (!ExtractCostCalculated.insert(EU.Scalar).second)
2056 continue;
2057
2058 // Uses by ephemeral values are free (because the ephemeral value will be
2059 // removed prior to code generation, and so the extraction will be
2060 // removed as well).
2061 if (EphValues.count(EU.User))
2062 continue;
2063
2064 // If we plan to rewrite the tree in a smaller type, we will need to sign
2065 // extend the extracted value back to the original type. Here, we account
2066 // for the extract and the added cost of the sign extend if needed.
2067 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2068 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2069 if (MinBWs.count(ScalarRoot)) {
2070 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2071 auto Extend =
2072 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2073 VecTy = VectorType::get(MinTy, BundleWidth);
2074 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2075 VecTy, EU.Lane);
2076 } else {
2077 ExtractCost +=
2078 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2079 }
2080 }
2081
2082 int SpillCost = getSpillCost();
2083 Cost += SpillCost + ExtractCost;
2084
2085 std::string Str;
2086 {
2087 raw_string_ostream OS(Str);
2088 OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2089 << "SLP: Extract Cost = " << ExtractCost << ".\n"
2090 << "SLP: Total Cost = " << Cost << ".\n";
2091 }
2092 DEBUG(dbgs() << Str)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << Str; } } while (false)
;
2093
2094 if (ViewSLPTree)
2095 ViewGraph(this, "SLP" + F->getName(), false, Str);
2096
2097 return Cost;
2098}
2099
2100int BoUpSLP::getGatherCost(Type *Ty) {
2101 int Cost = 0;
2102 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2103 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2104 return Cost;
2105}
2106
2107int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2108 // Find the type of the operands in VL.
2109 Type *ScalarTy = VL[0]->getType();
2110 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2111 ScalarTy = SI->getValueOperand()->getType();
2112 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2113 // Find the cost of inserting/extracting values from the vector.
2114 return getGatherCost(VecTy);
2115}
2116
2117// Reorder commutative operations in alternate shuffle if the resulting vectors
2118// are consecutive loads. This would allow us to vectorize the tree.
2119// If we have something like-
2120// load a[0] - load b[0]
2121// load b[1] + load a[1]
2122// load a[2] - load b[2]
2123// load a[3] + load b[3]
2124// Reordering the second load b[1] load a[1] would allow us to vectorize this
2125// code.
2126void BoUpSLP::reorderAltShuffleOperands(ArrayRef<Value *> VL,
2127 SmallVectorImpl<Value *> &Left,
2128 SmallVectorImpl<Value *> &Right) {
2129 // Push left and right operands of binary operation into Left and Right
2130 for (Value *i : VL) {
2131 Left.push_back(cast<Instruction>(i)->getOperand(0));
2132 Right.push_back(cast<Instruction>(i)->getOperand(1));
2133 }
2134
2135 // Reorder if we have a commutative operation and consecutive access
2136 // are on either side of the alternate instructions.
2137 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2138 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2139 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2140 Instruction *VL1 = cast<Instruction>(VL[j]);
2141 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2142 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2143 std::swap(Left[j], Right[j]);
2144 continue;
2145 } else if (VL2->isCommutative() &&
2146 isConsecutiveAccess(L, L1, *DL, *SE)) {
2147 std::swap(Left[j + 1], Right[j + 1]);
2148 continue;
2149 }
2150 // else unchanged
2151 }
2152 }
2153 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2154 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2155 Instruction *VL1 = cast<Instruction>(VL[j]);
2156 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2157 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2158 std::swap(Left[j], Right[j]);
2159 continue;
2160 } else if (VL2->isCommutative() &&
2161 isConsecutiveAccess(L, L1, *DL, *SE)) {
2162 std::swap(Left[j + 1], Right[j + 1]);
2163 continue;
2164 }
2165 // else unchanged
2166 }
2167 }
2168 }
2169}
2170
2171// Return true if I should be commuted before adding it's left and right
2172// operands to the arrays Left and Right.
2173//
2174// The vectorizer is trying to either have all elements one side being
2175// instruction with the same opcode to enable further vectorization, or having
2176// a splat to lower the vectorizing cost.
2177static bool shouldReorderOperands(int i, Instruction &I,
2178 SmallVectorImpl<Value *> &Left,
2179 SmallVectorImpl<Value *> &Right,
2180 bool AllSameOpcodeLeft,
2181 bool AllSameOpcodeRight, bool SplatLeft,
2182 bool SplatRight) {
2183 Value *VLeft = I.getOperand(0);
2184 Value *VRight = I.getOperand(1);
2185 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2186 if (SplatRight) {
2187 if (VRight == Right[i - 1])
2188 // Preserve SplatRight
2189 return false;
2190 if (VLeft == Right[i - 1]) {
2191 // Commuting would preserve SplatRight, but we don't want to break
2192 // SplatLeft either, i.e. preserve the original order if possible.
2193 // (FIXME: why do we care?)
2194 if (SplatLeft && VLeft == Left[i - 1])
2195 return false;
2196 return true;
2197 }
2198 }
2199 // Symmetrically handle Right side.
2200 if (SplatLeft) {
2201 if (VLeft == Left[i - 1])
2202 // Preserve SplatLeft
2203 return false;
2204 if (VRight == Left[i - 1])
2205 return true;
2206 }
2207
2208 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2209 Instruction *IRight = dyn_cast<Instruction>(VRight);
2210
2211 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2212 // it and not the right, in this case we want to commute.
2213 if (AllSameOpcodeRight) {
2214 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2215 if (IRight && RightPrevOpcode == IRight->getOpcode())
2216 // Do not commute, a match on the right preserves AllSameOpcodeRight
2217 return false;
2218 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2219 // We have a match and may want to commute, but first check if there is
2220 // not also a match on the existing operands on the Left to preserve
2221 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2222 // (FIXME: why do we care?)
2223 if (AllSameOpcodeLeft && ILeft &&
2224 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2225 return false;
2226 return true;
2227 }
2228 }
2229 // Symmetrically handle Left side.
2230 if (AllSameOpcodeLeft) {
2231 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2232 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2233 return false;
2234 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2235 return true;
2236 }
2237 return false;
2238}
2239
2240void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
2241 SmallVectorImpl<Value *> &Left,
2242 SmallVectorImpl<Value *> &Right) {
2243
2244 if (VL.size()) {
2245 // Peel the first iteration out of the loop since there's nothing
2246 // interesting to do anyway and it simplifies the checks in the loop.
2247 auto VLeft = cast<Instruction>(VL[0])->getOperand(0);
2248 auto VRight = cast<Instruction>(VL[0])->getOperand(1);
2249 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2250 // Favor having instruction to the right. FIXME: why?
2251 std::swap(VLeft, VRight);
2252 Left.push_back(VLeft);
2253 Right.push_back(VRight);
2254 }
2255
2256 // Keep track if we have instructions with all the same opcode on one side.
2257 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2258 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2259 // Keep track if we have one side with all the same value (broadcast).
2260 bool SplatLeft = true;
2261 bool SplatRight = true;
2262
2263 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2264 Instruction *I = cast<Instruction>(VL[i]);
2265 assert(I->isCommutative() && "Can only process commutative instruction")((I->isCommutative() && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("I->isCommutative() && \"Can only process commutative instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2265, __PRETTY_FUNCTION__))
;
2266 // Commute to favor either a splat or maximizing having the same opcodes on
2267 // one side.
2268 if (shouldReorderOperands(i, *I, Left, Right, AllSameOpcodeLeft,
2269 AllSameOpcodeRight, SplatLeft, SplatRight)) {
2270 Left.push_back(I->getOperand(1));
2271 Right.push_back(I->getOperand(0));
2272 } else {
2273 Left.push_back(I->getOperand(0));
2274 Right.push_back(I->getOperand(1));
2275 }
2276 // Update Splat* and AllSameOpcode* after the insertion.
2277 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2278 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2279 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2280 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2281 cast<Instruction>(Left[i])->getOpcode());
2282 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2283 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2284 cast<Instruction>(Right[i])->getOpcode());
2285 }
2286
2287 // If one operand end up being broadcast, return this operand order.
2288 if (SplatRight || SplatLeft)
2289 return;
2290
2291 // Finally check if we can get longer vectorizable chain by reordering
2292 // without breaking the good operand order detected above.
2293 // E.g. If we have something like-
2294 // load a[0] load b[0]
2295 // load b[1] load a[1]
2296 // load a[2] load b[2]
2297 // load a[3] load b[3]
2298 // Reordering the second load b[1] load a[1] would allow us to vectorize
2299 // this code and we still retain AllSameOpcode property.
2300 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2301 // such as-
2302 // add a[0],c[0] load b[0]
2303 // add a[1],c[2] load b[1]
2304 // b[2] load b[2]
2305 // add a[3],c[3] load b[3]
2306 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2307 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2308 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2309 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2310 std::swap(Left[j + 1], Right[j + 1]);
2311 continue;
2312 }
2313 }
2314 }
2315 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2316 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2317 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2318 std::swap(Left[j + 1], Right[j + 1]);
2319 continue;
2320 }
2321 }
2322 }
2323 // else unchanged
2324 }
2325}
2326
2327void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL) {
2328
2329 // Get the basic block this bundle is in. All instructions in the bundle
2330 // should be in this block.
2331 auto *Front = cast<Instruction>(VL.front());
2332 auto *BB = Front->getParent();
2333 assert(all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool {((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2335, __PRETTY_FUNCTION__))
2334 return cast<Instruction>(V)->getParent() == BB;((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2335, __PRETTY_FUNCTION__))
2335 }))((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2335, __PRETTY_FUNCTION__))
;
2336
2337 // The last instruction in the bundle in program order.
2338 Instruction *LastInst = nullptr;
2339
2340 // Find the last instruction. The common case should be that BB has been
2341 // scheduled, and the last instruction is VL.back(). So we start with
2342 // VL.back() and iterate over schedule data until we reach the end of the
2343 // bundle. The end of the bundle is marked by null ScheduleData.
2344 if (BlocksSchedules.count(BB)) {
2345 auto *Bundle = BlocksSchedules[BB]->getScheduleData(VL.back());
2346 if (Bundle && Bundle->isPartOfBundle())
2347 for (; Bundle; Bundle = Bundle->NextInBundle)
2348 LastInst = Bundle->Inst;
2349 }
2350
2351 // LastInst can still be null at this point if there's either not an entry
2352 // for BB in BlocksSchedules or there's no ScheduleData available for
2353 // VL.back(). This can be the case if buildTree_rec aborts for various
2354 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2355 // size is reached, etc.). ScheduleData is initialized in the scheduling
2356 // "dry-run".
2357 //
2358 // If this happens, we can still find the last instruction by brute force. We
2359 // iterate forwards from Front (inclusive) until we either see all
2360 // instructions in the bundle or reach the end of the block. If Front is the
2361 // last instruction in program order, LastInst will be set to Front, and we
2362 // will visit all the remaining instructions in the block.
2363 //
2364 // One of the reasons we exit early from buildTree_rec is to place an upper
2365 // bound on compile-time. Thus, taking an additional compile-time hit here is
2366 // not ideal. However, this should be exceedingly rare since it requires that
2367 // we both exit early from buildTree_rec and that the bundle be out-of-order
2368 // (causing us to iterate all the way to the end of the block).
2369 if (!LastInst) {
2370 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2371 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2372 if (Bundle.erase(&I))
2373 LastInst = &I;
2374 if (Bundle.empty())
2375 break;
2376 }
2377 }
2378
2379 // Set the insertion point after the last instruction in the bundle. Set the
2380 // debug location to Front.
2381 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2382 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2383}
2384
2385Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2386 Value *Vec = UndefValue::get(Ty);
2387 // Generate the 'InsertElement' instruction.
2388 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2389 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2390 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2391 GatherSeq.insert(Insrt);
2392 CSEBlocks.insert(Insrt->getParent());
2393
2394 // Add to our 'need-to-extract' list.
2395 if (ScalarToTreeEntry.count(VL[i])) {
2396 int Idx = ScalarToTreeEntry[VL[i]];
2397 TreeEntry *E = &VectorizableTree[Idx];
2398 // Find which lane we need to extract.
2399 int FoundLane = -1;
2400 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2401 // Is this the lane of the scalar that we are looking for ?
2402 if (E->Scalars[Lane] == VL[i]) {
2403 FoundLane = Lane;
2404 break;
2405 }
2406 }
2407 assert(FoundLane >= 0 && "Could not find the correct lane")((FoundLane >= 0 && "Could not find the correct lane"
) ? static_cast<void> (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2407, __PRETTY_FUNCTION__))
;
2408 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2409 }
2410 }
2411 }
2412
2413 return Vec;
2414}
2415
2416Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL) const {
2417 SmallDenseMap<Value*, int>::const_iterator Entry
2418 = ScalarToTreeEntry.find(VL[0]);
2419 if (Entry != ScalarToTreeEntry.end()) {
2420 int Idx = Entry->second;
2421 const TreeEntry *En = &VectorizableTree[Idx];
2422 if (En->isSame(VL) && En->VectorizedValue)
2423 return En->VectorizedValue;
2424 }
2425 return nullptr;
2426}
2427
2428Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2429 if (ScalarToTreeEntry.count(VL[0])) {
2430 int Idx = ScalarToTreeEntry[VL[0]];
2431 TreeEntry *E = &VectorizableTree[Idx];
2432 if (E->isSame(VL))
2433 return vectorizeTree(E);
2434 }
2435
2436 Type *ScalarTy = VL[0]->getType();
2437 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2438 ScalarTy = SI->getValueOperand()->getType();
2439 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2440
2441 return Gather(VL, VecTy);
2442}
2443
2444Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2445 IRBuilder<>::InsertPointGuard Guard(Builder);
2446
2447 if (E->VectorizedValue) {
2448 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
2449 return E->VectorizedValue;
2450 }
2451
2452 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2453 Type *ScalarTy = VL0->getType();
2454 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2455 ScalarTy = SI->getValueOperand()->getType();
2456 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2457
2458 if (E->NeedToGather) {
2459 setInsertPointAfterBundle(E->Scalars);
2460 auto *V = Gather(E->Scalars, VecTy);
2461 E->VectorizedValue = V;
2462 return V;
2463 }
2464
2465 unsigned Opcode = getSameOpcode(E->Scalars);
2466
2467 switch (Opcode) {
2468 case Instruction::PHI: {
2469 PHINode *PH = dyn_cast<PHINode>(VL0);
2470 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2471 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2472 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2473 E->VectorizedValue = NewPhi;
2474
2475 // PHINodes may have multiple entries from the same block. We want to
2476 // visit every block once.
2477 SmallSet<BasicBlock*, 4> VisitedBBs;
2478
2479 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2480 ValueList Operands;
2481 BasicBlock *IBB = PH->getIncomingBlock(i);
2482
2483 if (!VisitedBBs.insert(IBB).second) {
2484 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2485 continue;
2486 }
2487
2488 // Prepare the operand vector.
2489 for (Value *V : E->Scalars)
2490 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2491
2492 Builder.SetInsertPoint(IBB->getTerminator());
2493 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2494 Value *Vec = vectorizeTree(Operands);
2495 NewPhi->addIncoming(Vec, IBB);
2496 }
2497
2498 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2499, __PRETTY_FUNCTION__))
2499 "Invalid number of incoming values")((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2499, __PRETTY_FUNCTION__))
;
2500 return NewPhi;
2501 }
2502
2503 case Instruction::ExtractElement: {
2504 if (canReuseExtract(E->Scalars, Instruction::ExtractElement)) {
2505 Value *V = VL0->getOperand(0);
2506 E->VectorizedValue = V;
2507 return V;
2508 }
2509 setInsertPointAfterBundle(E->Scalars);
2510 auto *V = Gather(E->Scalars, VecTy);
2511 E->VectorizedValue = V;
2512 return V;
2513 }
2514 case Instruction::ExtractValue: {
2515 if (canReuseExtract(E->Scalars, Instruction::ExtractValue)) {
2516 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2517 Builder.SetInsertPoint(LI);
2518 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2519 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2520 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2521 E->VectorizedValue = V;
2522 return propagateMetadata(V, E->Scalars);
2523 }
2524 setInsertPointAfterBundle(E->Scalars);
2525 auto *V = Gather(E->Scalars, VecTy);
2526 E->VectorizedValue = V;
2527 return V;
2528 }
2529 case Instruction::ZExt:
2530 case Instruction::SExt:
2531 case Instruction::FPToUI:
2532 case Instruction::FPToSI:
2533 case Instruction::FPExt:
2534 case Instruction::PtrToInt:
2535 case Instruction::IntToPtr:
2536 case Instruction::SIToFP:
2537 case Instruction::UIToFP:
2538 case Instruction::Trunc:
2539 case Instruction::FPTrunc:
2540 case Instruction::BitCast: {
2541 ValueList INVL;
2542 for (Value *V : E->Scalars)
2543 INVL.push_back(cast<Instruction>(V)->getOperand(0));
2544
2545 setInsertPointAfterBundle(E->Scalars);
2546
2547 Value *InVec = vectorizeTree(INVL);
2548
2549 if (Value *V = alreadyVectorized(E->Scalars))
2550 return V;
2551
2552 CastInst *CI = dyn_cast<CastInst>(VL0);
2553 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2554 E->VectorizedValue = V;
2555 ++NumVectorInstructions;
2556 return V;
2557 }
2558 case Instruction::FCmp:
2559 case Instruction::ICmp: {
2560 ValueList LHSV, RHSV;
2561 for (Value *V : E->Scalars) {
2562 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2563 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2564 }
2565
2566 setInsertPointAfterBundle(E->Scalars);
2567
2568 Value *L = vectorizeTree(LHSV);
2569 Value *R = vectorizeTree(RHSV);
2570
2571 if (Value *V = alreadyVectorized(E->Scalars))
2572 return V;
2573
2574 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2575 Value *V;
2576 if (Opcode == Instruction::FCmp)
2577 V = Builder.CreateFCmp(P0, L, R);
2578 else
2579 V = Builder.CreateICmp(P0, L, R);
2580
2581 E->VectorizedValue = V;
2582 propagateIRFlags(E->VectorizedValue, E->Scalars);
2583 ++NumVectorInstructions;
2584 return V;
2585 }
2586 case Instruction::Select: {
2587 ValueList TrueVec, FalseVec, CondVec;
2588 for (Value *V : E->Scalars) {
2589 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2590 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2591 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2592 }
2593
2594 setInsertPointAfterBundle(E->Scalars);
2595
2596 Value *Cond = vectorizeTree(CondVec);
2597 Value *True = vectorizeTree(TrueVec);
2598 Value *False = vectorizeTree(FalseVec);
2599
2600 if (Value *V = alreadyVectorized(E->Scalars))
2601 return V;
2602
2603 Value *V = Builder.CreateSelect(Cond, True, False);
2604 E->VectorizedValue = V;
2605 ++NumVectorInstructions;
2606 return V;
2607 }
2608 case Instruction::Add:
2609 case Instruction::FAdd:
2610 case Instruction::Sub:
2611 case Instruction::FSub:
2612 case Instruction::Mul:
2613 case Instruction::FMul:
2614 case Instruction::UDiv:
2615 case Instruction::SDiv:
2616 case Instruction::FDiv:
2617 case Instruction::URem:
2618 case Instruction::SRem:
2619 case Instruction::FRem:
2620 case Instruction::Shl:
2621 case Instruction::LShr:
2622 case Instruction::AShr:
2623 case Instruction::And:
2624 case Instruction::Or:
2625 case Instruction::Xor: {
2626 ValueList LHSVL, RHSVL;
2627 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2628 reorderInputsAccordingToOpcode(E->Scalars, LHSVL, RHSVL);
2629 else
2630 for (Value *V : E->Scalars) {
2631 LHSVL.push_back(cast<Instruction>(V)->getOperand(0));
2632 RHSVL.push_back(cast<Instruction>(V)->getOperand(1));
2633 }
2634
2635 setInsertPointAfterBundle(E->Scalars);
2636
2637 Value *LHS = vectorizeTree(LHSVL);
2638 Value *RHS = vectorizeTree(RHSVL);
2639
2640 if (Value *V = alreadyVectorized(E->Scalars))
2641 return V;
2642
2643 BinaryOperator *BinOp = cast<BinaryOperator>(VL0);
2644 Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS);
2645 E->VectorizedValue = V;
2646 propagateIRFlags(E->VectorizedValue, E->Scalars);
2647 ++NumVectorInstructions;
2648
2649 if (Instruction *I = dyn_cast<Instruction>(V))
2650 return propagateMetadata(I, E->Scalars);
2651
2652 return V;
2653 }
2654 case Instruction::Load: {
2655 // Loads are inserted at the head of the tree because we don't want to
2656 // sink them all the way down past store instructions.
2657 setInsertPointAfterBundle(E->Scalars);
2658
2659 LoadInst *LI = cast<LoadInst>(VL0);
2660 Type *ScalarLoadTy = LI->getType();
2661 unsigned AS = LI->getPointerAddressSpace();
2662
2663 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2664 VecTy->getPointerTo(AS));
2665
2666 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2667 // ExternalUses list to make sure that an extract will be generated in the
2668 // future.
2669 if (ScalarToTreeEntry.count(LI->getPointerOperand()))
2670 ExternalUses.push_back(
2671 ExternalUser(LI->getPointerOperand(), cast<User>(VecPtr), 0));
2672
2673 unsigned Alignment = LI->getAlignment();
2674 LI = Builder.CreateLoad(VecPtr);
2675 if (!Alignment) {
2676 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2677 }
2678 LI->setAlignment(Alignment);
2679 E->VectorizedValue = LI;
2680 ++NumVectorInstructions;
2681 return propagateMetadata(LI, E->Scalars);
2682 }
2683 case Instruction::Store: {
2684 StoreInst *SI = cast<StoreInst>(VL0);
2685 unsigned Alignment = SI->getAlignment();
2686 unsigned AS = SI->getPointerAddressSpace();
2687
2688 ValueList ValueOp;
2689 for (Value *V : E->Scalars)
2690 ValueOp.push_back(cast<StoreInst>(V)->getValueOperand());
2691
2692 setInsertPointAfterBundle(E->Scalars);
2693
2694 Value *VecValue = vectorizeTree(ValueOp);
2695 Value *VecPtr = Builder.CreateBitCast(SI->getPointerOperand(),
2696 VecTy->getPointerTo(AS));
2697 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
2698
2699 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2700 // ExternalUses list to make sure that an extract will be generated in the
2701 // future.
2702 if (ScalarToTreeEntry.count(SI->getPointerOperand()))
2703 ExternalUses.push_back(
2704 ExternalUser(SI->getPointerOperand(), cast<User>(VecPtr), 0));
2705
2706 if (!Alignment) {
2707 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
2708 }
2709 S->setAlignment(Alignment);
2710 E->VectorizedValue = S;
2711 ++NumVectorInstructions;
2712 return propagateMetadata(S, E->Scalars);
2713 }
2714 case Instruction::GetElementPtr: {
2715 setInsertPointAfterBundle(E->Scalars);
2716
2717 ValueList Op0VL;
2718 for (Value *V : E->Scalars)
2719 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
2720
2721 Value *Op0 = vectorizeTree(Op0VL);
2722
2723 std::vector<Value *> OpVecs;
2724 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
2725 ++j) {
2726 ValueList OpVL;
2727 for (Value *V : E->Scalars)
2728 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
2729
2730 Value *OpVec = vectorizeTree(OpVL);
2731 OpVecs.push_back(OpVec);
2732 }
2733
2734 Value *V = Builder.CreateGEP(
2735 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
2736 E->VectorizedValue = V;
2737 ++NumVectorInstructions;
2738
2739 if (Instruction *I = dyn_cast<Instruction>(V))
2740 return propagateMetadata(I, E->Scalars);
2741
2742 return V;
2743 }
2744 case Instruction::Call: {
2745 CallInst *CI = cast<CallInst>(VL0);
2746 setInsertPointAfterBundle(E->Scalars);
2747 Function *FI;
2748 Intrinsic::ID IID = Intrinsic::not_intrinsic;
2749 Value *ScalarArg = nullptr;
2750 if (CI && (FI = CI->getCalledFunction())) {
2751 IID = FI->getIntrinsicID();
2752 }
2753 std::vector<Value *> OpVecs;
2754 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
2755 ValueList OpVL;
2756 // ctlz,cttz and powi are special intrinsics whose second argument is
2757 // a scalar. This argument should not be vectorized.
2758 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
2759 CallInst *CEI = cast<CallInst>(E->Scalars[0]);
2760 ScalarArg = CEI->getArgOperand(j);
2761 OpVecs.push_back(CEI->getArgOperand(j));
2762 continue;
2763 }
2764 for (Value *V : E->Scalars) {
2765 CallInst *CEI = cast<CallInst>(V);
2766 OpVL.push_back(CEI->getArgOperand(j));
2767 }
2768
2769 Value *OpVec = vectorizeTree(OpVL);
2770 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
2771 OpVecs.push_back(OpVec);
2772 }
2773
2774 Module *M = F->getParent();
2775 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2776 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
2777 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
2778 SmallVector<OperandBundleDef, 1> OpBundles;
2779 CI->getOperandBundlesAsDefs(OpBundles);
2780 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
2781
2782 // The scalar argument uses an in-tree scalar so we add the new vectorized
2783 // call to ExternalUses list to make sure that an extract will be
2784 // generated in the future.
2785 if (ScalarArg && ScalarToTreeEntry.count(ScalarArg))
2786 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
2787
2788 E->VectorizedValue = V;
2789 propagateIRFlags(E->VectorizedValue, E->Scalars);
2790 ++NumVectorInstructions;
2791 return V;
2792 }
2793 case Instruction::ShuffleVector: {
2794 ValueList LHSVL, RHSVL;
2795 assert(isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand")((isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(VL0) && \"Invalid Shuffle Vector Operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2795, __PRETTY_FUNCTION__))
;
2796 reorderAltShuffleOperands(E->Scalars, LHSVL, RHSVL);
2797 setInsertPointAfterBundle(E->Scalars);
2798
2799 Value *LHS = vectorizeTree(LHSVL);
2800 Value *RHS = vectorizeTree(RHSVL);
2801
2802 if (Value *V = alreadyVectorized(E->Scalars))
2803 return V;
2804
2805 // Create a vector of LHS op1 RHS
2806 BinaryOperator *BinOp0 = cast<BinaryOperator>(VL0);
2807 Value *V0 = Builder.CreateBinOp(BinOp0->getOpcode(), LHS, RHS);
2808
2809 // Create a vector of LHS op2 RHS
2810 Instruction *VL1 = cast<Instruction>(E->Scalars[1]);
2811 BinaryOperator *BinOp1 = cast<BinaryOperator>(VL1);
2812 Value *V1 = Builder.CreateBinOp(BinOp1->getOpcode(), LHS, RHS);
2813
2814 // Create shuffle to take alternate operations from the vector.
2815 // Also, gather up odd and even scalar ops to propagate IR flags to
2816 // each vector operation.
2817 ValueList OddScalars, EvenScalars;
2818 unsigned e = E->Scalars.size();
2819 SmallVector<Constant *, 8> Mask(e);
2820 for (unsigned i = 0; i < e; ++i) {
2821 if (i & 1) {
2822 Mask[i] = Builder.getInt32(e + i);
2823 OddScalars.push_back(E->Scalars[i]);
2824 } else {
2825 Mask[i] = Builder.getInt32(i);
2826 EvenScalars.push_back(E->Scalars[i]);
2827 }
2828 }
2829
2830 Value *ShuffleMask = ConstantVector::get(Mask);
2831 propagateIRFlags(V0, EvenScalars);
2832 propagateIRFlags(V1, OddScalars);
2833
2834 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2835 E->VectorizedValue = V;
2836 ++NumVectorInstructions;
2837 if (Instruction *I = dyn_cast<Instruction>(V))
2838 return propagateMetadata(I, E->Scalars);
2839
2840 return V;
2841 }
2842 default:
2843 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2843)
;
2844 }
2845 return nullptr;
2846}
2847
2848Value *BoUpSLP::vectorizeTree() {
2849 ExtraValueToDebugLocsMap ExternallyUsedValues;
2850 return vectorizeTree(ExternallyUsedValues);
2851}
2852
2853Value *
2854BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
2855
2856 // All blocks must be scheduled before any instructions are inserted.
2857 for (auto &BSIter : BlocksSchedules) {
2858 scheduleBlock(BSIter.second.get());
2859 }
2860
2861 Builder.SetInsertPoint(&F->getEntryBlock().front());
2862 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
2863
2864 // If the vectorized tree can be rewritten in a smaller type, we truncate the
2865 // vectorized root. InstCombine will then rewrite the entire expression. We
2866 // sign extend the extracted values below.
2867 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2868 if (MinBWs.count(ScalarRoot)) {
2869 if (auto *I = dyn_cast<Instruction>(VectorRoot))
2870 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
2871 auto BundleWidth = VectorizableTree[0].Scalars.size();
2872 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2873 auto *VecTy = VectorType::get(MinTy, BundleWidth);
2874 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
2875 VectorizableTree[0].VectorizedValue = Trunc;
2876 }
2877
2878 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
2879
2880 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
2881 // specified by ScalarType.
2882 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
2883 if (!MinBWs.count(ScalarRoot))
2884 return Ex;
2885 if (MinBWs[ScalarRoot].second)
2886 return Builder.CreateSExt(Ex, ScalarType);
2887 return Builder.CreateZExt(Ex, ScalarType);
2888 };
2889
2890 // Extract all of the elements with the external uses.
2891 for (const auto &ExternalUse : ExternalUses) {
2892 Value *Scalar = ExternalUse.Scalar;
2893 llvm::User *User = ExternalUse.User;
2894
2895 // Skip users that we already RAUW. This happens when one instruction
2896 // has multiple uses of the same value.
2897 if (User && !is_contained(Scalar->users(), User))
2898 continue;
2899 assert(ScalarToTreeEntry.count(Scalar) && "Invalid scalar")((ScalarToTreeEntry.count(Scalar) && "Invalid scalar"
) ? static_cast<void> (0) : __assert_fail ("ScalarToTreeEntry.count(Scalar) && \"Invalid scalar\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2899, __PRETTY_FUNCTION__))
;
2900
2901 int Idx = ScalarToTreeEntry[Scalar];
2902 TreeEntry *E = &VectorizableTree[Idx];
2903 assert(!E->NeedToGather && "Extracting from a gather list")((!E->NeedToGather && "Extracting from a gather list"
) ? static_cast<void> (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2903, __PRETTY_FUNCTION__))
;
2904
2905 Value *Vec = E->VectorizedValue;
2906 assert(Vec && "Can't find vectorizable value")((Vec && "Can't find vectorizable value") ? static_cast
<void> (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2906, __PRETTY_FUNCTION__))
;
2907
2908 Value *Lane = Builder.getInt32(ExternalUse.Lane);
2909 // If User == nullptr, the Scalar is used as extra arg. Generate
2910 // ExtractElement instruction and update the record for this scalar in
2911 // ExternallyUsedValues.
2912 if (!User) {
2913 assert(ExternallyUsedValues.count(Scalar) &&((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2915, __PRETTY_FUNCTION__))
2914 "Scalar with nullptr as an external user must be registered in "((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2915, __PRETTY_FUNCTION__))
2915 "ExternallyUsedValues map")((ExternallyUsedValues.count(Scalar) && "Scalar with nullptr as an external user must be registered in "
"ExternallyUsedValues map") ? static_cast<void> (0) : __assert_fail
("ExternallyUsedValues.count(Scalar) && \"Scalar with nullptr as an external user must be registered in \" \"ExternallyUsedValues map\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2915, __PRETTY_FUNCTION__))
;
2916 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
2917 Builder.SetInsertPoint(VecI->getParent(),
2918 std::next(VecI->getIterator()));
2919 } else {
2920 Builder.SetInsertPoint(&F->getEntryBlock().front());
2921 }
2922 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2923 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2924 CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
2925 auto &Locs = ExternallyUsedValues[Scalar];
2926 ExternallyUsedValues.insert({Ex, Locs});
2927 ExternallyUsedValues.erase(Scalar);
2928 continue;
2929 }
2930
2931 // Generate extracts for out-of-tree users.
2932 // Find the insertion point for the extractelement lane.
2933 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
2934 if (PHINode *PH = dyn_cast<PHINode>(User)) {
2935 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
2936 if (PH->getIncomingValue(i) == Scalar) {
2937 TerminatorInst *IncomingTerminator =
2938 PH->getIncomingBlock(i)->getTerminator();
2939 if (isa<CatchSwitchInst>(IncomingTerminator)) {
2940 Builder.SetInsertPoint(VecI->getParent(),
2941 std::next(VecI->getIterator()));
2942 } else {
2943 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
2944 }
2945 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2946 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2947 CSEBlocks.insert(PH->getIncomingBlock(i));
2948 PH->setOperand(i, Ex);
2949 }
2950 }
2951 } else {
2952 Builder.SetInsertPoint(cast<Instruction>(User));
2953 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2954 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2955 CSEBlocks.insert(cast<Instruction>(User)->getParent());
2956 User->replaceUsesOfWith(Scalar, Ex);
2957 }
2958 } else {
2959 Builder.SetInsertPoint(&F->getEntryBlock().front());
2960 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2961 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2962 CSEBlocks.insert(&F->getEntryBlock());
2963 User->replaceUsesOfWith(Scalar, Ex);
2964 }
2965
2966 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
2967 }
2968
2969 // For each vectorized value:
2970 for (TreeEntry &EIdx : VectorizableTree) {
2971 TreeEntry *Entry = &EIdx;
2972
2973 // For each lane:
2974 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
2975 Value *Scalar = Entry->Scalars[Lane];
2976 // No need to handle users of gathered values.
2977 if (Entry->NeedToGather)
2978 continue;
2979
2980 assert(Entry->VectorizedValue && "Can't find vectorizable value")((Entry->VectorizedValue && "Can't find vectorizable value"
) ? static_cast<void> (0) : __assert_fail ("Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2980, __PRETTY_FUNCTION__))
;
2981
2982 Type *Ty = Scalar->getType();
2983 if (!Ty->isVoidTy()) {
2984#ifndef NDEBUG
2985 for (User *U : Scalar->users()) {
2986 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
2987
2988 assert((ScalarToTreeEntry.count(U) ||(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2991, __PRETTY_FUNCTION__))
2989 // It is legal to replace users in the ignorelist by undef.(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2991, __PRETTY_FUNCTION__))
2990 is_contained(UserIgnoreList, U)) &&(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2991, __PRETTY_FUNCTION__))
2991 "Replacing out-of-tree value with undef")(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2991, __PRETTY_FUNCTION__))
;
2992 }
2993#endif
2994 Value *Undef = UndefValue::get(Ty);
2995 Scalar->replaceAllUsesWith(Undef);
2996 }
2997 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
2998 eraseInstruction(cast<Instruction>(Scalar));
2999 }
3000 }
3001
3002 Builder.ClearInsertionPoint();
3003
3004 return VectorizableTree[0].VectorizedValue;
3005}
3006
3007void BoUpSLP::optimizeGatherSequence() {
3008 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
3009 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
3010 // LICM InsertElementInst sequences.
3011 for (Instruction *it : GatherSeq) {
3012 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
3013
3014 if (!Insert)
3015 continue;
3016
3017 // Check if this block is inside a loop.
3018 Loop *L = LI->getLoopFor(Insert->getParent());
3019 if (!L)
3020 continue;
3021
3022 // Check if it has a preheader.
3023 BasicBlock *PreHeader = L->getLoopPreheader();
3024 if (!PreHeader)
3025 continue;
3026
3027 // If the vector or the element that we insert into it are
3028 // instructions that are defined in this basic block then we can't
3029 // hoist this instruction.
3030 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
3031 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
3032 if (CurrVec && L->contains(CurrVec))
3033 continue;
3034 if (NewElem && L->contains(NewElem))
3035 continue;
3036
3037 // We can hoist this instruction. Move it to the pre-header.
3038 Insert->moveBefore(PreHeader->getTerminator());
3039 }
3040
3041 // Make a list of all reachable blocks in our CSE queue.
3042 SmallVector<const DomTreeNode *, 8> CSEWorkList;
3043 CSEWorkList.reserve(CSEBlocks.size());
3044 for (BasicBlock *BB : CSEBlocks)
3045 if (DomTreeNode *N = DT->getNode(BB)) {
3046 assert(DT->isReachableFromEntry(N))((DT->isReachableFromEntry(N)) ? static_cast<void> (
0) : __assert_fail ("DT->isReachableFromEntry(N)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3046, __PRETTY_FUNCTION__))
;
3047 CSEWorkList.push_back(N);
3048 }
3049
3050 // Sort blocks by domination. This ensures we visit a block after all blocks
3051 // dominating it are visited.
3052 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3053 [this](const DomTreeNode *A, const DomTreeNode *B) {
3054 return DT->properlyDominates(A, B);
3055 });
3056
3057 // Perform O(N^2) search over the gather sequences and merge identical
3058 // instructions. TODO: We can further optimize this scan if we split the
3059 // instructions into different buckets based on the insert lane.
3060 SmallVector<Instruction *, 16> Visited;
3061 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
1
Assuming 'I' is not equal to 'E'
2
Loop condition is true. Entering loop body
4
Loop condition is true. Entering loop body
6
Loop condition is true. Entering loop body
8
Loop condition is true. Entering loop body
3062 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3063, __PRETTY_FUNCTION__))
3063 "Worklist not sorted properly!")(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3063, __PRETTY_FUNCTION__))
;
3064 BasicBlock *BB = (*I)->getBlock();
9
Called C++ object pointer is null
3065 // For all instructions in blocks containing gather sequences:
3066 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3
Loop condition is false. Execution continues on line 3061
5
Loop condition is false. Execution continues on line 3061
7
Loop condition is false. Execution continues on line 3061
3067 Instruction *In = &*it++;
3068 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3069 continue;
3070
3071 // Check if we can replace this instruction with any of the
3072 // visited instructions.
3073 for (Instruction *v : Visited) {
3074 if (In->isIdenticalTo(v) &&
3075 DT->dominates(v->getParent(), In->getParent())) {
3076 In->replaceAllUsesWith(v);
3077 eraseInstruction(In);
3078 In = nullptr;
3079 break;
3080 }
3081 }
3082 if (In) {
3083 assert(!is_contained(Visited, In))((!is_contained(Visited, In)) ? static_cast<void> (0) :
__assert_fail ("!is_contained(Visited, In)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3083, __PRETTY_FUNCTION__))
;
3084 Visited.push_back(In);
3085 }
3086 }
3087 }
3088 CSEBlocks.clear();
3089 GatherSeq.clear();
3090}
3091
3092// Groups the instructions to a bundle (which is then a single scheduling entity)
3093// and schedules instructions until the bundle gets ready.
3094bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3095 BoUpSLP *SLP) {
3096 if (isa<PHINode>(VL[0]))
3097 return true;
3098
3099 // Initialize the instruction bundle.
3100 Instruction *OldScheduleEnd = ScheduleEnd;
3101 ScheduleData *PrevInBundle = nullptr;
3102 ScheduleData *Bundle = nullptr;
3103 bool ReSchedule = false;
3104 DEBUG(dbgs() << "SLP: bundle: " << *VL[0] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *VL[0] <<
"\n"; } } while (false)
;
3105
3106 // Make sure that the scheduling region contains all
3107 // instructions of the bundle.
3108 for (Value *V : VL) {
3109 if (!extendSchedulingRegion(V))
3110 return false;
3111 }
3112
3113 for (Value *V : VL) {
3114 ScheduleData *BundleMember = getScheduleData(V);
3115 assert(BundleMember &&((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3116, __PRETTY_FUNCTION__))
3116 "no ScheduleData for bundle member (maybe not in same basic block)")((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3116, __PRETTY_FUNCTION__))
;
3117 if (BundleMember->IsScheduled) {
3118 // A bundle member was scheduled as single instruction before and now
3119 // needs to be scheduled as part of the bundle. We just get rid of the
3120 // existing schedule.
3121 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
3122 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
3123 ReSchedule = true;
3124 }
3125 assert(BundleMember->isSchedulingEntity() &&((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3126, __PRETTY_FUNCTION__))
3126 "bundle member already part of other bundle")((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3126, __PRETTY_FUNCTION__))
;
3127 if (PrevInBundle) {
3128 PrevInBundle->NextInBundle = BundleMember;
3129 } else {
3130 Bundle = BundleMember;
3131 }
3132 BundleMember->UnscheduledDepsInBundle = 0;
3133 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3134
3135 // Group the instructions to a bundle.
3136 BundleMember->FirstInBundle = Bundle;
3137 PrevInBundle = BundleMember;
3138 }
3139 if (ScheduleEnd != OldScheduleEnd) {
3140 // The scheduling region got new instructions at the lower end (or it is a
3141 // new region for the first bundle). This makes it necessary to
3142 // recalculate all dependencies.
3143 // It is seldom that this needs to be done a second time after adding the
3144 // initial bundle to the region.
3145 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3146 ScheduleData *SD = getScheduleData(I);
3147 SD->clearDependencies();
3148 }
3149 ReSchedule = true;
3150 }
3151 if (ReSchedule) {
3152 resetSchedule();
3153 initialFillReadyList(ReadyInsts);
3154 }
3155
3156 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
3157 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
3158
3159 calculateDependencies(Bundle, true, SLP);
3160
3161 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3162 // means that there are no cyclic dependencies and we can schedule it.
3163 // Note that's important that we don't "schedule" the bundle yet (see
3164 // cancelScheduling).
3165 while (!Bundle->isReady() && !ReadyInsts.empty()) {
3166
3167 ScheduleData *pickedSD = ReadyInsts.back();
3168 ReadyInsts.pop_back();
3169
3170 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3171 schedule(pickedSD, ReadyInsts);
3172 }
3173 }
3174 if (!Bundle->isReady()) {
3175 cancelScheduling(VL);
3176 return false;
3177 }
3178 return true;
3179}
3180
3181void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL) {
3182 if (isa<PHINode>(VL[0]))
3183 return;
3184
3185 ScheduleData *Bundle = getScheduleData(VL[0]);
3186 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3187 assert(!Bundle->IsScheduled &&((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3188, __PRETTY_FUNCTION__))
3188 "Can't cancel bundle which is already scheduled")((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3188, __PRETTY_FUNCTION__))
;
3189 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3190, __PRETTY_FUNCTION__))
3190 "tried to unbundle something which is not a bundle")((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3190, __PRETTY_FUNCTION__))
;
3191
3192 // Un-bundle: make single instructions out of the bundle.
3193 ScheduleData *BundleMember = Bundle;
3194 while (BundleMember) {
3195 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")((BundleMember->FirstInBundle == Bundle && "corrupt bundle links"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3195, __PRETTY_FUNCTION__))
;
3196 BundleMember->FirstInBundle = BundleMember;
3197 ScheduleData *Next = BundleMember->NextInBundle;
3198 BundleMember->NextInBundle = nullptr;
3199 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3200 if (BundleMember->UnscheduledDepsInBundle == 0) {
3201 ReadyInsts.insert(BundleMember);
3202 }
3203 BundleMember = Next;
3204 }
3205}
3206
3207bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V) {
3208 if (getScheduleData(V))
3209 return true;
3210 Instruction *I = dyn_cast<Instruction>(V);
3211 assert(I && "bundle member must be an instruction")((I && "bundle member must be an instruction") ? static_cast
<void> (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3211, __PRETTY_FUNCTION__))
;
3212 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")((!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3212, __PRETTY_FUNCTION__))
;
3213 if (!ScheduleStart) {
3214 // It's the first instruction in the new region.
3215 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3216 ScheduleStart = I;
3217 ScheduleEnd = I->getNextNode();
3218 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3218, __PRETTY_FUNCTION__))
;
3219 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3220 return true;
3221 }
3222 // Search up and down at the same time, because we don't know if the new
3223 // instruction is above or below the existing scheduling region.
3224 BasicBlock::reverse_iterator UpIter =
3225 ++ScheduleStart->getIterator().getReverse();
3226 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3227 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3228 BasicBlock::iterator LowerEnd = BB->end();
3229 for (;;) {
3230 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3231 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3232 return false;
3233 }
3234
3235 if (UpIter != UpperEnd) {
3236 if (&*UpIter == I) {
3237 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3238 ScheduleStart = I;
3239 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3240 return true;
3241 }
3242 UpIter++;
3243 }
3244 if (DownIter != LowerEnd) {
3245 if (&*DownIter == I) {
3246 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3247 nullptr);
3248 ScheduleEnd = I->getNextNode();
3249 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3249, __PRETTY_FUNCTION__))
;
3250 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
3251 return true;
3252 }
3253 DownIter++;
3254 }
3255 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3256, __PRETTY_FUNCTION__))
3256 "instruction not found in block")(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3256, __PRETTY_FUNCTION__))
;
3257 }
3258 return true;
3259}
3260
3261void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3262 Instruction *ToI,
3263 ScheduleData *PrevLoadStore,
3264 ScheduleData *NextLoadStore) {
3265 ScheduleData *CurrentLoadStore = PrevLoadStore;
3266 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3267 ScheduleData *SD = ScheduleDataMap[I];
3268 if (!SD) {
3269 // Allocate a new ScheduleData for the instruction.
3270 if (ChunkPos >= ChunkSize) {
3271 ScheduleDataChunks.push_back(
3272 llvm::make_unique<ScheduleData[]>(ChunkSize));
3273 ChunkPos = 0;
3274 }
3275 SD = &(ScheduleDataChunks.back()[ChunkPos++]);
3276 ScheduleDataMap[I] = SD;
3277 SD->Inst = I;
3278 }
3279 assert(!isInSchedulingRegion(SD) &&((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3280, __PRETTY_FUNCTION__))
3280 "new ScheduleData already in scheduling region")((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3280, __PRETTY_FUNCTION__))
;
3281 SD->init(SchedulingRegionID);
3282
3283 if (I->mayReadOrWriteMemory()) {
3284 // Update the linked list of memory accessing instructions.
3285 if (CurrentLoadStore) {
3286 CurrentLoadStore->NextLoadStore = SD;
3287 } else {
3288 FirstLoadStoreInRegion = SD;
3289 }
3290 CurrentLoadStore = SD;
3291 }
3292 }
3293 if (NextLoadStore) {
3294 if (CurrentLoadStore)
3295 CurrentLoadStore->NextLoadStore = NextLoadStore;
3296 } else {
3297 LastLoadStoreInRegion = CurrentLoadStore;
3298 }
3299}
3300
3301void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3302 bool InsertInReadyList,
3303 BoUpSLP *SLP) {
3304 assert(SD->isSchedulingEntity())((SD->isSchedulingEntity()) ? static_cast<void> (0) :
__assert_fail ("SD->isSchedulingEntity()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3304, __PRETTY_FUNCTION__))
;
3305
3306 SmallVector<ScheduleData *, 10> WorkList;
3307 WorkList.push_back(SD);
3308
3309 while (!WorkList.empty()) {
3310 ScheduleData *SD = WorkList.back();
3311 WorkList.pop_back();
3312
3313 ScheduleData *BundleMember = SD;
3314 while (BundleMember) {
3315 assert(isInSchedulingRegion(BundleMember))((isInSchedulingRegion(BundleMember)) ? static_cast<void>
(0) : __assert_fail ("isInSchedulingRegion(BundleMember)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3315, __PRETTY_FUNCTION__))
;
3316 if (!BundleMember->hasValidDependencies()) {
3317
3318 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
3319 BundleMember->Dependencies = 0;
3320 BundleMember->resetUnscheduledDeps();
3321
3322 // Handle def-use chain dependencies.
3323 for (User *U : BundleMember->Inst->users()) {
3324 if (isa<Instruction>(U)) {
3325 ScheduleData *UseSD = getScheduleData(U);
3326 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3327 BundleMember->Dependencies++;
3328 ScheduleData *DestBundle = UseSD->FirstInBundle;
3329 if (!DestBundle->IsScheduled) {
3330 BundleMember->incrementUnscheduledDeps(1);
3331 }
3332 if (!DestBundle->hasValidDependencies()) {
3333 WorkList.push_back(DestBundle);
3334 }
3335 }
3336 } else {
3337 // I'm not sure if this can ever happen. But we need to be safe.
3338 // This lets the instruction/bundle never be scheduled and
3339 // eventually disable vectorization.
3340 BundleMember->Dependencies++;
3341 BundleMember->incrementUnscheduledDeps(1);
3342 }
3343 }
3344
3345 // Handle the memory dependencies.
3346 ScheduleData *DepDest = BundleMember->NextLoadStore;
3347 if (DepDest) {
3348 Instruction *SrcInst = BundleMember->Inst;
3349 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3350 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3351 unsigned numAliased = 0;
3352 unsigned DistToSrc = 1;
3353
3354 while (DepDest) {
3355 assert(isInSchedulingRegion(DepDest))((isInSchedulingRegion(DepDest)) ? static_cast<void> (0
) : __assert_fail ("isInSchedulingRegion(DepDest)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3355, __PRETTY_FUNCTION__))
;
3356
3357 // We have two limits to reduce the complexity:
3358 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3359 // SLP->isAliased (which is the expensive part in this loop).
3360 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3361 // the whole loop (even if the loop is fast, it's quadratic).
3362 // It's important for the loop break condition (see below) to
3363 // check this limit even between two read-only instructions.
3364 if (DistToSrc >= MaxMemDepDistance ||
3365 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3366 (numAliased >= AliasedCheckLimit ||
3367 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3368
3369 // We increment the counter only if the locations are aliased
3370 // (instead of counting all alias checks). This gives a better
3371 // balance between reduced runtime and accurate dependencies.
3372 numAliased++;
3373
3374 DepDest->MemoryDependencies.push_back(BundleMember);
3375 BundleMember->Dependencies++;
3376 ScheduleData *DestBundle = DepDest->FirstInBundle;
3377 if (!DestBundle->IsScheduled) {
3378 BundleMember->incrementUnscheduledDeps(1);
3379 }
3380 if (!DestBundle->hasValidDependencies()) {
3381 WorkList.push_back(DestBundle);
3382 }
3383 }
3384 DepDest = DepDest->NextLoadStore;
3385
3386 // Example, explaining the loop break condition: Let's assume our
3387 // starting instruction is i0 and MaxMemDepDistance = 3.
3388 //
3389 // +--------v--v--v
3390 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3391 // +--------^--^--^
3392 //
3393 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3394 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3395 // Previously we already added dependencies from i3 to i6,i7,i8
3396 // (because of MaxMemDepDistance). As we added a dependency from
3397 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3398 // and we can abort this loop at i6.
3399 if (DistToSrc >= 2 * MaxMemDepDistance)
3400 break;
3401 DistToSrc++;
3402 }
3403 }
3404 }
3405 BundleMember = BundleMember->NextInBundle;
3406 }
3407 if (InsertInReadyList && SD->isReady()) {
3408 ReadyInsts.push_back(SD);
3409 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
3410 }
3411 }
3412}
3413
3414void BoUpSLP::BlockScheduling::resetSchedule() {
3415 assert(ScheduleStart &&((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3416, __PRETTY_FUNCTION__))
3416 "tried to reset schedule on block which has not been scheduled")((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3416, __PRETTY_FUNCTION__))
;
3417 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3418 ScheduleData *SD = getScheduleData(I);
3419 assert(isInSchedulingRegion(SD))((isInSchedulingRegion(SD)) ? static_cast<void> (0) : __assert_fail
("isInSchedulingRegion(SD)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3419, __PRETTY_FUNCTION__))
;
3420 SD->IsScheduled = false;
3421 SD->resetUnscheduledDeps();
3422 }
3423 ReadyInsts.clear();
3424}
3425
3426void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3427
3428 if (!BS->ScheduleStart)
3429 return;
3430
3431 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
3432
3433 BS->resetSchedule();
3434
3435 // For the real scheduling we use a more sophisticated ready-list: it is
3436 // sorted by the original instruction location. This lets the final schedule
3437 // be as close as possible to the original instruction order.
3438 struct ScheduleDataCompare {
3439 bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3440 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3441 }
3442 };
3443 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3444
3445 // Ensure that all dependency data is updated and fill the ready-list with
3446 // initial instructions.
3447 int Idx = 0;
3448 int NumToSchedule = 0;
3449 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3450 I = I->getNextNode()) {
3451 ScheduleData *SD = BS->getScheduleData(I);
3452 assert(((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3454, __PRETTY_FUNCTION__))
3453 SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) &&((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3454, __PRETTY_FUNCTION__))
3454 "scheduler and vectorizer have different opinion on what is a bundle")((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3454, __PRETTY_FUNCTION__))
;
3455 SD->FirstInBundle->SchedulingPriority = Idx++;
3456 if (SD->isSchedulingEntity()) {
3457 BS->calculateDependencies(SD, false, this);
3458 NumToSchedule++;
3459 }
3460 }
3461 BS->initialFillReadyList(ReadyInsts);
3462
3463 Instruction *LastScheduledInst = BS->ScheduleEnd;
3464
3465 // Do the "real" scheduling.
3466 while (!ReadyInsts.empty()) {
3467 ScheduleData *picked = *ReadyInsts.begin();
3468 ReadyInsts.erase(ReadyInsts.begin());
3469
3470 // Move the scheduled instruction(s) to their dedicated places, if not
3471 // there yet.
3472 ScheduleData *BundleMember = picked;
3473 while (BundleMember) {
3474 Instruction *pickedInst = BundleMember->Inst;
3475 if (LastScheduledInst->getNextNode() != pickedInst) {
3476 BS->BB->getInstList().remove(pickedInst);
3477 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3478 pickedInst);
3479 }
3480 LastScheduledInst = pickedInst;
3481 BundleMember = BundleMember->NextInBundle;
3482 }
3483
3484 BS->schedule(picked, ReadyInsts);
3485 NumToSchedule--;
3486 }
3487 assert(NumToSchedule == 0 && "could not schedule all instructions")((NumToSchedule == 0 && "could not schedule all instructions"
) ? static_cast<void> (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3487, __PRETTY_FUNCTION__))
;
3488
3489 // Avoid duplicate scheduling of the block.
3490 BS->ScheduleStart = nullptr;
3491}
3492
3493unsigned BoUpSLP::getVectorElementSize(Value *V) {
3494 // If V is a store, just return the width of the stored value without
3495 // traversing the expression tree. This is the common case.
3496 if (auto *Store = dyn_cast<StoreInst>(V))
3497 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3498
3499 // If V is not a store, we can traverse the expression tree to find loads
3500 // that feed it. The type of the loaded value may indicate a more suitable
3501 // width than V's type. We want to base the vector element size on the width
3502 // of memory operations where possible.
3503 SmallVector<Instruction *, 16> Worklist;
3504 SmallPtrSet<Instruction *, 16> Visited;
3505 if (auto *I = dyn_cast<Instruction>(V))
3506 Worklist.push_back(I);
3507
3508 // Traverse the expression tree in bottom-up order looking for loads. If we
3509 // encounter an instruciton we don't yet handle, we give up.
3510 auto MaxWidth = 0u;
3511 auto FoundUnknownInst = false;
3512 while (!Worklist.empty() && !FoundUnknownInst) {
3513 auto *I = Worklist.pop_back_val();
3514 Visited.insert(I);
3515
3516 // We should only be looking at scalar instructions here. If the current
3517 // instruction has a vector type, give up.
3518 auto *Ty = I->getType();
3519 if (isa<VectorType>(Ty))
3520 FoundUnknownInst = true;
3521
3522 // If the current instruction is a load, update MaxWidth to reflect the
3523 // width of the loaded value.
3524 else if (isa<LoadInst>(I))
3525 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3526
3527 // Otherwise, we need to visit the operands of the instruction. We only
3528 // handle the interesting cases from buildTree here. If an operand is an
3529 // instruction we haven't yet visited, we add it to the worklist.
3530 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3531 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3532 for (Use &U : I->operands())
3533 if (auto *J = dyn_cast<Instruction>(U.get()))
3534 if (!Visited.count(J))
3535 Worklist.push_back(J);
3536 }
3537
3538 // If we don't yet handle the instruction, give up.
3539 else
3540 FoundUnknownInst = true;
3541 }
3542
3543 // If we didn't encounter a memory access in the expression tree, or if we
3544 // gave up for some reason, just return the width of V.
3545 if (!MaxWidth || FoundUnknownInst)
3546 return DL->getTypeSizeInBits(V->getType());
3547
3548 // Otherwise, return the maximum width we found.
3549 return MaxWidth;
3550}
3551
3552// Determine if a value V in a vectorizable expression Expr can be demoted to a
3553// smaller type with a truncation. We collect the values that will be demoted
3554// in ToDemote and additional roots that require investigating in Roots.
3555static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
3556 SmallVectorImpl<Value *> &ToDemote,
3557 SmallVectorImpl<Value *> &Roots) {
3558
3559 // We can always demote constants.
3560 if (isa<Constant>(V)) {
3561 ToDemote.push_back(V);
3562 return true;
3563 }
3564
3565 // If the value is not an instruction in the expression with only one use, it
3566 // cannot be demoted.
3567 auto *I = dyn_cast<Instruction>(V);
3568 if (!I || !I->hasOneUse() || !Expr.count(I))
3569 return false;
3570
3571 switch (I->getOpcode()) {
3572
3573 // We can always demote truncations and extensions. Since truncations can
3574 // seed additional demotion, we save the truncated value.
3575 case Instruction::Trunc:
3576 Roots.push_back(I->getOperand(0));
3577 case Instruction::ZExt:
3578 case Instruction::SExt:
3579 break;
3580
3581 // We can demote certain binary operations if we can demote both of their
3582 // operands.
3583 case Instruction::Add:
3584 case Instruction::Sub:
3585 case Instruction::Mul:
3586 case Instruction::And:
3587 case Instruction::Or:
3588 case Instruction::Xor:
3589 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3590 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3591 return false;
3592 break;
3593
3594 // We can demote selects if we can demote their true and false values.
3595 case Instruction::Select: {
3596 SelectInst *SI = cast<SelectInst>(I);
3597 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3598 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3599 return false;
3600 break;
3601 }
3602
3603 // We can demote phis if we can demote all their incoming operands. Note that
3604 // we don't need to worry about cycles since we ensure single use above.
3605 case Instruction::PHI: {
3606 PHINode *PN = cast<PHINode>(I);
3607 for (Value *IncValue : PN->incoming_values())
3608 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3609 return false;
3610 break;
3611 }
3612
3613 // Otherwise, conservatively give up.
3614 default:
3615 return false;
3616 }
3617
3618 // Record the value that we can demote.
3619 ToDemote.push_back(V);
3620 return true;
3621}
3622
3623void BoUpSLP::computeMinimumValueSizes() {
3624 // If there are no external uses, the expression tree must be rooted by a
3625 // store. We can't demote in-memory values, so there is nothing to do here.
3626 if (ExternalUses.empty())
3627 return;
3628
3629 // We only attempt to truncate integer expressions.
3630 auto &TreeRoot = VectorizableTree[0].Scalars;
3631 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3632 if (!TreeRootIT)
3633 return;
3634
3635 // If the expression is not rooted by a store, these roots should have
3636 // external uses. We will rely on InstCombine to rewrite the expression in
3637 // the narrower type. However, InstCombine only rewrites single-use values.
3638 // This means that if a tree entry other than a root is used externally, it
3639 // must have multiple uses and InstCombine will not rewrite it. The code
3640 // below ensures that only the roots are used externally.
3641 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3642 for (auto &EU : ExternalUses)
3643 if (!Expr.erase(EU.Scalar))
3644 return;
3645 if (!Expr.empty())
3646 return;
3647
3648 // Collect the scalar values of the vectorizable expression. We will use this
3649 // context to determine which values can be demoted. If we see a truncation,
3650 // we mark it as seeding another demotion.
3651 for (auto &Entry : VectorizableTree)
3652 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3653
3654 // Ensure the roots of the vectorizable tree don't form a cycle. They must
3655 // have a single external user that is not in the vectorizable tree.
3656 for (auto *Root : TreeRoot)
3657 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
3658 return;
3659
3660 // Conservatively determine if we can actually truncate the roots of the
3661 // expression. Collect the values that can be demoted in ToDemote and
3662 // additional roots that require investigating in Roots.
3663 SmallVector<Value *, 32> ToDemote;
3664 SmallVector<Value *, 4> Roots;
3665 for (auto *Root : TreeRoot)
3666 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
3667 return;
3668
3669 // The maximum bit width required to represent all the values that can be
3670 // demoted without loss of precision. It would be safe to truncate the roots
3671 // of the expression to this width.
3672 auto MaxBitWidth = 8u;
3673
3674 // We first check if all the bits of the roots are demanded. If they're not,
3675 // we can truncate the roots to this narrower type.
3676 for (auto *Root : TreeRoot) {
3677 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
3678 MaxBitWidth = std::max<unsigned>(
3679 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
3680 }
3681
3682 // True if the roots can be zero-extended back to their original type, rather
3683 // than sign-extended. We know that if the leading bits are not demanded, we
3684 // can safely zero-extend. So we initialize IsKnownPositive to True.
3685 bool IsKnownPositive = true;
3686
3687 // If all the bits of the roots are demanded, we can try a little harder to
3688 // compute a narrower type. This can happen, for example, if the roots are
3689 // getelementptr indices. InstCombine promotes these indices to the pointer
3690 // width. Thus, all their bits are technically demanded even though the
3691 // address computation might be vectorized in a smaller type.
3692 //
3693 // We start by looking at each entry that can be demoted. We compute the
3694 // maximum bit width required to store the scalar by using ValueTracking to
3695 // compute the number of high-order bits we can truncate.
3696 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
3697 MaxBitWidth = 8u;
3698
3699 // Determine if the sign bit of all the roots is known to be zero. If not,
3700 // IsKnownPositive is set to False.
3701 IsKnownPositive = all_of(TreeRoot, [&](Value *R) {
3702 KnownBits Known = computeKnownBits(R, *DL);
3703 return Known.isNonNegative();
3704 });
3705
3706 // Determine the maximum number of bits required to store the scalar
3707 // values.
3708 for (auto *Scalar : ToDemote) {
3709 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, 0, DT);
3710 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
3711 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
3712 }
3713
3714 // If we can't prove that the sign bit is zero, we must add one to the
3715 // maximum bit width to account for the unknown sign bit. This preserves
3716 // the existing sign bit so we can safely sign-extend the root back to the
3717 // original type. Otherwise, if we know the sign bit is zero, we will
3718 // zero-extend the root instead.
3719 //
3720 // FIXME: This is somewhat suboptimal, as there will be cases where adding
3721 // one to the maximum bit width will yield a larger-than-necessary
3722 // type. In general, we need to add an extra bit only if we can't
3723 // prove that the upper bit of the original type is equal to the
3724 // upper bit of the proposed smaller type. If these two bits are the
3725 // same (either zero or one) we know that sign-extending from the
3726 // smaller type will result in the same value. Here, since we can't
3727 // yet prove this, we are just making the proposed smaller type
3728 // larger to ensure correctness.
3729 if (!IsKnownPositive)
3730 ++MaxBitWidth;
3731 }
3732
3733 // Round MaxBitWidth up to the next power-of-two.
3734 if (!isPowerOf2_64(MaxBitWidth))
3735 MaxBitWidth = NextPowerOf2(MaxBitWidth);
3736
3737 // If the maximum bit width we compute is less than the with of the roots'
3738 // type, we can proceed with the narrowing. Otherwise, do nothing.
3739 if (MaxBitWidth >= TreeRootIT->getBitWidth())
3740 return;
3741
3742 // If we can truncate the root, we must collect additional values that might
3743 // be demoted as a result. That is, those seeded by truncations we will
3744 // modify.
3745 while (!Roots.empty())
3746 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
3747
3748 // Finally, map the values we can demote to the maximum bit with we computed.
3749 for (auto *Scalar : ToDemote)
3750 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
3751}
3752
3753namespace {
3754/// The SLPVectorizer Pass.
3755struct SLPVectorizer : public FunctionPass {
3756 SLPVectorizerPass Impl;
3757
3758 /// Pass identification, replacement for typeid
3759 static char ID;
3760
3761 explicit SLPVectorizer() : FunctionPass(ID) {
3762 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
3763 }
3764
3765
3766 bool doInitialization(Module &M) override {
3767 return false;
3768 }
3769
3770 bool runOnFunction(Function &F) override {
3771 if (skipFunction(F))
3772 return false;
3773
3774 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
3775 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
3776 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
3777 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
3778 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3779 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
3780 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
3781 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
3782 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
3783 auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
3784
3785 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
3786 }
3787
3788 void getAnalysisUsage(AnalysisUsage &AU) const override {
3789 FunctionPass::getAnalysisUsage(AU);
3790 AU.addRequired<AssumptionCacheTracker>();
3791 AU.addRequired<ScalarEvolutionWrapperPass>();
3792 AU.addRequired<AAResultsWrapperPass>();
3793 AU.addRequired<TargetTransformInfoWrapperPass>();
3794 AU.addRequired<LoopInfoWrapperPass>();
3795 AU.addRequired<DominatorTreeWrapperPass>();
3796 AU.addRequired<DemandedBitsWrapperPass>();
3797 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
3798 AU.addPreserved<LoopInfoWrapperPass>();
3799 AU.addPreserved<DominatorTreeWrapperPass>();
3800 AU.addPreserved<AAResultsWrapperPass>();
3801 AU.addPreserved<GlobalsAAWrapperPass>();
3802 AU.setPreservesCFG();
3803 }
3804};
3805} // end anonymous namespace
3806
3807PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
3808 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
3809 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
3810 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
3811 auto *AA = &AM.getResult<AAManager>(F);
3812 auto *LI = &AM.getResult<LoopAnalysis>(F);
3813 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
3814 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
3815 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
3816 auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
3817
3818 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
3819 if (!Changed)
3820 return PreservedAnalyses::all();
3821
3822 PreservedAnalyses PA;
3823 PA.preserveSet<CFGAnalyses>();
3824 PA.preserve<AAManager>();
3825 PA.preserve<GlobalsAA>();
3826 return PA;
3827}
3828
3829bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
3830 TargetTransformInfo *TTI_,
3831 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
3832 LoopInfo *LI_, DominatorTree *DT_,
3833 AssumptionCache *AC_, DemandedBits *DB_,
3834 OptimizationRemarkEmitter *ORE_) {
3835 SE = SE_;
3836 TTI = TTI_;
3837 TLI = TLI_;
3838 AA = AA_;
3839 LI = LI_;
3840 DT = DT_;
3841 AC = AC_;
3842 DB = DB_;
3843 DL = &F.getParent()->getDataLayout();
3844
3845 Stores.clear();
3846 GEPs.clear();
3847 bool Changed = false;
3848
3849 // If the target claims to have no vector registers don't attempt
3850 // vectorization.
3851 if (!TTI->getNumberOfRegisters(true))
3852 return false;
3853
3854 // Don't vectorize when the attribute NoImplicitFloat is used.
3855 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
3856 return false;
3857
3858 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
3859
3860 // Use the bottom up slp vectorizer to construct chains that start with
3861 // store instructions.
3862 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
3863
3864 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
3865 // delete instructions.
3866
3867 // Scan the blocks in the function in post order.
3868 for (auto BB : post_order(&F.getEntryBlock())) {
3869 collectSeedInstructions(BB);
3870
3871 // Vectorize trees that end at stores.
3872 if (!Stores.empty()) {
3873 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
3874 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
3875 Changed |= vectorizeStoreChains(R);
3876 }
3877
3878 // Vectorize trees that end at reductions.
3879 Changed |= vectorizeChainsInBlock(BB, R);
3880
3881 // Vectorize the index computations of getelementptr instructions. This
3882 // is primarily intended to catch gather-like idioms ending at
3883 // non-consecutive loads.
3884 if (!GEPs.empty()) {
3885 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
3886 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
3887 Changed |= vectorizeGEPIndices(BB, R);
3888 }
3889 }
3890
3891 if (Changed) {
3892 R.optimizeGatherSequence();
3893 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
3894 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
3895 }
3896 return Changed;
3897}
3898
3899/// \brief Check that the Values in the slice in VL array are still existent in
3900/// the WeakTrackingVH array.
3901/// Vectorization of part of the VL array may cause later values in the VL array
3902/// to become invalid. We track when this has happened in the WeakTrackingVH
3903/// array.
3904static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
3905 ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
3906 unsigned SliceSize) {
3907 VL = VL.slice(SliceBegin, SliceSize);
3908 VH = VH.slice(SliceBegin, SliceSize);
3909 return !std::equal(VL.begin(), VL.end(), VH.begin());
3910}
3911
3912bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
3913 unsigned VecRegSize) {
3914 unsigned ChainLen = Chain.size();
3915 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
3916 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
3917 unsigned Sz = R.getVectorElementSize(Chain[0]);
3918 unsigned VF = VecRegSize / Sz;
3919
3920 if (!isPowerOf2_32(Sz) || VF < 2)
3921 return false;
3922
3923 // Keep track of values that were deleted by vectorizing in the loop below.
3924 SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
3925
3926 bool Changed = false;
3927 // Look for profitable vectorizable trees at all offsets, starting at zero.
3928 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
3929 if (i + VF > e)
3930 break;
3931
3932 // Check that a previous iteration of this loop did not delete the Value.
3933 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
3934 continue;
3935
3936 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
3937 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
3938 ArrayRef<Value *> Operands = Chain.slice(i, VF);
3939
3940 R.buildTree(Operands);
3941 if (R.isTreeTinyAndNotFullyVectorizable())
3942 continue;
3943
3944 R.computeMinimumValueSizes();
3945
3946 int Cost = R.getTreeCost();
3947
3948 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
3949 if (Cost < -SLPCostThreshold) {
3950 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
3951 using namespace ore;
3952 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "StoresVectorized",
3953 cast<StoreInst>(Chain[i]))
3954 << "Stores SLP vectorized with cost " << NV("Cost", Cost)
3955 << " and with tree size "
3956 << NV("TreeSize", R.getTreeSize()));
3957
3958 R.vectorizeTree();
3959
3960 // Move to the next bundle.
3961 i += VF - 1;
3962 Changed = true;
3963 }
3964 }
3965
3966 return Changed;
3967}
3968
3969bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
3970 BoUpSLP &R) {
3971 SetVector<StoreInst *> Heads, Tails;
3972 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
3973
3974 // We may run into multiple chains that merge into a single chain. We mark the
3975 // stores that we vectorized so that we don't visit the same store twice.
3976 BoUpSLP::ValueSet VectorizedStores;
3977 bool Changed = false;
3978
3979 // Do a quadratic search on all of the given stores and find
3980 // all of the pairs of stores that follow each other.
3981 SmallVector<unsigned, 16> IndexQueue;
3982 for (unsigned i = 0, e = Stores.size(); i < e; ++i) {
3983 IndexQueue.clear();
3984 // If a store has multiple consecutive store candidates, search Stores
3985 // array according to the sequence: from i+1 to e, then from i-1 to 0.
3986 // This is because usually pairing with immediate succeeding or preceding
3987 // candidate create the best chance to find slp vectorization opportunity.
3988 unsigned j = 0;
3989 for (j = i + 1; j < e; ++j)
3990 IndexQueue.push_back(j);
3991 for (j = i; j > 0; --j)
3992 IndexQueue.push_back(j - 1);
3993
3994 for (auto &k : IndexQueue) {
3995 if (isConsecutiveAccess(Stores[i], Stores[k], *DL, *SE)) {
3996 Tails.insert(Stores[k]);
3997 Heads.insert(Stores[i]);
3998 ConsecutiveChain[Stores[i]] = Stores[k];
3999 break;
4000 }
4001 }
4002 }
4003
4004 // For stores that start but don't end a link in the chain:
4005 for (SetVector<StoreInst *>::iterator it = Heads.begin(), e = Heads.end();
4006 it != e; ++it) {
4007 if (Tails.count(*it))
4008 continue;
4009
4010 // We found a store instr that starts a chain. Now follow the chain and try
4011 // to vectorize it.
4012 BoUpSLP::ValueList Operands;
4013 StoreInst *I = *it;
4014 // Collect the chain into a list.
4015 while (Tails.count(I) || Heads.count(I)) {
4016 if (VectorizedStores.count(I))
4017 break;
4018 Operands.push_back(I);
4019 // Move to the next value in the chain.
4020 I = ConsecutiveChain[I];
4021 }
4022
4023 // FIXME: Is division-by-2 the correct step? Should we assert that the
4024 // register size is a power-of-2?
4025 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4026 Size /= 2) {
4027 if (vectorizeStoreChain(Operands, R, Size)) {
4028 // Mark the vectorized stores so that we don't vectorize them again.
4029 VectorizedStores.insert(Operands.begin(), Operands.end());
4030 Changed = true;
4031 break;
4032 }
4033 }
4034 }
4035
4036 return Changed;
4037}
4038
4039void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4040
4041 // Initialize the collections. We will make a single pass over the block.
4042 Stores.clear();
4043 GEPs.clear();
4044
4045 // Visit the store and getelementptr instructions in BB and organize them in
4046 // Stores and GEPs according to the underlying objects of their pointer
4047 // operands.
4048 for (Instruction &I : *BB) {
4049
4050 // Ignore store instructions that are volatile or have a pointer operand
4051 // that doesn't point to a scalar type.
4052 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4053 if (!SI->isSimple())
4054 continue;
4055 if (!isValidElementType(SI->getValueOperand()->getType()))
4056 continue;
4057 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4058 }
4059
4060 // Ignore getelementptr instructions that have more than one index, a
4061 // constant index, or a pointer operand that doesn't point to a scalar
4062 // type.
4063 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4064 auto Idx = GEP->idx_begin()->get();
4065 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4066 continue;
4067 if (!isValidElementType(Idx->getType()))
4068 continue;
4069 if (GEP->getType()->isVectorTy())
4070 continue;
4071 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4072 }
4073 }
4074}
4075
4076bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4077 if (!A || !B)
4078 return false;
4079 Value *VL[] = { A, B };
4080 return tryToVectorizeList(VL, R, None, true);
4081}
4082
4083bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4084 ArrayRef<Value *> BuildVector,
4085 bool AllowReorder) {
4086 if (VL.size() < 2)
4087 return false;
4088
4089 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
4090 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
4091
4092 // Check that all of the parts are scalar instructions of the same type.
4093 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4094 if (!I0)
4095 return false;
4096
4097 unsigned Opcode0 = I0->getOpcode();
4098
4099 unsigned Sz = R.getVectorElementSize(I0);
4100 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4101 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4102 if (MaxVF < 2)
4103 return false;
4104
4105 for (Value *V : VL) {
4106 Type *Ty = V->getType();
4107 if (!isValidElementType(Ty))
4108 return false;
4109 Instruction *Inst = dyn_cast<Instruction>(V);
4110 if (!Inst || Inst->getOpcode() != Opcode0)
4111 return false;
4112 }
4113
4114 bool Changed = false;
4115
4116 // Keep track of values that were deleted by vectorizing in the loop below.
4117 SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4118
4119 unsigned NextInst = 0, MaxInst = VL.size();
4120 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4121 VF /= 2) {
4122 // No actual vectorization should happen, if number of parts is the same as
4123 // provided vectorization factor (i.e. the scalar type is used for vector
4124 // code during codegen).
4125 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4126 if (TTI->getNumberOfParts(VecTy) == VF)
4127 continue;
4128 for (unsigned I = NextInst; I < MaxInst; ++I) {
4129 unsigned OpsWidth = 0;
4130
4131 if (I + VF > MaxInst)
4132 OpsWidth = MaxInst - I;
4133 else
4134 OpsWidth = VF;
4135
4136 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4137 break;
4138
4139 // Check that a previous iteration of this loop did not delete the Value.
4140 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4141 continue;
4142
4143 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
4144 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
4145 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4146
4147 ArrayRef<Value *> BuildVectorSlice;
4148 if (!BuildVector.empty())
4149 BuildVectorSlice = BuildVector.slice(I, OpsWidth);
4150
4151 R.buildTree(Ops, BuildVectorSlice);
4152 // TODO: check if we can allow reordering for more cases.
4153 if (AllowReorder && R.shouldReorder()) {
4154 // Conceptually, there is nothing actually preventing us from trying to
4155 // reorder a larger list. In fact, we do exactly this when vectorizing
4156 // reductions. However, at this point, we only expect to get here when
4157 // there are exactly two operations.
4158 assert(Ops.size() == 2)((Ops.size() == 2) ? static_cast<void> (0) : __assert_fail
("Ops.size() == 2", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4158, __PRETTY_FUNCTION__))
;
4159 assert(BuildVectorSlice.empty())((BuildVectorSlice.empty()) ? static_cast<void> (0) : __assert_fail
("BuildVectorSlice.empty()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4159, __PRETTY_FUNCTION__))
;
4160 Value *ReorderedOps[] = {Ops[1], Ops[0]};
4161 R.buildTree(ReorderedOps, None);
4162 }
4163 if (R.isTreeTinyAndNotFullyVectorizable())
4164 continue;
4165
4166 R.computeMinimumValueSizes();
4167 int Cost = R.getTreeCost();
4168
4169 if (Cost < -SLPCostThreshold) {
4170 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
4171 R.getORE()->emit(OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedList",
4172 cast<Instruction>(Ops[0]))
4173 << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4174 << " and with tree size "
4175 << ore::NV("TreeSize", R.getTreeSize()));
4176
4177 Value *VectorizedRoot = R.vectorizeTree();
4178
4179 // Reconstruct the build vector by extracting the vectorized root. This
4180 // way we handle the case where some elements of the vector are
4181 // undefined.
4182 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
4183 if (!BuildVectorSlice.empty()) {
4184 // The insert point is the last build vector instruction. The
4185 // vectorized root will precede it. This guarantees that we get an
4186 // instruction. The vectorized tree could have been constant folded.
4187 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4188 unsigned VecIdx = 0;
4189 for (auto &V : BuildVectorSlice) {
4190 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4191 ++BasicBlock::iterator(InsertAfter));
4192 Instruction *I = cast<Instruction>(V);
4193 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I))((isa<InsertElementInst>(I) || isa<InsertValueInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<InsertElementInst>(I) || isa<InsertValueInst>(I)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4193, __PRETTY_FUNCTION__))
;
4194 Instruction *Extract =
4195 cast<Instruction>(Builder.CreateExtractElement(
4196 VectorizedRoot, Builder.getInt32(VecIdx++)));
4197 I->setOperand(1, Extract);
4198 I->removeFromParent();
4199 I->insertAfter(Extract);
4200 InsertAfter = I;
4201 }
4202 }
4203 // Move to the next bundle.
4204 I += VF - 1;
4205 NextInst = I + 1;
4206 Changed = true;
4207 }
4208 }
4209 }
4210
4211 return Changed;
4212}
4213
4214bool SLPVectorizerPass::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
4215 if (!V)
4216 return false;
4217
4218 Value *P = V->getParent();
4219
4220 // Vectorize in current basic block only.
4221 auto *Op0 = dyn_cast<Instruction>(V->getOperand(0));
4222 auto *Op1 = dyn_cast<Instruction>(V->getOperand(1));
4223 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4224 return false;
4225
4226 // Try to vectorize V.
4227 if (tryToVectorizePair(Op0, Op1, R))
4228 return true;
4229
4230 auto *A = dyn_cast<BinaryOperator>(Op0);
4231 auto *B = dyn_cast<BinaryOperator>(Op1);
4232 // Try to skip B.
4233 if (B && B->hasOneUse()) {
4234 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4235 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4236 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4237 return true;
4238 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4239 return true;
4240 }
4241
4242 // Try to skip A.
4243 if (A && A->hasOneUse()) {
4244 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4245 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4246 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4247 return true;
4248 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4249 return true;
4250 }
4251 return false;
4252}
4253
4254/// \brief Generate a shuffle mask to be used in a reduction tree.
4255///
4256/// \param VecLen The length of the vector to be reduced.
4257/// \param NumEltsToRdx The number of elements that should be reduced in the
4258/// vector.
4259/// \param IsPairwise Whether the reduction is a pairwise or splitting
4260/// reduction. A pairwise reduction will generate a mask of
4261/// <0,2,...> or <1,3,..> while a splitting reduction will generate
4262/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4263/// \param IsLeft True will generate a mask of even elements, odd otherwise.
4264static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4265 bool IsPairwise, bool IsLeft,
4266 IRBuilder<> &Builder) {
4267 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"
) ? static_cast<void> (0) : __assert_fail ("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4267, __PRETTY_FUNCTION__))
;
4268
4269 SmallVector<Constant *, 32> ShuffleMask(
4270 VecLen, UndefValue::get(Builder.getInt32Ty()));
4271
4272 if (IsPairwise)
4273 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4274 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4275 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4276 else
4277 // Move the upper half of the vector to the lower half.
4278 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4279 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4280
4281 return ConstantVector::get(ShuffleMask);
4282}
4283
4284namespace {
4285/// Model horizontal reductions.
4286///
4287/// A horizontal reduction is a tree of reduction operations (currently add and
4288/// fadd) that has operations that can be put into a vector as its leaf.
4289/// For example, this tree:
4290///
4291/// mul mul mul mul
4292/// \ / \ /
4293/// + +
4294/// \ /
4295/// +
4296/// This tree has "mul" as its reduced values and "+" as its reduction
4297/// operations. A reduction might be feeding into a store or a binary operation
4298/// feeding a phi.
4299/// ...
4300/// \ /
4301/// +
4302/// |
4303/// phi +=
4304///
4305/// Or:
4306/// ...
4307/// \ /
4308/// +
4309/// |
4310/// *p =
4311///
4312class HorizontalReduction {
4313 SmallVector<Value *, 16> ReductionOps;
4314 SmallVector<Value *, 32> ReducedVals;
4315 // Use map vector to make stable output.
4316 MapVector<Instruction *, Value *> ExtraArgs;
4317
4318 BinaryOperator *ReductionRoot = nullptr;
4319
4320 /// The opcode of the reduction.
4321 Instruction::BinaryOps ReductionOpcode = Instruction::BinaryOpsEnd;
4322 /// The opcode of the values we perform a reduction on.
4323 unsigned ReducedValueOpcode = 0;
4324 /// Should we model this reduction as a pairwise reduction tree or a tree that
4325 /// splits the vector in halves and adds those halves.
4326 bool IsPairwiseReduction = false;
4327
4328 /// Checks if the ParentStackElem.first should be marked as a reduction
4329 /// operation with an extra argument or as extra argument itself.
4330 void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
4331 Value *ExtraArg) {
4332 if (ExtraArgs.count(ParentStackElem.first)) {
4333 ExtraArgs[ParentStackElem.first] = nullptr;
4334 // We ran into something like:
4335 // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
4336 // The whole ParentStackElem.first should be considered as an extra value
4337 // in this case.
4338 // Do not perform analysis of remaining operands of ParentStackElem.first
4339 // instruction, this whole instruction is an extra argument.
4340 ParentStackElem.second = ParentStackElem.first->getNumOperands();
4341 } else {
4342 // We ran into something like:
4343 // ParentStackElem.first += ... + ExtraArg + ...
4344 ExtraArgs[ParentStackElem.first] = ExtraArg;
4345 }
4346 }
4347
4348public:
4349 HorizontalReduction() = default;
4350
4351 /// \brief Try to find a reduction tree.
4352 bool matchAssociativeReduction(PHINode *Phi, BinaryOperator *B) {
4353 assert((!Phi || is_contained(Phi->operands(), B)) &&(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4354, __PRETTY_FUNCTION__))
4354 "Thi phi needs to use the binary operator")(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4354, __PRETTY_FUNCTION__))
;
4355
4356 // We could have a initial reductions that is not an add.
4357 // r *= v1 + v2 + v3 + v4
4358 // In such a case start looking for a tree rooted in the first '+'.
4359 if (Phi) {
4360 if (B->getOperand(0) == Phi) {
4361 Phi = nullptr;
4362 B = dyn_cast<BinaryOperator>(B->getOperand(1));
4363 } else if (B->getOperand(1) == Phi) {
4364 Phi = nullptr;
4365 B = dyn_cast<BinaryOperator>(B->getOperand(0));
4366 }
4367 }
4368
4369 if (!B)
4370 return false;
4371
4372 Type *Ty = B->getType();
4373 if (!isValidElementType(Ty))
4374 return false;
4375
4376 ReductionOpcode = B->getOpcode();
4377 ReducedValueOpcode = 0;
4378 ReductionRoot = B;
4379
4380 // We currently only support adds.
4381 if ((ReductionOpcode != Instruction::Add &&
4382 ReductionOpcode != Instruction::FAdd) ||
4383 !B->isAssociative())
4384 return false;
4385
4386 // Post order traverse the reduction tree starting at B. We only handle true
4387 // trees containing only binary operators or selects.
4388 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
4389 Stack.push_back(std::make_pair(B, 0));
4390 while (!Stack.empty()) {
4391 Instruction *TreeN = Stack.back().first;
4392 unsigned EdgeToVist = Stack.back().second++;
4393 bool IsReducedValue = TreeN->getOpcode() != ReductionOpcode;
4394
4395 // Postorder vist.
4396 if (EdgeToVist == 2 || IsReducedValue) {
4397 if (IsReducedValue)
4398 ReducedVals.push_back(TreeN);
4399 else {
4400 auto I = ExtraArgs.find(TreeN);
4401 if (I != ExtraArgs.end() && !I->second) {
4402 // Check if TreeN is an extra argument of its parent operation.
4403 if (Stack.size() <= 1) {
4404 // TreeN can't be an extra argument as it is a root reduction
4405 // operation.
4406 return false;
4407 }
4408 // Yes, TreeN is an extra argument, do not add it to a list of
4409 // reduction operations.
4410 // Stack[Stack.size() - 2] always points to the parent operation.
4411 markExtraArg(Stack[Stack.size() - 2], TreeN);
4412 ExtraArgs.erase(TreeN);
4413 } else
4414 ReductionOps.push_back(TreeN);
4415 }
4416 // Retract.
4417 Stack.pop_back();
4418 continue;
4419 }
4420
4421 // Visit left or right.
4422 Value *NextV = TreeN->getOperand(EdgeToVist);
4423 if (NextV != Phi) {
4424 auto *I = dyn_cast<Instruction>(NextV);
4425 // Continue analysis if the next operand is a reduction operation or
4426 // (possibly) a reduced value. If the reduced value opcode is not set,
4427 // the first met operation != reduction operation is considered as the
4428 // reduced value class.
4429 if (I && (!ReducedValueOpcode || I->getOpcode() == ReducedValueOpcode ||
4430 I->getOpcode() == ReductionOpcode)) {
4431 // Only handle trees in the current basic block.
4432 if (I->getParent() != B->getParent()) {
4433 // I is an extra argument for TreeN (its parent operation).
4434 markExtraArg(Stack.back(), I);
4435 continue;
4436 }
4437
4438 // Each tree node needs to have one user except for the ultimate
4439 // reduction.
4440 if (!I->hasOneUse() && I != B) {
4441 // I is an extra argument for TreeN (its parent operation).
4442 markExtraArg(Stack.back(), I);
4443 continue;
4444 }
4445
4446 if (I->getOpcode() == ReductionOpcode) {
4447 // We need to be able to reassociate the reduction operations.
4448 if (!I->isAssociative()) {
4449 // I is an extra argument for TreeN (its parent operation).
4450 markExtraArg(Stack.back(), I);
4451 continue;
4452 }
4453 } else if (ReducedValueOpcode &&
4454 ReducedValueOpcode != I->getOpcode()) {
4455 // Make sure that the opcodes of the operations that we are going to
4456 // reduce match.
4457 // I is an extra argument for TreeN (its parent operation).
4458 markExtraArg(Stack.back(), I);
4459 continue;
4460 } else if (!ReducedValueOpcode)
4461 ReducedValueOpcode = I->getOpcode();
4462
4463 Stack.push_back(std::make_pair(I, 0));
4464 continue;
4465 }
4466 }
4467 // NextV is an extra argument for TreeN (its parent operation).
4468 markExtraArg(Stack.back(), NextV);
4469 }
4470 return true;
4471 }
4472
4473 /// \brief Attempt to vectorize the tree found by
4474 /// matchAssociativeReduction.
4475 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
4476 if (ReducedVals.empty())
4477 return false;
4478
4479 // If there is a sufficient number of reduction values, reduce
4480 // to a nearby power-of-2. Can safely generate oversized
4481 // vectors and rely on the backend to split them to legal sizes.
4482 unsigned NumReducedVals = ReducedVals.size();
4483 if (NumReducedVals < 4)
4484 return false;
4485
4486 unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
4487
4488 Value *VectorizedTree = nullptr;
4489 IRBuilder<> Builder(ReductionRoot);
4490 FastMathFlags Unsafe;
4491 Unsafe.setUnsafeAlgebra();
4492 Builder.setFastMathFlags(Unsafe);
4493 unsigned i = 0;
4494
4495 BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
4496 // The same extra argument may be used several time, so log each attempt
4497 // to use it.
4498 for (auto &Pair : ExtraArgs)
4499 ExternallyUsedValues[Pair.second].push_back(Pair.first);
4500 while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
4501 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
4502 V.buildTree(VL, ExternallyUsedValues, ReductionOps);
4503 if (V.shouldReorder()) {
4504 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
4505 V.buildTree(Reversed, ExternallyUsedValues, ReductionOps);
4506 }
4507 if (V.isTreeTinyAndNotFullyVectorizable())
4508 break;
4509
4510 V.computeMinimumValueSizes();
4511
4512 // Estimate cost.
4513 int Cost =
4514 V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
4515 if (Cost >= -SLPCostThreshold)
4516 break;
4517
4518 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
4519 << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
;
4520 auto *I0 = cast<Instruction>(VL[0]);
4521 V.getORE()->emit(
4522 OptimizationRemark(SV_NAME"slp-vectorizer", "VectorizedHorizontalReduction", I0)
4523 << "Vectorized horizontal reduction with cost "
4524 << ore::NV("Cost", Cost) << " and with tree size "
4525 << ore::NV("TreeSize", V.getTreeSize()));
4526
4527 // Vectorize a tree.
4528 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
4529 Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
4530
4531 // Emit a reduction.
4532 Value *ReducedSubTree =
4533 emitReduction(VectorizedRoot, Builder, ReduxWidth, ReductionOps, TTI);
4534 if (VectorizedTree) {
4535 Builder.SetCurrentDebugLocation(Loc);
4536 VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree,
4537 ReducedSubTree, "bin.rdx");
4538 propagateIRFlags(VectorizedTree, ReductionOps);
4539 } else
4540 VectorizedTree = ReducedSubTree;
4541 i += ReduxWidth;
4542 ReduxWidth = PowerOf2Floor(NumReducedVals - i);
4543 }
4544
4545 if (VectorizedTree) {
4546 // Finish the reduction.
4547 for (; i < NumReducedVals; ++i) {
4548 auto *I = cast<Instruction>(ReducedVals[i]);
4549 Builder.SetCurrentDebugLocation(I->getDebugLoc());
4550 VectorizedTree =
4551 Builder.CreateBinOp(ReductionOpcode, VectorizedTree, I);
4552 propagateIRFlags(VectorizedTree, ReductionOps);
4553 }
4554 for (auto &Pair : ExternallyUsedValues) {
4555 assert(!Pair.second.empty() &&((!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? static_cast<void> (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4556, __PRETTY_FUNCTION__))
4556 "At least one DebugLoc must be inserted")((!Pair.second.empty() && "At least one DebugLoc must be inserted"
) ? static_cast<void> (0) : __assert_fail ("!Pair.second.empty() && \"At least one DebugLoc must be inserted\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4556, __PRETTY_FUNCTION__))
;
4557 // Add each externally used value to the final reduction.
4558 for (auto *I : Pair.second) {
4559 Builder.SetCurrentDebugLocation(I->getDebugLoc());
4560 VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree,
4561 Pair.first, "bin.extra");
4562 propagateIRFlags(VectorizedTree, I);
4563 }
4564 }
4565 // Update users.
4566 ReductionRoot->replaceAllUsesWith(VectorizedTree);
4567 }
4568 return VectorizedTree != nullptr;
4569 }
4570
4571 unsigned numReductionValues() const {
4572 return ReducedVals.size();
4573 }
4574
4575private:
4576 /// \brief Calculate the cost of a reduction.
4577 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
4578 unsigned ReduxWidth) {
4579 Type *ScalarTy = FirstReducedVal->getType();
4580 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
4581
4582 int PairwiseRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, true);
4583 int SplittingRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, false);
4584
4585 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
4586 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
4587
4588 int ScalarReduxCost =
4589 (ReduxWidth - 1) *
4590 TTI->getArithmeticInstrCost(ReductionOpcode, ScalarTy);
4591
4592 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4593 << " for reduction that starts with " << *FirstReducedValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4594 << " (It is a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4595 << (IsPairwiseReduction ? "pairwise" : "splitting")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4596 << " reduction)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
;
4597
4598 return VecReduxCost - ScalarReduxCost;
4599 }
4600
4601 /// \brief Emit a horizontal reduction of the vectorized value.
4602 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
4603 unsigned ReduxWidth, ArrayRef<Value *> RedOps,
4604 const TargetTransformInfo *TTI) {
4605 assert(VectorizedValue && "Need to have a vectorized tree node")((VectorizedValue && "Need to have a vectorized tree node"
) ? static_cast<void> (0) : __assert_fail ("VectorizedValue && \"Need to have a vectorized tree node\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4605, __PRETTY_FUNCTION__))
;
4606 assert(isPowerOf2_32(ReduxWidth) &&((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4607, __PRETTY_FUNCTION__))
4607 "We only handle power-of-two reductions for now")((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4607, __PRETTY_FUNCTION__))
;
4608
4609 if (!IsPairwiseReduction)
4610 return createSimpleTargetReduction(
4611 Builder, TTI, ReductionOpcode, VectorizedValue,
4612 TargetTransformInfo::ReductionFlags(), RedOps);
4613
4614 Value *TmpVec = VectorizedValue;
4615 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
4616 Value *LeftMask =
4617 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
4618 Value *RightMask =
4619 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
4620
4621 Value *LeftShuf = Builder.CreateShuffleVector(
4622 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
4623 Value *RightShuf = Builder.CreateShuffleVector(
4624 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
4625 "rdx.shuf.r");
4626 TmpVec =
4627 Builder.CreateBinOp(ReductionOpcode, LeftShuf, RightShuf, "bin.rdx");
4628 propagateIRFlags(TmpVec, RedOps);
4629 }
4630
4631 // The result is in the first element of the vector.
4632 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
4633 }
4634};
4635} // end anonymous namespace
4636
4637/// \brief Recognize construction of vectors like
4638/// %ra = insertelement <4 x float> undef, float %s0, i32 0
4639/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
4640/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
4641/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
4642///
4643/// Returns true if it matches
4644///
4645static bool findBuildVector(InsertElementInst *FirstInsertElem,
4646 SmallVectorImpl<Value *> &BuildVector,
4647 SmallVectorImpl<Value *> &BuildVectorOpds) {
4648 if (!isa<UndefValue>(FirstInsertElem->getOperand(0)))
4649 return false;
4650
4651 InsertElementInst *IE = FirstInsertElem;
4652 while (true) {
4653 BuildVector.push_back(IE);
4654 BuildVectorOpds.push_back(IE->getOperand(1));
4655
4656 if (IE->use_empty())
4657 return false;
4658
4659 InsertElementInst *NextUse = dyn_cast<InsertElementInst>(IE->user_back());
4660 if (!NextUse)
4661 return true;
4662
4663 // If this isn't the final use, make sure the next insertelement is the only
4664 // use. It's OK if the final constructed vector is used multiple times
4665 if (!IE->hasOneUse())
4666 return false;
4667
4668 IE = NextUse;
4669 }
4670
4671 return false;
4672}
4673
4674/// \brief Like findBuildVector, but looks backwards for construction of aggregate.
4675///
4676/// \return true if it matches.
4677static bool findBuildAggregate(InsertValueInst *IV,
4678 SmallVectorImpl<Value *> &BuildVector,
4679 SmallVectorImpl<Value *> &BuildVectorOpds) {
4680 Value *V;
4681 do {
4682 BuildVector.push_back(IV);
4683 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
4684 V = IV->getAggregateOperand();
4685 if (isa<UndefValue>(V))
4686 break;
4687 IV = dyn_cast<InsertValueInst>(V);
4688 if (!IV || !IV->hasOneUse())
4689 return false;
4690 } while (true);
4691 std::reverse(BuildVector.begin(), BuildVector.end());
4692 std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
4693 return true;
4694}
4695
4696static bool PhiTypeSorterFunc(Value *V, Value *V2) {
4697 return V->getType() < V2->getType();
4698}
4699
4700/// \brief Try and get a reduction value from a phi node.
4701///
4702/// Given a phi node \p P in a block \p ParentBB, consider possible reductions
4703/// if they come from either \p ParentBB or a containing loop latch.
4704///
4705/// \returns A candidate reduction value if possible, or \code nullptr \endcode
4706/// if not possible.
4707static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
4708 BasicBlock *ParentBB, LoopInfo *LI) {
4709 // There are situations where the reduction value is not dominated by the
4710 // reduction phi. Vectorizing such cases has been reported to cause
4711 // miscompiles. See PR25787.
4712 auto DominatedReduxValue = [&](Value *R) {
4713 return (
4714 dyn_cast<Instruction>(R) &&
4715 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
4716 };
4717
4718 Value *Rdx = nullptr;
4719
4720 // Return the incoming value if it comes from the same BB as the phi node.
4721 if (P->getIncomingBlock(0) == ParentBB) {
4722 Rdx = P->getIncomingValue(0);
4723 } else if (P->getIncomingBlock(1) == ParentBB) {
4724 Rdx = P->getIncomingValue(1);
4725 }
4726
4727 if (Rdx && DominatedReduxValue(Rdx))
4728 return Rdx;
4729
4730 // Otherwise, check whether we have a loop latch to look at.
4731 Loop *BBL = LI->getLoopFor(ParentBB);
4732 if (!BBL)
4733 return nullptr;
4734 BasicBlock *BBLatch = BBL->getLoopLatch();
4735 if (!BBLatch)
4736 return nullptr;
4737
4738 // There is a loop latch, return the incoming value if it comes from
4739 // that. This reduction pattern occasionally turns up.
4740 if (P->getIncomingBlock(0) == BBLatch) {
4741 Rdx = P->getIncomingValue(0);
4742 } else if (P->getIncomingBlock(1) == BBLatch) {
4743 Rdx = P->getIncomingValue(1);
4744 }
4745
4746 if (Rdx && DominatedReduxValue(Rdx))
4747 return Rdx;
4748
4749 return nullptr;
4750}
4751
4752namespace {
4753/// Tracks instructons and its children.
4754class WeakTrackingVHWithLevel final : public CallbackVH {
4755 /// Operand index of the instruction currently beeing analized.
4756 unsigned Level = 0;
4757 /// Is this the instruction that should be vectorized, or are we now
4758 /// processing children (i.e. operands of this instruction) for potential
4759 /// vectorization?
4760 bool IsInitial = true;
4761
4762public:
4763 explicit WeakTrackingVHWithLevel() = default;
4764 WeakTrackingVHWithLevel(Value *V) : CallbackVH(V){};
4765 /// Restart children analysis each time it is repaced by the new instruction.
4766 void allUsesReplacedWith(Value *New) override {
4767 setValPtr(New);
4768 Level = 0;
4769 IsInitial = true;
4770 }
4771 /// Check if the instruction was not deleted during vectorization.
4772 bool isValid() const { return !getValPtr(); }
4773 /// Is the istruction itself must be vectorized?
4774 bool isInitial() const { return IsInitial; }
4775 /// Try to vectorize children.
4776 void clearInitial() { IsInitial = false; }
4777 /// Are all children processed already?
4778 bool isFinal() const {
4779 assert(getValPtr() &&((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4781, __PRETTY_FUNCTION__))
4780 (isa<Instruction>(getValPtr()) &&((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4781, __PRETTY_FUNCTION__))
4781 cast<Instruction>(getValPtr())->getNumOperands() >= Level))((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4781, __PRETTY_FUNCTION__))
;
4782 return getValPtr() &&
4783 cast<Instruction>(getValPtr())->getNumOperands() == Level;
4784 }
4785 /// Get next child operation.
4786 Value *nextOperand() {
4787 assert(getValPtr() && isa<Instruction>(getValPtr()) &&((getValPtr() && isa<Instruction>(getValPtr()) &&
cast<Instruction>(getValPtr())->getNumOperands() >
Level) ? static_cast<void> (0) : __assert_fail ("getValPtr() && isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() > Level"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4788, __PRETTY_FUNCTION__))
4788 cast<Instruction>(getValPtr())->getNumOperands() > Level)((getValPtr() && isa<Instruction>(getValPtr()) &&
cast<Instruction>(getValPtr())->getNumOperands() >
Level) ? static_cast<void> (0) : __assert_fail ("getValPtr() && isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() > Level"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4788, __PRETTY_FUNCTION__))
;
4789 return cast<Instruction>(getValPtr())->getOperand(Level++);
4790 }
4791 virtual ~WeakTrackingVHWithLevel() = default;
4792};
4793} // namespace
4794
4795/// \brief Attempt to reduce a horizontal reduction.
4796/// If it is legal to match a horizontal reduction feeding
4797/// the phi node P with reduction operators Root in a basic block BB, then check
4798/// if it can be done.
4799/// \returns true if a horizontal reduction was matched and reduced.
4800/// \returns false if a horizontal reduction was not matched.
4801static bool canBeVectorized(
4802 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
4803 TargetTransformInfo *TTI,
4804 const function_ref<bool(BinaryOperator *, BoUpSLP &)> Vectorize) {
4805 if (!ShouldVectorizeHor)
4806 return false;
4807
4808 if (!Root)
4809 return false;
4810
4811 if (Root->getParent() != BB)
4812 return false;
4813 SmallVector<WeakTrackingVHWithLevel, 8> Stack(1, Root);
4814 SmallSet<Value *, 8> VisitedInstrs;
4815 bool Res = false;
4816 while (!Stack.empty()) {
4817 Value *V = Stack.back();
4818 if (!V) {
4819 Stack.pop_back();
4820 continue;
4821 }
4822 auto *Inst = dyn_cast<Instruction>(V);
4823 if (!Inst || isa<PHINode>(Inst)) {
4824 Stack.pop_back();
4825 continue;
4826 }
4827 if (Stack.back().isInitial()) {
4828 Stack.back().clearInitial();
4829 if (auto *BI = dyn_cast<BinaryOperator>(Inst)) {
4830 HorizontalReduction HorRdx;
4831 if (HorRdx.matchAssociativeReduction(P, BI)) {
4832 if (HorRdx.tryToReduce(R, TTI)) {
4833 Res = true;
4834 P = nullptr;
4835 continue;
4836 }
4837 }
4838 if (P) {
4839 Inst = dyn_cast<Instruction>(BI->getOperand(0));
4840 if (Inst == P)
4841 Inst = dyn_cast<Instruction>(BI->getOperand(1));
4842 if (!Inst) {
4843 P = nullptr;
4844 continue;
4845 }
4846 }
4847 }
4848 P = nullptr;
4849 if (Vectorize(dyn_cast<BinaryOperator>(Inst), R)) {
4850 Res = true;
4851 continue;
4852 }
4853 }
4854 if (Stack.back().isFinal()) {
4855 Stack.pop_back();
4856 continue;
4857 }
4858
4859 if (auto *NextV = dyn_cast<Instruction>(Stack.back().nextOperand()))
4860 if (NextV->getParent() == BB && VisitedInstrs.insert(NextV).second &&
4861 Stack.size() < RecursionMaxDepth)
4862 Stack.push_back(NextV);
4863 }
4864 return Res;
4865}
4866
4867bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
4868 BasicBlock *BB, BoUpSLP &R,
4869 TargetTransformInfo *TTI) {
4870 if (!V)
4871 return false;
4872 auto *I = dyn_cast<Instruction>(V);
4873 if (!I)
4874 return false;
4875
4876 if (!isa<BinaryOperator>(I))
4877 P = nullptr;
4878 // Try to match and vectorize a horizontal reduction.
4879 return canBeVectorized(P, I, BB, R, TTI,
4880 [this](BinaryOperator *BI, BoUpSLP &R) -> bool {
4881 return tryToVectorize(BI, R);
4882 });
4883}
4884
4885bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
4886 bool Changed = false;
4887 SmallVector<Value *, 4> Incoming;
4888 SmallSet<Value *, 16> VisitedInstrs;
4889
4890 bool HaveVectorizedPhiNodes = true;
4891 while (HaveVectorizedPhiNodes) {
4892 HaveVectorizedPhiNodes = false;
4893
4894 // Collect the incoming values from the PHIs.
4895 Incoming.clear();
4896 for (Instruction &I : *BB) {
4897 PHINode *P = dyn_cast<PHINode>(&I);
4898 if (!P)
4899 break;
4900
4901 if (!VisitedInstrs.count(P))
4902 Incoming.push_back(P);
4903 }
4904
4905 // Sort by type.
4906 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
4907
4908 // Try to vectorize elements base on their type.
4909 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
4910 E = Incoming.end();
4911 IncIt != E;) {
4912
4913 // Look for the next elements with the same type.
4914 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
4915 while (SameTypeIt != E &&
4916 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
4917 VisitedInstrs.insert(*SameTypeIt);
4918 ++SameTypeIt;
4919 }
4920
4921 // Try to vectorize them.
4922 unsigned NumElts = (SameTypeIt - IncIt);
4923 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { errs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false)
;
4924 // The order in which the phi nodes appear in the program does not matter.
4925 // So allow tryToVectorizeList to reorder them if it is beneficial. This
4926 // is done when there are exactly two elements since tryToVectorizeList
4927 // asserts that there are only two values when AllowReorder is true.
4928 bool AllowReorder = NumElts == 2;
4929 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
4930 None, AllowReorder)) {
4931 // Success start over because instructions might have been changed.
4932 HaveVectorizedPhiNodes = true;
4933 Changed = true;
4934 break;
4935 }
4936
4937 // Start over at the next instruction of a different type (or the end).
4938 IncIt = SameTypeIt;
4939 }
4940 }
4941
4942 VisitedInstrs.clear();
4943
4944 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
4945 // We may go through BB multiple times so skip the one we have checked.
4946 if (!VisitedInstrs.insert(&*it).second)
4947 continue;
4948
4949 if (isa<DbgInfoIntrinsic>(it))
4950 continue;
4951
4952 // Try to vectorize reductions that use PHINodes.
4953 if (PHINode *P = dyn_cast<PHINode>(it)) {
4954 // Check that the PHI is a reduction PHI.
4955 if (P->getNumIncomingValues() != 2)
4956 return Changed;
4957
4958 // Try to match and vectorize a horizontal reduction.
4959 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
4960 TTI)) {
4961 Changed = true;
4962 it = BB->begin();
4963 e = BB->end();
4964 continue;
4965 }
4966 continue;
4967 }
4968
4969 if (ShouldStartVectorizeHorAtStore) {
4970 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
4971 // Try to match and vectorize a horizontal reduction.
4972 if (vectorizeRootInstruction(nullptr, SI->getValueOperand(), BB, R,
4973 TTI)) {
4974 Changed = true;
4975 it = BB->begin();
4976 e = BB->end();
4977 continue;
4978 }
4979 }
4980 }
4981
4982 // Try to vectorize horizontal reductions feeding into a return.
4983 if (ReturnInst *RI = dyn_cast<ReturnInst>(it)) {
4984 if (RI->getNumOperands() != 0) {
4985 // Try to match and vectorize a horizontal reduction.
4986 if (vectorizeRootInstruction(nullptr, RI->getOperand(0), BB, R, TTI)) {
4987 Changed = true;
4988 it = BB->begin();
4989 e = BB->end();
4990 continue;
4991 }
4992 }
4993 }
4994
4995 // Try to vectorize trees that start at compare instructions.
4996 if (CmpInst *CI = dyn_cast<CmpInst>(it)) {
4997 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R)) {
4998 Changed = true;
4999 // We would like to start over since some instructions are deleted
5000 // and the iterator may become invalid value.
5001 it = BB->begin();
5002 e = BB->end();
5003 continue;
5004 }
5005
5006 for (int I = 0; I < 2; ++I) {
5007 if (vectorizeRootInstruction(nullptr, CI->getOperand(I), BB, R, TTI)) {
5008 Changed = true;
5009 // We would like to start over since some instructions are deleted
5010 // and the iterator may become invalid value.
5011 it = BB->begin();
5012 e = BB->end();
5013 break;
5014 }
5015 }
5016 continue;
5017 }
5018
5019 // Try to vectorize trees that start at insertelement instructions.
5020 if (InsertElementInst *FirstInsertElem = dyn_cast<InsertElementInst>(it)) {
5021 SmallVector<Value *, 16> BuildVector;
5022 SmallVector<Value *, 16> BuildVectorOpds;
5023 if (!findBuildVector(FirstInsertElem, BuildVector, BuildVectorOpds))
5024 continue;
5025
5026 // Vectorize starting with the build vector operands ignoring the
5027 // BuildVector instructions for the purpose of scheduling and user
5028 // extraction.
5029 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector)) {
5030 Changed = true;
5031 it = BB->begin();
5032 e = BB->end();
5033 }
5034
5035 continue;
5036 }
5037
5038 // Try to vectorize trees that start at insertvalue instructions feeding into
5039 // a store.
5040 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
5041 if (InsertValueInst *LastInsertValue = dyn_cast<InsertValueInst>(SI->getValueOperand())) {
5042 const DataLayout &DL = BB->getModule()->getDataLayout();
5043 if (R.canMapToVector(SI->getValueOperand()->getType(), DL)) {
5044 SmallVector<Value *, 16> BuildVector;
5045 SmallVector<Value *, 16> BuildVectorOpds;
5046 if (!findBuildAggregate(LastInsertValue, BuildVector, BuildVectorOpds))
5047 continue;
5048
5049 DEBUG(dbgs() << "SLP: store of array mappable to vector: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: store of array mappable to vector: "
<< *SI << "\n"; } } while (false)
;
5050 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector, false)) {
5051 Changed = true;
5052 it = BB->begin();
5053 e = BB->end();
5054 }
5055 continue;
5056 }
5057 }
5058 }
5059 }
5060
5061 return Changed;
5062}
5063
5064bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
5065 auto Changed = false;
5066 for (auto &Entry : GEPs) {
5067
5068 // If the getelementptr list has fewer than two elements, there's nothing
5069 // to do.
5070 if (Entry.second.size() < 2)
5071 continue;
5072
5073 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
5074 << Entry.second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
;
5075
5076 // We process the getelementptr list in chunks of 16 (like we do for
5077 // stores) to minimize compile-time.
5078 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
5079 auto Len = std::min<unsigned>(BE - BI, 16);
5080 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
5081
5082 // Initialize a set a candidate getelementptrs. Note that we use a
5083 // SetVector here to preserve program order. If the index computations
5084 // are vectorizable and begin with loads, we want to minimize the chance
5085 // of having to reorder them later.
5086 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
5087
5088 // Some of the candidates may have already been vectorized after we
5089 // initially collected them. If so, the WeakTrackingVHs will have
5090 // nullified the
5091 // values, so remove them from the set of candidates.
5092 Candidates.remove(nullptr);
5093
5094 // Remove from the set of candidates all pairs of getelementptrs with
5095 // constant differences. Such getelementptrs are likely not good
5096 // candidates for vectorization in a bottom-up phase since one can be
5097 // computed from the other. We also ensure all candidate getelementptr
5098 // indices are unique.
5099 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
5100 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
5101 if (!Candidates.count(GEPI))
5102 continue;
5103 auto *SCEVI = SE->getSCEV(GEPList[I]);
5104 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
5105 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
5106 auto *SCEVJ = SE->getSCEV(GEPList[J]);
5107 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
5108 Candidates.remove(GEPList[I]);
5109 Candidates.remove(GEPList[J]);
5110 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
5111 Candidates.remove(GEPList[J]);
5112 }
5113 }
5114 }
5115
5116 // We break out of the above computation as soon as we know there are
5117 // fewer than two candidates remaining.
5118 if (Candidates.size() < 2)
5119 continue;
5120
5121 // Add the single, non-constant index of each candidate to the bundle. We
5122 // ensured the indices met these constraints when we originally collected
5123 // the getelementptrs.
5124 SmallVector<Value *, 16> Bundle(Candidates.size());
5125 auto BundleIndex = 0u;
5126 for (auto *V : Candidates) {
5127 auto *GEP = cast<GetElementPtrInst>(V);
5128 auto *GEPIdx = GEP->idx_begin()->get();
5129 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx))((GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx
)) ? static_cast<void> (0) : __assert_fail ("GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)"
, "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303373/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 5129, __PRETTY_FUNCTION__))
;
5130 Bundle[BundleIndex++] = GEPIdx;
5131 }
5132
5133 // Try and vectorize the indices. We are currently only interested in
5134 // gather-like cases of the form:
5135 //
5136 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
5137 //
5138 // where the loads of "a", the loads of "b", and the subtractions can be
5139 // performed in parallel. It's likely that detecting this pattern in a
5140 // bottom-up phase will be simpler and less costly than building a
5141 // full-blown top-down phase beginning at the consecutive loads.
5142 Changed |= tryToVectorizeList(Bundle, R);
5143 }
5144 }
5145 return Changed;
5146}
5147
5148bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
5149 bool Changed = false;
5150 // Attempt to sort and vectorize each of the store-groups.
5151 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
5152 ++it) {
5153 if (it->second.size() < 2)
5154 continue;
5155
5156 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
5157 << it->second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
;
5158
5159 // Process the stores in chunks of 16.
5160 // TODO: The limit of 16 inhibits greater vectorization factors.
5161 // For example, AVX2 supports v32i8. Increasing this limit, however,
5162 // may cause a significant compile-time increase.
5163 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
5164 unsigned Len = std::min<unsigned>(CE - CI, 16);
5165 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
5166 }
5167 }
5168 return Changed;
5169}
5170
5171char SLPVectorizer::ID = 0;
5172static const char lv_name[] = "SLP Vectorizer";
5173INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)static void *initializeSLPVectorizerPassOnce(PassRegistry &
Registry) {
5174INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
5175INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
5176INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
5177INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
5178INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry);
5179INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
5180INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
5181INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "slp-vectorizer", &
SLPVectorizer::ID, PassInfo::NormalCtor_t(callDefaultCtor<
SLPVectorizer>), false, false); Registry.registerPass(*PI,
true); return PI; } static llvm::once_flag InitializeSLPVectorizerPassFlag
; void llvm::initializeSLPVectorizerPass(PassRegistry &Registry
) { llvm::call_once(InitializeSLPVectorizerPassFlag, initializeSLPVectorizerPassOnce
, std::ref(Registry)); }
5182
5183namespace llvm {
5184Pass *createSLPVectorizerPass() { return new SLPVectorizer(); }
5185}