Bug Summary

File:lib/Transforms/Vectorize/SLPVectorizer.cpp
Warning:line 2896, column 22
Called C++ object pointer is null

Annotated Source Code

1//===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9// This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10// stores that can be put together into vector-stores. Next, it attempts to
11// construct vectorizable tree using the use-def chains. If a profitable tree
12// was found, the SLP vectorizer performs vectorization on the tree.
13//
14// The pass is inspired by the work described in the paper:
15// "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16//
17//===----------------------------------------------------------------------===//
18#include "llvm/Transforms/Vectorize/SLPVectorizer.h"
19#include "llvm/ADT/Optional.h"
20#include "llvm/ADT/PostOrderIterator.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/CodeMetrics.h"
24#include "llvm/Analysis/GlobalsModRef.h"
25#include "llvm/Analysis/LoopAccessAnalysis.h"
26#include "llvm/Analysis/ScalarEvolutionExpressions.h"
27#include "llvm/Analysis/ValueTracking.h"
28#include "llvm/Analysis/VectorUtils.h"
29#include "llvm/IR/DataLayout.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/Instructions.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Module.h"
35#include "llvm/IR/NoFolder.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/Value.h"
38#include "llvm/IR/Verifier.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/CommandLine.h"
41#include "llvm/Support/Debug.h"
42#include "llvm/Support/raw_ostream.h"
43#include "llvm/Transforms/Vectorize.h"
44#include <algorithm>
45#include <memory>
46
47using namespace llvm;
48using namespace slpvectorizer;
49
50#define SV_NAME"slp-vectorizer" "slp-vectorizer"
51#define DEBUG_TYPE"SLP" "SLP"
52
53STATISTIC(NumVectorInstructions, "Number of vector instructions generated")static llvm::Statistic NumVectorInstructions = {"SLP", "NumVectorInstructions"
, "Number of vector instructions generated", {0}, false}
;
54
55static cl::opt<int>
56 SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
57 cl::desc("Only vectorize if you gain more than this "
58 "number "));
59
60static cl::opt<bool>
61ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
62 cl::desc("Attempt to vectorize horizontal reductions"));
63
64static cl::opt<bool> ShouldStartVectorizeHorAtStore(
65 "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
66 cl::desc(
67 "Attempt to vectorize horizontal reductions feeding into a store"));
68
69static cl::opt<int>
70MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
71 cl::desc("Attempt to vectorize for this register size in bits"));
72
73/// Limits the size of scheduling regions in a block.
74/// It avoid long compile times for _very_ large blocks where vector
75/// instructions are spread over a wide range.
76/// This limit is way higher than needed by real-world functions.
77static cl::opt<int>
78ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
79 cl::desc("Limit the size of the SLP scheduling region per block"));
80
81static cl::opt<int> MinVectorRegSizeOption(
82 "slp-min-reg-size", cl::init(128), cl::Hidden,
83 cl::desc("Attempt to vectorize for this register size in bits"));
84
85static cl::opt<unsigned> RecursionMaxDepth(
86 "slp-recursion-max-depth", cl::init(12), cl::Hidden,
87 cl::desc("Limit the recursion depth when building a vectorizable tree"));
88
89static cl::opt<unsigned> MinTreeSize(
90 "slp-min-tree-size", cl::init(3), cl::Hidden,
91 cl::desc("Only vectorize small trees if they are fully vectorizable"));
92
93// Limit the number of alias checks. The limit is chosen so that
94// it has no negative effect on the llvm benchmarks.
95static const unsigned AliasedCheckLimit = 10;
96
97// Another limit for the alias checks: The maximum distance between load/store
98// instructions where alias checks are done.
99// This limit is useful for very large basic blocks.
100static const unsigned MaxMemDepDistance = 160;
101
102/// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
103/// regions to be handled.
104static const int MinScheduleRegionSize = 16;
105
106/// \brief Predicate for the element types that the SLP vectorizer supports.
107///
108/// The most important thing to filter here are types which are invalid in LLVM
109/// vectors. We also filter target specific types which have absolutely no
110/// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
111/// avoids spending time checking the cost model and realizing that they will
112/// be inevitably scalarized.
113static bool isValidElementType(Type *Ty) {
114 return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
115 !Ty->isPPC_FP128Ty();
116}
117
118/// \returns true if all of the instructions in \p VL are in the same block or
119/// false otherwise.
120static bool allSameBlock(ArrayRef<Value *> VL) {
121 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
122 if (!I0)
123 return false;
124 BasicBlock *BB = I0->getParent();
125 for (int i = 1, e = VL.size(); i < e; i++) {
126 Instruction *I = dyn_cast<Instruction>(VL[i]);
127 if (!I)
128 return false;
129
130 if (BB != I->getParent())
131 return false;
132 }
133 return true;
134}
135
136/// \returns True if all of the values in \p VL are constants.
137static bool allConstant(ArrayRef<Value *> VL) {
138 for (Value *i : VL)
139 if (!isa<Constant>(i))
140 return false;
141 return true;
142}
143
144/// \returns True if all of the values in \p VL are identical.
145static bool isSplat(ArrayRef<Value *> VL) {
146 for (unsigned i = 1, e = VL.size(); i < e; ++i)
147 if (VL[i] != VL[0])
148 return false;
149 return true;
150}
151
152///\returns Opcode that can be clubbed with \p Op to create an alternate
153/// sequence which can later be merged as a ShuffleVector instruction.
154static unsigned getAltOpcode(unsigned Op) {
155 switch (Op) {
156 case Instruction::FAdd:
157 return Instruction::FSub;
158 case Instruction::FSub:
159 return Instruction::FAdd;
160 case Instruction::Add:
161 return Instruction::Sub;
162 case Instruction::Sub:
163 return Instruction::Add;
164 default:
165 return 0;
166 }
167}
168
169///\returns bool representing if Opcode \p Op can be part
170/// of an alternate sequence which can later be merged as
171/// a ShuffleVector instruction.
172static bool canCombineAsAltInst(unsigned Op) {
173 return Op == Instruction::FAdd || Op == Instruction::FSub ||
174 Op == Instruction::Sub || Op == Instruction::Add;
175}
176
177/// \returns ShuffleVector instruction if instructions in \p VL have
178/// alternate fadd,fsub / fsub,fadd/add,sub/sub,add sequence.
179/// (i.e. e.g. opcodes of fadd,fsub,fadd,fsub...)
180static unsigned isAltInst(ArrayRef<Value *> VL) {
181 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
182 unsigned Opcode = I0->getOpcode();
183 unsigned AltOpcode = getAltOpcode(Opcode);
184 for (int i = 1, e = VL.size(); i < e; i++) {
185 Instruction *I = dyn_cast<Instruction>(VL[i]);
186 if (!I || I->getOpcode() != ((i & 1) ? AltOpcode : Opcode))
187 return 0;
188 }
189 return Instruction::ShuffleVector;
190}
191
192/// \returns The opcode if all of the Instructions in \p VL have the same
193/// opcode, or zero.
194static unsigned getSameOpcode(ArrayRef<Value *> VL) {
195 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
196 if (!I0)
197 return 0;
198 unsigned Opcode = I0->getOpcode();
199 for (int i = 1, e = VL.size(); i < e; i++) {
200 Instruction *I = dyn_cast<Instruction>(VL[i]);
201 if (!I || Opcode != I->getOpcode()) {
202 if (canCombineAsAltInst(Opcode) && i == 1)
203 return isAltInst(VL);
204 return 0;
205 }
206 }
207 return Opcode;
208}
209
210/// Get the intersection (logical and) of all of the potential IR flags
211/// of each scalar operation (VL) that will be converted into a vector (I).
212/// Flag set: NSW, NUW, exact, and all of fast-math.
213static void propagateIRFlags(Value *I, ArrayRef<Value *> VL) {
214 if (auto *VecOp = dyn_cast<Instruction>(I)) {
215 if (auto *Intersection = dyn_cast<Instruction>(VL[0])) {
216 // Intersection is initialized to the 0th scalar,
217 // so start counting from index '1'.
218 for (int i = 1, e = VL.size(); i < e; ++i) {
219 if (auto *Scalar = dyn_cast<Instruction>(VL[i]))
220 Intersection->andIRFlags(Scalar);
221 }
222 VecOp->copyIRFlags(Intersection);
223 }
224 }
225}
226
227/// \returns true if all of the values in \p VL have the same type or false
228/// otherwise.
229static bool allSameType(ArrayRef<Value *> VL) {
230 Type *Ty = VL[0]->getType();
231 for (int i = 1, e = VL.size(); i < e; i++)
232 if (VL[i]->getType() != Ty)
233 return false;
234
235 return true;
236}
237
238/// \returns True if Extract{Value,Element} instruction extracts element Idx.
239static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
240 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 241, __PRETTY_FUNCTION__))
241 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 241, __PRETTY_FUNCTION__))
;
242 if (Opcode == Instruction::ExtractElement) {
243 ConstantInt *CI = dyn_cast<ConstantInt>(E->getOperand(1));
244 return CI && CI->getZExtValue() == Idx;
245 } else {
246 ExtractValueInst *EI = cast<ExtractValueInst>(E);
247 return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
248 }
249}
250
251/// \returns True if in-tree use also needs extract. This refers to
252/// possible scalar operand in vectorized instruction.
253static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
254 TargetLibraryInfo *TLI) {
255
256 unsigned Opcode = UserInst->getOpcode();
257 switch (Opcode) {
258 case Instruction::Load: {
259 LoadInst *LI = cast<LoadInst>(UserInst);
260 return (LI->getPointerOperand() == Scalar);
261 }
262 case Instruction::Store: {
263 StoreInst *SI = cast<StoreInst>(UserInst);
264 return (SI->getPointerOperand() == Scalar);
265 }
266 case Instruction::Call: {
267 CallInst *CI = cast<CallInst>(UserInst);
268 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
269 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
270 return (CI->getArgOperand(1) == Scalar);
271 }
272 }
273 default:
274 return false;
275 }
276}
277
278/// \returns the AA location that is being access by the instruction.
279static MemoryLocation getLocation(Instruction *I, AliasAnalysis *AA) {
280 if (StoreInst *SI = dyn_cast<StoreInst>(I))
281 return MemoryLocation::get(SI);
282 if (LoadInst *LI = dyn_cast<LoadInst>(I))
283 return MemoryLocation::get(LI);
284 return MemoryLocation();
285}
286
287/// \returns True if the instruction is not a volatile or atomic load/store.
288static bool isSimple(Instruction *I) {
289 if (LoadInst *LI = dyn_cast<LoadInst>(I))
290 return LI->isSimple();
291 if (StoreInst *SI = dyn_cast<StoreInst>(I))
292 return SI->isSimple();
293 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
294 return !MI->isVolatile();
295 return true;
296}
297
298namespace llvm {
299namespace slpvectorizer {
300/// Bottom Up SLP Vectorizer.
301class BoUpSLP {
302public:
303 typedef SmallVector<Value *, 8> ValueList;
304 typedef SmallVector<Instruction *, 16> InstrList;
305 typedef SmallPtrSet<Value *, 16> ValueSet;
306 typedef SmallVector<StoreInst *, 8> StoreList;
307
308 BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
309 TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
310 DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB,
311 const DataLayout *DL)
312 : NumLoadsWantToKeepOrder(0), NumLoadsWantToChangeOrder(0), F(Func),
313 SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC), DB(DB),
314 DL(DL), Builder(Se->getContext()) {
315 CodeMetrics::collectEphemeralValues(F, AC, EphValues);
316 // Use the vector register size specified by the target unless overridden
317 // by a command-line option.
318 // TODO: It would be better to limit the vectorization factor based on
319 // data type rather than just register size. For example, x86 AVX has
320 // 256-bit registers, but it does not support integer operations
321 // at that width (that requires AVX2).
322 if (MaxVectorRegSizeOption.getNumOccurrences())
323 MaxVecRegSize = MaxVectorRegSizeOption;
324 else
325 MaxVecRegSize = TTI->getRegisterBitWidth(true);
326
327 MinVecRegSize = MinVectorRegSizeOption;
328 }
329
330 /// \brief Vectorize the tree that starts with the elements in \p VL.
331 /// Returns the vectorized root.
332 Value *vectorizeTree();
333
334 /// \returns the cost incurred by unwanted spills and fills, caused by
335 /// holding live values over call sites.
336 int getSpillCost();
337
338 /// \returns the vectorization cost of the subtree that starts at \p VL.
339 /// A negative number means that this is profitable.
340 int getTreeCost();
341
342 /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
343 /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
344 void buildTree(ArrayRef<Value *> Roots,
345 ArrayRef<Value *> UserIgnoreLst = None);
346
347 /// Clear the internal data structures that are created by 'buildTree'.
348 void deleteTree() {
349 VectorizableTree.clear();
350 ScalarToTreeEntry.clear();
351 MustGather.clear();
352 ExternalUses.clear();
353 NumLoadsWantToKeepOrder = 0;
354 NumLoadsWantToChangeOrder = 0;
355 for (auto &Iter : BlocksSchedules) {
356 BlockScheduling *BS = Iter.second.get();
357 BS->clear();
358 }
359 MinBWs.clear();
360 }
361
362 /// \brief Perform LICM and CSE on the newly generated gather sequences.
363 void optimizeGatherSequence();
364
365 /// \returns true if it is beneficial to reverse the vector order.
366 bool shouldReorder() const {
367 return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
368 }
369
370 /// \return The vector element size in bits to use when vectorizing the
371 /// expression tree ending at \p V. If V is a store, the size is the width of
372 /// the stored value. Otherwise, the size is the width of the largest loaded
373 /// value reaching V. This method is used by the vectorizer to calculate
374 /// vectorization factors.
375 unsigned getVectorElementSize(Value *V);
376
377 /// Compute the minimum type sizes required to represent the entries in a
378 /// vectorizable tree.
379 void computeMinimumValueSizes();
380
381 // \returns maximum vector register size as set by TTI or overridden by cl::opt.
382 unsigned getMaxVecRegSize() const {
383 return MaxVecRegSize;
384 }
385
386 // \returns minimum vector register size as set by cl::opt.
387 unsigned getMinVecRegSize() const {
388 return MinVecRegSize;
389 }
390
391 /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
392 ///
393 /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
394 unsigned canMapToVector(Type *T, const DataLayout &DL) const;
395
396 /// \returns True if the VectorizableTree is both tiny and not fully
397 /// vectorizable. We do not vectorize such trees.
398 bool isTreeTinyAndNotFullyVectorizable();
399
400private:
401 struct TreeEntry;
402
403 /// \returns the cost of the vectorizable entry.
404 int getEntryCost(TreeEntry *E);
405
406 /// This is the recursive part of buildTree.
407 void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth);
408
409 /// \returns True if the ExtractElement/ExtractValue instructions in VL can
410 /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
411 bool canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const;
412
413 /// Vectorize a single entry in the tree.
414 Value *vectorizeTree(TreeEntry *E);
415
416 /// Vectorize a single entry in the tree, starting in \p VL.
417 Value *vectorizeTree(ArrayRef<Value *> VL);
418
419 /// \returns the pointer to the vectorized value if \p VL is already
420 /// vectorized, or NULL. They may happen in cycles.
421 Value *alreadyVectorized(ArrayRef<Value *> VL) const;
422
423 /// \returns the scalarization cost for this type. Scalarization in this
424 /// context means the creation of vectors from a group of scalars.
425 int getGatherCost(Type *Ty);
426
427 /// \returns the scalarization cost for this list of values. Assuming that
428 /// this subtree gets vectorized, we may need to extract the values from the
429 /// roots. This method calculates the cost of extracting the values.
430 int getGatherCost(ArrayRef<Value *> VL);
431
432 /// \brief Set the Builder insert point to one after the last instruction in
433 /// the bundle
434 void setInsertPointAfterBundle(ArrayRef<Value *> VL);
435
436 /// \returns a vector from a collection of scalars in \p VL.
437 Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
438
439 /// \returns whether the VectorizableTree is fully vectorizable and will
440 /// be beneficial even the tree height is tiny.
441 bool isFullyVectorizableTinyTree();
442
443 /// \reorder commutative operands in alt shuffle if they result in
444 /// vectorized code.
445 void reorderAltShuffleOperands(ArrayRef<Value *> VL,
446 SmallVectorImpl<Value *> &Left,
447 SmallVectorImpl<Value *> &Right);
448 /// \reorder commutative operands to get better probability of
449 /// generating vectorized code.
450 void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
451 SmallVectorImpl<Value *> &Left,
452 SmallVectorImpl<Value *> &Right);
453 struct TreeEntry {
454 TreeEntry() : Scalars(), VectorizedValue(nullptr),
455 NeedToGather(0) {}
456
457 /// \returns true if the scalars in VL are equal to this entry.
458 bool isSame(ArrayRef<Value *> VL) const {
459 assert(VL.size() == Scalars.size() && "Invalid size")((VL.size() == Scalars.size() && "Invalid size") ? static_cast
<void> (0) : __assert_fail ("VL.size() == Scalars.size() && \"Invalid size\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 459, __PRETTY_FUNCTION__))
;
460 return std::equal(VL.begin(), VL.end(), Scalars.begin());
461 }
462
463 /// A vector of scalars.
464 ValueList Scalars;
465
466 /// The Scalars are vectorized into this value. It is initialized to Null.
467 Value *VectorizedValue;
468
469 /// Do we need to gather this sequence ?
470 bool NeedToGather;
471 };
472
473 /// Create a new VectorizableTree entry.
474 TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized) {
475 VectorizableTree.emplace_back();
476 int idx = VectorizableTree.size() - 1;
477 TreeEntry *Last = &VectorizableTree[idx];
478 Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
479 Last->NeedToGather = !Vectorized;
480 if (Vectorized) {
481 for (int i = 0, e = VL.size(); i != e; ++i) {
482 assert(!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!")((!ScalarToTreeEntry.count(VL[i]) && "Scalar already in tree!"
) ? static_cast<void> (0) : __assert_fail ("!ScalarToTreeEntry.count(VL[i]) && \"Scalar already in tree!\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 482, __PRETTY_FUNCTION__))
;
483 ScalarToTreeEntry[VL[i]] = idx;
484 }
485 } else {
486 MustGather.insert(VL.begin(), VL.end());
487 }
488 return Last;
489 }
490
491 /// -- Vectorization State --
492 /// Holds all of the tree entries.
493 std::vector<TreeEntry> VectorizableTree;
494
495 /// Maps a specific scalar to its tree entry.
496 SmallDenseMap<Value*, int> ScalarToTreeEntry;
497
498 /// A list of scalars that we found that we need to keep as scalars.
499 ValueSet MustGather;
500
501 /// This POD struct describes one external user in the vectorized tree.
502 struct ExternalUser {
503 ExternalUser (Value *S, llvm::User *U, int L) :
504 Scalar(S), User(U), Lane(L){}
505 // Which scalar in our function.
506 Value *Scalar;
507 // Which user that uses the scalar.
508 llvm::User *User;
509 // Which lane does the scalar belong to.
510 int Lane;
511 };
512 typedef SmallVector<ExternalUser, 16> UserList;
513
514 /// Checks if two instructions may access the same memory.
515 ///
516 /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
517 /// is invariant in the calling loop.
518 bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
519 Instruction *Inst2) {
520
521 // First check if the result is already in the cache.
522 AliasCacheKey key = std::make_pair(Inst1, Inst2);
523 Optional<bool> &result = AliasCache[key];
524 if (result.hasValue()) {
525 return result.getValue();
526 }
527 MemoryLocation Loc2 = getLocation(Inst2, AA);
528 bool aliased = true;
529 if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
530 // Do the alias check.
531 aliased = AA->alias(Loc1, Loc2);
532 }
533 // Store the result in the cache.
534 result = aliased;
535 return aliased;
536 }
537
538 typedef std::pair<Instruction *, Instruction *> AliasCacheKey;
539
540 /// Cache for alias results.
541 /// TODO: consider moving this to the AliasAnalysis itself.
542 DenseMap<AliasCacheKey, Optional<bool>> AliasCache;
543
544 /// Removes an instruction from its block and eventually deletes it.
545 /// It's like Instruction::eraseFromParent() except that the actual deletion
546 /// is delayed until BoUpSLP is destructed.
547 /// This is required to ensure that there are no incorrect collisions in the
548 /// AliasCache, which can happen if a new instruction is allocated at the
549 /// same address as a previously deleted instruction.
550 void eraseInstruction(Instruction *I) {
551 I->removeFromParent();
552 I->dropAllReferences();
553 DeletedInstructions.push_back(std::unique_ptr<Instruction>(I));
554 }
555
556 /// Temporary store for deleted instructions. Instructions will be deleted
557 /// eventually when the BoUpSLP is destructed.
558 SmallVector<std::unique_ptr<Instruction>, 8> DeletedInstructions;
559
560 /// A list of values that need to extracted out of the tree.
561 /// This list holds pairs of (Internal Scalar : External User).
562 UserList ExternalUses;
563
564 /// Values used only by @llvm.assume calls.
565 SmallPtrSet<const Value *, 32> EphValues;
566
567 /// Holds all of the instructions that we gathered.
568 SetVector<Instruction *> GatherSeq;
569 /// A list of blocks that we are going to CSE.
570 SetVector<BasicBlock *> CSEBlocks;
571
572 /// Contains all scheduling relevant data for an instruction.
573 /// A ScheduleData either represents a single instruction or a member of an
574 /// instruction bundle (= a group of instructions which is combined into a
575 /// vector instruction).
576 struct ScheduleData {
577
578 // The initial value for the dependency counters. It means that the
579 // dependencies are not calculated yet.
580 enum { InvalidDeps = -1 };
581
582 ScheduleData()
583 : Inst(nullptr), FirstInBundle(nullptr), NextInBundle(nullptr),
584 NextLoadStore(nullptr), SchedulingRegionID(0), SchedulingPriority(0),
585 Dependencies(InvalidDeps), UnscheduledDeps(InvalidDeps),
586 UnscheduledDepsInBundle(InvalidDeps), IsScheduled(false) {}
587
588 void init(int BlockSchedulingRegionID) {
589 FirstInBundle = this;
590 NextInBundle = nullptr;
591 NextLoadStore = nullptr;
592 IsScheduled = false;
593 SchedulingRegionID = BlockSchedulingRegionID;
594 UnscheduledDepsInBundle = UnscheduledDeps;
595 clearDependencies();
596 }
597
598 /// Returns true if the dependency information has been calculated.
599 bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
600
601 /// Returns true for single instructions and for bundle representatives
602 /// (= the head of a bundle).
603 bool isSchedulingEntity() const { return FirstInBundle == this; }
604
605 /// Returns true if it represents an instruction bundle and not only a
606 /// single instruction.
607 bool isPartOfBundle() const {
608 return NextInBundle != nullptr || FirstInBundle != this;
609 }
610
611 /// Returns true if it is ready for scheduling, i.e. it has no more
612 /// unscheduled depending instructions/bundles.
613 bool isReady() const {
614 assert(isSchedulingEntity() &&((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 615, __PRETTY_FUNCTION__))
615 "can't consider non-scheduling entity for ready list")((isSchedulingEntity() && "can't consider non-scheduling entity for ready list"
) ? static_cast<void> (0) : __assert_fail ("isSchedulingEntity() && \"can't consider non-scheduling entity for ready list\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 615, __PRETTY_FUNCTION__))
;
616 return UnscheduledDepsInBundle == 0 && !IsScheduled;
617 }
618
619 /// Modifies the number of unscheduled dependencies, also updating it for
620 /// the whole bundle.
621 int incrementUnscheduledDeps(int Incr) {
622 UnscheduledDeps += Incr;
623 return FirstInBundle->UnscheduledDepsInBundle += Incr;
624 }
625
626 /// Sets the number of unscheduled dependencies to the number of
627 /// dependencies.
628 void resetUnscheduledDeps() {
629 incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
630 }
631
632 /// Clears all dependency information.
633 void clearDependencies() {
634 Dependencies = InvalidDeps;
635 resetUnscheduledDeps();
636 MemoryDependencies.clear();
637 }
638
639 void dump(raw_ostream &os) const {
640 if (!isSchedulingEntity()) {
641 os << "/ " << *Inst;
642 } else if (NextInBundle) {
643 os << '[' << *Inst;
644 ScheduleData *SD = NextInBundle;
645 while (SD) {
646 os << ';' << *SD->Inst;
647 SD = SD->NextInBundle;
648 }
649 os << ']';
650 } else {
651 os << *Inst;
652 }
653 }
654
655 Instruction *Inst;
656
657 /// Points to the head in an instruction bundle (and always to this for
658 /// single instructions).
659 ScheduleData *FirstInBundle;
660
661 /// Single linked list of all instructions in a bundle. Null if it is a
662 /// single instruction.
663 ScheduleData *NextInBundle;
664
665 /// Single linked list of all memory instructions (e.g. load, store, call)
666 /// in the block - until the end of the scheduling region.
667 ScheduleData *NextLoadStore;
668
669 /// The dependent memory instructions.
670 /// This list is derived on demand in calculateDependencies().
671 SmallVector<ScheduleData *, 4> MemoryDependencies;
672
673 /// This ScheduleData is in the current scheduling region if this matches
674 /// the current SchedulingRegionID of BlockScheduling.
675 int SchedulingRegionID;
676
677 /// Used for getting a "good" final ordering of instructions.
678 int SchedulingPriority;
679
680 /// The number of dependencies. Constitutes of the number of users of the
681 /// instruction plus the number of dependent memory instructions (if any).
682 /// This value is calculated on demand.
683 /// If InvalidDeps, the number of dependencies is not calculated yet.
684 ///
685 int Dependencies;
686
687 /// The number of dependencies minus the number of dependencies of scheduled
688 /// instructions. As soon as this is zero, the instruction/bundle gets ready
689 /// for scheduling.
690 /// Note that this is negative as long as Dependencies is not calculated.
691 int UnscheduledDeps;
692
693 /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
694 /// single instructions.
695 int UnscheduledDepsInBundle;
696
697 /// True if this instruction is scheduled (or considered as scheduled in the
698 /// dry-run).
699 bool IsScheduled;
700 };
701
702#ifndef NDEBUG
703 friend inline raw_ostream &operator<<(raw_ostream &os,
704 const BoUpSLP::ScheduleData &SD) {
705 SD.dump(os);
706 return os;
707 }
708#endif
709
710 /// Contains all scheduling data for a basic block.
711 ///
712 struct BlockScheduling {
713
714 BlockScheduling(BasicBlock *BB)
715 : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize),
716 ScheduleStart(nullptr), ScheduleEnd(nullptr),
717 FirstLoadStoreInRegion(nullptr), LastLoadStoreInRegion(nullptr),
718 ScheduleRegionSize(0),
719 ScheduleRegionSizeLimit(ScheduleRegionSizeBudget),
720 // Make sure that the initial SchedulingRegionID is greater than the
721 // initial SchedulingRegionID in ScheduleData (which is 0).
722 SchedulingRegionID(1) {}
723
724 void clear() {
725 ReadyInsts.clear();
726 ScheduleStart = nullptr;
727 ScheduleEnd = nullptr;
728 FirstLoadStoreInRegion = nullptr;
729 LastLoadStoreInRegion = nullptr;
730
731 // Reduce the maximum schedule region size by the size of the
732 // previous scheduling run.
733 ScheduleRegionSizeLimit -= ScheduleRegionSize;
734 if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
735 ScheduleRegionSizeLimit = MinScheduleRegionSize;
736 ScheduleRegionSize = 0;
737
738 // Make a new scheduling region, i.e. all existing ScheduleData is not
739 // in the new region yet.
740 ++SchedulingRegionID;
741 }
742
743 ScheduleData *getScheduleData(Value *V) {
744 ScheduleData *SD = ScheduleDataMap[V];
745 if (SD && SD->SchedulingRegionID == SchedulingRegionID)
746 return SD;
747 return nullptr;
748 }
749
750 bool isInSchedulingRegion(ScheduleData *SD) {
751 return SD->SchedulingRegionID == SchedulingRegionID;
752 }
753
754 /// Marks an instruction as scheduled and puts all dependent ready
755 /// instructions into the ready-list.
756 template <typename ReadyListType>
757 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
758 SD->IsScheduled = true;
759 DEBUG(dbgs() << "SLP: schedule " << *SD << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule " << *SD <<
"\n"; } } while (false)
;
760
761 ScheduleData *BundleMember = SD;
762 while (BundleMember) {
763 // Handle the def-use chain dependencies.
764 for (Use &U : BundleMember->Inst->operands()) {
765 ScheduleData *OpDef = getScheduleData(U.get());
766 if (OpDef && OpDef->hasValidDependencies() &&
767 OpDef->incrementUnscheduledDeps(-1) == 0) {
768 // There are no more unscheduled dependencies after decrementing,
769 // so we can put the dependent instruction into the ready list.
770 ScheduleData *DepBundle = OpDef->FirstInBundle;
771 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 772, __PRETTY_FUNCTION__))
772 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 772, __PRETTY_FUNCTION__))
;
773 ReadyList.insert(DepBundle);
774 DEBUG(dbgs() << "SLP: gets ready (def): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (def): " <<
*DepBundle << "\n"; } } while (false)
;
775 }
776 }
777 // Handle the memory dependencies.
778 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
779 if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
780 // There are no more unscheduled dependencies after decrementing,
781 // so we can put the dependent instruction into the ready list.
782 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
783 assert(!DepBundle->IsScheduled &&((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 784, __PRETTY_FUNCTION__))
784 "already scheduled bundle gets ready")((!DepBundle->IsScheduled && "already scheduled bundle gets ready"
) ? static_cast<void> (0) : __assert_fail ("!DepBundle->IsScheduled && \"already scheduled bundle gets ready\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 784, __PRETTY_FUNCTION__))
;
785 ReadyList.insert(DepBundle);
786 DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready (mem): " <<
*DepBundle << "\n"; } } while (false)
;
787 }
788 }
789 BundleMember = BundleMember->NextInBundle;
790 }
791 }
792
793 /// Put all instructions into the ReadyList which are ready for scheduling.
794 template <typename ReadyListType>
795 void initialFillReadyList(ReadyListType &ReadyList) {
796 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
797 ScheduleData *SD = getScheduleData(I);
798 if (SD->isSchedulingEntity() && SD->isReady()) {
799 ReadyList.insert(SD);
800 DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initially in ready list: "
<< *I << "\n"; } } while (false)
;
801 }
802 }
803 }
804
805 /// Checks if a bundle of instructions can be scheduled, i.e. has no
806 /// cyclic dependencies. This is only a dry-run, no instructions are
807 /// actually moved at this stage.
808 bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP);
809
810 /// Un-bundles a group of instructions.
811 void cancelScheduling(ArrayRef<Value *> VL);
812
813 /// Extends the scheduling region so that V is inside the region.
814 /// \returns true if the region size is within the limit.
815 bool extendSchedulingRegion(Value *V);
816
817 /// Initialize the ScheduleData structures for new instructions in the
818 /// scheduling region.
819 void initScheduleData(Instruction *FromI, Instruction *ToI,
820 ScheduleData *PrevLoadStore,
821 ScheduleData *NextLoadStore);
822
823 /// Updates the dependency information of a bundle and of all instructions/
824 /// bundles which depend on the original bundle.
825 void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
826 BoUpSLP *SLP);
827
828 /// Sets all instruction in the scheduling region to un-scheduled.
829 void resetSchedule();
830
831 BasicBlock *BB;
832
833 /// Simple memory allocation for ScheduleData.
834 std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
835
836 /// The size of a ScheduleData array in ScheduleDataChunks.
837 int ChunkSize;
838
839 /// The allocator position in the current chunk, which is the last entry
840 /// of ScheduleDataChunks.
841 int ChunkPos;
842
843 /// Attaches ScheduleData to Instruction.
844 /// Note that the mapping survives during all vectorization iterations, i.e.
845 /// ScheduleData structures are recycled.
846 DenseMap<Value *, ScheduleData *> ScheduleDataMap;
847
848 struct ReadyList : SmallVector<ScheduleData *, 8> {
849 void insert(ScheduleData *SD) { push_back(SD); }
850 };
851
852 /// The ready-list for scheduling (only used for the dry-run).
853 ReadyList ReadyInsts;
854
855 /// The first instruction of the scheduling region.
856 Instruction *ScheduleStart;
857
858 /// The first instruction _after_ the scheduling region.
859 Instruction *ScheduleEnd;
860
861 /// The first memory accessing instruction in the scheduling region
862 /// (can be null).
863 ScheduleData *FirstLoadStoreInRegion;
864
865 /// The last memory accessing instruction in the scheduling region
866 /// (can be null).
867 ScheduleData *LastLoadStoreInRegion;
868
869 /// The current size of the scheduling region.
870 int ScheduleRegionSize;
871
872 /// The maximum size allowed for the scheduling region.
873 int ScheduleRegionSizeLimit;
874
875 /// The ID of the scheduling region. For a new vectorization iteration this
876 /// is incremented which "removes" all ScheduleData from the region.
877 int SchedulingRegionID;
878 };
879
880 /// Attaches the BlockScheduling structures to basic blocks.
881 MapVector<BasicBlock *, std::unique_ptr<BlockScheduling>> BlocksSchedules;
882
883 /// Performs the "real" scheduling. Done before vectorization is actually
884 /// performed in a basic block.
885 void scheduleBlock(BlockScheduling *BS);
886
887 /// List of users to ignore during scheduling and that don't need extracting.
888 ArrayRef<Value *> UserIgnoreList;
889
890 // Number of load bundles that contain consecutive loads.
891 int NumLoadsWantToKeepOrder;
892
893 // Number of load bundles that contain consecutive loads in reversed order.
894 int NumLoadsWantToChangeOrder;
895
896 // Analysis and block reference.
897 Function *F;
898 ScalarEvolution *SE;
899 TargetTransformInfo *TTI;
900 TargetLibraryInfo *TLI;
901 AliasAnalysis *AA;
902 LoopInfo *LI;
903 DominatorTree *DT;
904 AssumptionCache *AC;
905 DemandedBits *DB;
906 const DataLayout *DL;
907 unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
908 unsigned MinVecRegSize; // Set by cl::opt (default: 128).
909 /// Instruction builder to construct the vectorized tree.
910 IRBuilder<> Builder;
911
912 /// A map of scalar integer values to the smallest bit width with which they
913 /// can legally be represented. The values map to (width, signed) pairs,
914 /// where "width" indicates the minimum bit width and "signed" is True if the
915 /// value must be signed-extended, rather than zero-extended, back to its
916 /// original width.
917 MapVector<Value *, std::pair<uint64_t, bool>> MinBWs;
918};
919
920} // end namespace llvm
921} // end namespace slpvectorizer
922
923void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
924 ArrayRef<Value *> UserIgnoreLst) {
925 deleteTree();
926 UserIgnoreList = UserIgnoreLst;
927 if (!allSameType(Roots))
928 return;
929 buildTree_rec(Roots, 0);
930
931 // Collect the values that we need to extract from the tree.
932 for (TreeEntry &EIdx : VectorizableTree) {
933 TreeEntry *Entry = &EIdx;
934
935 // For each lane:
936 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
937 Value *Scalar = Entry->Scalars[Lane];
938
939 // No need to handle users of gathered values.
940 if (Entry->NeedToGather)
941 continue;
942
943 for (User *U : Scalar->users()) {
944 DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Checking user:" << *U <<
".\n"; } } while (false)
;
945
946 Instruction *UserInst = dyn_cast<Instruction>(U);
947 if (!UserInst)
948 continue;
949
950 // Skip in-tree scalars that become vectors
951 if (ScalarToTreeEntry.count(U)) {
952 int Idx = ScalarToTreeEntry[U];
953 TreeEntry *UseEntry = &VectorizableTree[Idx];
954 Value *UseScalar = UseEntry->Scalars[0];
955 // Some in-tree scalars will remain as scalar in vectorized
956 // instructions. If that is the case, the one in Lane 0 will
957 // be used.
958 if (UseScalar != U ||
959 !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
960 DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *Udo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
961 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tInternal user will be removed:"
<< *U << ".\n"; } } while (false)
;
962 assert(!VectorizableTree[Idx].NeedToGather && "Bad state")((!VectorizableTree[Idx].NeedToGather && "Bad state")
? static_cast<void> (0) : __assert_fail ("!VectorizableTree[Idx].NeedToGather && \"Bad state\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 962, __PRETTY_FUNCTION__))
;
963 continue;
964 }
965 }
966
967 // Ignore users in the user ignore list.
968 if (is_contained(UserIgnoreList, UserInst))
969 continue;
970
971 DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
972 Lane << " from " << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to extract:" << *
U << " from lane " << Lane << " from " <<
*Scalar << ".\n"; } } while (false)
;
973 ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
974 }
975 }
976 }
977}
978
979
980void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth) {
981 bool isAltShuffle = false;
982 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!")(((allConstant(VL) || allSameType(VL)) && "Invalid types!"
) ? static_cast<void> (0) : __assert_fail ("(allConstant(VL) || allSameType(VL)) && \"Invalid types!\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 982, __PRETTY_FUNCTION__))
;
983
984 if (Depth == RecursionMaxDepth) {
985 DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to max recursion depth.\n"
; } } while (false)
;
986 newTreeEntry(VL, false);
987 return;
988 }
989
990 // Don't handle vectors.
991 if (VL[0]->getType()->isVectorTy()) {
992 DEBUG(dbgs() << "SLP: Gathering due to vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to vector type.\n"
; } } while (false)
;
993 newTreeEntry(VL, false);
994 return;
995 }
996
997 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
998 if (SI->getValueOperand()->getType()->isVectorTy()) {
999 DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to store vector type.\n"
; } } while (false)
;
1000 newTreeEntry(VL, false);
1001 return;
1002 }
1003 unsigned Opcode = getSameOpcode(VL);
1004
1005 // Check that this shuffle vector refers to the alternate
1006 // sequence of opcodes.
1007 if (Opcode == Instruction::ShuffleVector) {
1008 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
1009 unsigned Op = I0->getOpcode();
1010 if (Op != Instruction::ShuffleVector)
1011 isAltShuffle = true;
1012 }
1013
1014 // If all of the operands are identical or constant we have a simple solution.
1015 if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !Opcode) {
1016 DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to C,S,B,O. \n"
; } } while (false)
;
1017 newTreeEntry(VL, false);
1018 return;
1019 }
1020
1021 // We now know that this is a vector of instructions of the same type from
1022 // the same block.
1023
1024 // Don't vectorize ephemeral values.
1025 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1026 if (EphValues.count(VL[i])) {
1027 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
1028 ") is ephemeral.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is ephemeral.\n"; } } while (false)
;
1029 newTreeEntry(VL, false);
1030 return;
1031 }
1032 }
1033
1034 // Check if this is a duplicate of another entry.
1035 if (ScalarToTreeEntry.count(VL[0])) {
1036 int Idx = ScalarToTreeEntry[VL[0]];
1037 TreeEntry *E = &VectorizableTree[Idx];
1038 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1039 DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tChecking bundle: " <<
*VL[i] << ".\n"; } } while (false)
;
1040 if (E->Scalars[i] != VL[i]) {
1041 DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to partial overlap.\n"
; } } while (false)
;
1042 newTreeEntry(VL, false);
1043 return;
1044 }
1045 }
1046 DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *VL[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Perfect diamond merge at " <<
*VL[0] << ".\n"; } } while (false)
;
1047 return;
1048 }
1049
1050 // Check that none of the instructions in the bundle are already in the tree.
1051 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1052 if (ScalarToTreeEntry.count(VL[i])) {
1053 DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
1054 ") is already in tree.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: The instruction (" << *
VL[i] << ") is already in tree.\n"; } } while (false)
;
1055 newTreeEntry(VL, false);
1056 return;
1057 }
1058 }
1059
1060 // If any of the scalars is marked as a value that needs to stay scalar then
1061 // we need to gather the scalars.
1062 for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1063 if (MustGather.count(VL[i])) {
1064 DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering due to gathered scalar.\n"
; } } while (false)
;
1065 newTreeEntry(VL, false);
1066 return;
1067 }
1068 }
1069
1070 // Check that all of the users of the scalars that we want to vectorize are
1071 // schedulable.
1072 Instruction *VL0 = cast<Instruction>(VL[0]);
1073 BasicBlock *BB = cast<Instruction>(VL0)->getParent();
1074
1075 if (!DT->isReachableFromEntry(BB)) {
1076 // Don't go into unreachable blocks. They may contain instructions with
1077 // dependency cycles which confuse the final scheduling.
1078 DEBUG(dbgs() << "SLP: bundle in unreachable block.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle in unreachable block.\n"
; } } while (false)
;
1079 newTreeEntry(VL, false);
1080 return;
1081 }
1082
1083 // Check that every instructions appears once in this bundle.
1084 for (unsigned i = 0, e = VL.size(); i < e; ++i)
1085 for (unsigned j = i+1; j < e; ++j)
1086 if (VL[i] == VL[j]) {
1087 DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Scalar used twice in bundle.\n"
; } } while (false)
;
1088 newTreeEntry(VL, false);
1089 return;
1090 }
1091
1092 auto &BSRef = BlocksSchedules[BB];
1093 if (!BSRef) {
1094 BSRef = llvm::make_unique<BlockScheduling>(BB);
1095 }
1096 BlockScheduling &BS = *BSRef.get();
1097
1098 if (!BS.tryScheduleBundle(VL, this)) {
1099 DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are not able to schedule this bundle!\n"
; } } while (false)
;
1100 assert((!BS.getScheduleData(VL[0]) ||(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1102, __PRETTY_FUNCTION__))
1101 !BS.getScheduleData(VL[0])->isPartOfBundle()) &&(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1102, __PRETTY_FUNCTION__))
1102 "tryScheduleBundle should cancelScheduling on failure")(((!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->
isPartOfBundle()) && "tryScheduleBundle should cancelScheduling on failure"
) ? static_cast<void> (0) : __assert_fail ("(!BS.getScheduleData(VL[0]) || !BS.getScheduleData(VL[0])->isPartOfBundle()) && \"tryScheduleBundle should cancelScheduling on failure\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1102, __PRETTY_FUNCTION__))
;
1103 newTreeEntry(VL, false);
1104 return;
1105 }
1106 DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: We are able to schedule this bundle.\n"
; } } while (false)
;
1107
1108 switch (Opcode) {
1109 case Instruction::PHI: {
1110 PHINode *PH = dyn_cast<PHINode>(VL0);
1111
1112 // Check for terminator values (e.g. invoke).
1113 for (unsigned j = 0; j < VL.size(); ++j)
1114 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1115 TerminatorInst *Term = dyn_cast<TerminatorInst>(
1116 cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1117 if (Term) {
1118 DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n"
; } } while (false)
;
1119 BS.cancelScheduling(VL);
1120 newTreeEntry(VL, false);
1121 return;
1122 }
1123 }
1124
1125 newTreeEntry(VL, true);
1126 DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of PHINodes.\n"
; } } while (false)
;
1127
1128 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1129 ValueList Operands;
1130 // Prepare the operand vector.
1131 for (Value *j : VL)
1132 Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1133 PH->getIncomingBlock(i)));
1134
1135 buildTree_rec(Operands, Depth + 1);
1136 }
1137 return;
1138 }
1139 case Instruction::ExtractValue:
1140 case Instruction::ExtractElement: {
1141 bool Reuse = canReuseExtract(VL, Opcode);
1142 if (Reuse) {
1143 DEBUG(dbgs() << "SLP: Reusing extract sequence.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Reusing extract sequence.\n"
; } } while (false)
;
1144 } else {
1145 BS.cancelScheduling(VL);
1146 }
1147 newTreeEntry(VL, Reuse);
1148 return;
1149 }
1150 case Instruction::Load: {
1151 // Check that a vectorized load would load the same memory as a scalar
1152 // load.
1153 // For example we don't want vectorize loads that are smaller than 8 bit.
1154 // Even though we have a packed struct {<i2, i2, i2, i2>} LLVM treats
1155 // loading/storing it as an i8 struct. If we vectorize loads/stores from
1156 // such a struct we read/write packed bits disagreeing with the
1157 // unvectorized version.
1158 Type *ScalarTy = VL[0]->getType();
1159
1160 if (DL->getTypeSizeInBits(ScalarTy) !=
1161 DL->getTypeAllocSizeInBits(ScalarTy)) {
1162 BS.cancelScheduling(VL);
1163 newTreeEntry(VL, false);
1164 DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering loads of non-packed type.\n"
; } } while (false)
;
1165 return;
1166 }
1167
1168 // Make sure all loads in the bundle are simple - we can't vectorize
1169 // atomic or volatile loads.
1170 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1171 LoadInst *L = cast<LoadInst>(VL[i]);
1172 if (!L->isSimple()) {
1173 BS.cancelScheduling(VL);
1174 newTreeEntry(VL, false);
1175 DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-simple loads.\n"
; } } while (false)
;
1176 return;
1177 }
1178 }
1179
1180 // Check if the loads are consecutive, reversed, or neither.
1181 // TODO: What we really want is to sort the loads, but for now, check
1182 // the two likely directions.
1183 bool Consecutive = true;
1184 bool ReverseConsecutive = true;
1185 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1186 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1187 Consecutive = false;
1188 break;
1189 } else {
1190 ReverseConsecutive = false;
1191 }
1192 }
1193
1194 if (Consecutive) {
1195 ++NumLoadsWantToKeepOrder;
1196 newTreeEntry(VL, true);
1197 DEBUG(dbgs() << "SLP: added a vector of loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of loads.\n";
} } while (false)
;
1198 return;
1199 }
1200
1201 // If none of the load pairs were consecutive when checked in order,
1202 // check the reverse order.
1203 if (ReverseConsecutive)
1204 for (unsigned i = VL.size() - 1; i > 0; --i)
1205 if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1206 ReverseConsecutive = false;
1207 break;
1208 }
1209
1210 BS.cancelScheduling(VL);
1211 newTreeEntry(VL, false);
1212
1213 if (ReverseConsecutive) {
1214 ++NumLoadsWantToChangeOrder;
1215 DEBUG(dbgs() << "SLP: Gathering reversed loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering reversed loads.\n"
; } } while (false)
;
1216 } else {
1217 DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering non-consecutive loads.\n"
; } } while (false)
;
1218 }
1219 return;
1220 }
1221 case Instruction::ZExt:
1222 case Instruction::SExt:
1223 case Instruction::FPToUI:
1224 case Instruction::FPToSI:
1225 case Instruction::FPExt:
1226 case Instruction::PtrToInt:
1227 case Instruction::IntToPtr:
1228 case Instruction::SIToFP:
1229 case Instruction::UIToFP:
1230 case Instruction::Trunc:
1231 case Instruction::FPTrunc:
1232 case Instruction::BitCast: {
1233 Type *SrcTy = VL0->getOperand(0)->getType();
1234 for (unsigned i = 0; i < VL.size(); ++i) {
1235 Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1236 if (Ty != SrcTy || !isValidElementType(Ty)) {
1237 BS.cancelScheduling(VL);
1238 newTreeEntry(VL, false);
1239 DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering casts with different src types.\n"
; } } while (false)
;
1240 return;
1241 }
1242 }
1243 newTreeEntry(VL, true);
1244 DEBUG(dbgs() << "SLP: added a vector of casts.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of casts.\n";
} } while (false)
;
1245
1246 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1247 ValueList Operands;
1248 // Prepare the operand vector.
1249 for (Value *j : VL)
1250 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1251
1252 buildTree_rec(Operands, Depth+1);
1253 }
1254 return;
1255 }
1256 case Instruction::ICmp:
1257 case Instruction::FCmp: {
1258 // Check that all of the compares have the same predicate.
1259 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1260 Type *ComparedTy = cast<Instruction>(VL[0])->getOperand(0)->getType();
1261 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1262 CmpInst *Cmp = cast<CmpInst>(VL[i]);
1263 if (Cmp->getPredicate() != P0 ||
1264 Cmp->getOperand(0)->getType() != ComparedTy) {
1265 BS.cancelScheduling(VL);
1266 newTreeEntry(VL, false);
1267 DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering cmp with different predicate.\n"
; } } while (false)
;
1268 return;
1269 }
1270 }
1271
1272 newTreeEntry(VL, true);
1273 DEBUG(dbgs() << "SLP: added a vector of compares.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of compares.\n"
; } } while (false)
;
1274
1275 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1276 ValueList Operands;
1277 // Prepare the operand vector.
1278 for (Value *j : VL)
1279 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1280
1281 buildTree_rec(Operands, Depth+1);
1282 }
1283 return;
1284 }
1285 case Instruction::Select:
1286 case Instruction::Add:
1287 case Instruction::FAdd:
1288 case Instruction::Sub:
1289 case Instruction::FSub:
1290 case Instruction::Mul:
1291 case Instruction::FMul:
1292 case Instruction::UDiv:
1293 case Instruction::SDiv:
1294 case Instruction::FDiv:
1295 case Instruction::URem:
1296 case Instruction::SRem:
1297 case Instruction::FRem:
1298 case Instruction::Shl:
1299 case Instruction::LShr:
1300 case Instruction::AShr:
1301 case Instruction::And:
1302 case Instruction::Or:
1303 case Instruction::Xor: {
1304 newTreeEntry(VL, true);
1305 DEBUG(dbgs() << "SLP: added a vector of bin op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of bin op.\n"
; } } while (false)
;
1306
1307 // Sort operands of the instructions so that each side is more likely to
1308 // have the same opcode.
1309 if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1310 ValueList Left, Right;
1311 reorderInputsAccordingToOpcode(VL, Left, Right);
1312 buildTree_rec(Left, Depth + 1);
1313 buildTree_rec(Right, Depth + 1);
1314 return;
1315 }
1316
1317 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1318 ValueList Operands;
1319 // Prepare the operand vector.
1320 for (Value *j : VL)
1321 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1322
1323 buildTree_rec(Operands, Depth+1);
1324 }
1325 return;
1326 }
1327 case Instruction::GetElementPtr: {
1328 // We don't combine GEPs with complicated (nested) indexing.
1329 for (unsigned j = 0; j < VL.size(); ++j) {
1330 if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1331 DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n"
; } } while (false)
;
1332 BS.cancelScheduling(VL);
1333 newTreeEntry(VL, false);
1334 return;
1335 }
1336 }
1337
1338 // We can't combine several GEPs into one vector if they operate on
1339 // different types.
1340 Type *Ty0 = cast<Instruction>(VL0)->getOperand(0)->getType();
1341 for (unsigned j = 0; j < VL.size(); ++j) {
1342 Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1343 if (Ty0 != CurTy) {
1344 DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (different types).\n"
; } } while (false)
;
1345 BS.cancelScheduling(VL);
1346 newTreeEntry(VL, false);
1347 return;
1348 }
1349 }
1350
1351 // We don't combine GEPs with non-constant indexes.
1352 for (unsigned j = 0; j < VL.size(); ++j) {
1353 auto Op = cast<Instruction>(VL[j])->getOperand(1);
1354 if (!isa<ConstantInt>(Op)) {
1355 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
1356 dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n"
; } } while (false)
;
1357 BS.cancelScheduling(VL);
1358 newTreeEntry(VL, false);
1359 return;
1360 }
1361 }
1362
1363 newTreeEntry(VL, true);
1364 DEBUG(dbgs() << "SLP: added a vector of GEPs.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of GEPs.\n"; }
} while (false)
;
1365 for (unsigned i = 0, e = 2; i < e; ++i) {
1366 ValueList Operands;
1367 // Prepare the operand vector.
1368 for (Value *j : VL)
1369 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1370
1371 buildTree_rec(Operands, Depth + 1);
1372 }
1373 return;
1374 }
1375 case Instruction::Store: {
1376 // Check if the stores are consecutive or of we need to swizzle them.
1377 for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1378 if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1379 BS.cancelScheduling(VL);
1380 newTreeEntry(VL, false);
1381 DEBUG(dbgs() << "SLP: Non-consecutive store.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-consecutive store.\n"; }
} while (false)
;
1382 return;
1383 }
1384
1385 newTreeEntry(VL, true);
1386 DEBUG(dbgs() << "SLP: added a vector of stores.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a vector of stores.\n"
; } } while (false)
;
1387
1388 ValueList Operands;
1389 for (Value *j : VL)
1390 Operands.push_back(cast<Instruction>(j)->getOperand(0));
1391
1392 buildTree_rec(Operands, Depth + 1);
1393 return;
1394 }
1395 case Instruction::Call: {
1396 // Check if the calls are all to the same vectorizable intrinsic.
1397 CallInst *CI = cast<CallInst>(VL[0]);
1398 // Check if this is an Intrinsic call or something that can be
1399 // represented by an intrinsic call
1400 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1401 if (!isTriviallyVectorizable(ID)) {
1402 BS.cancelScheduling(VL);
1403 newTreeEntry(VL, false);
1404 DEBUG(dbgs() << "SLP: Non-vectorizable call.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Non-vectorizable call.\n"; }
} while (false)
;
1405 return;
1406 }
1407 Function *Int = CI->getCalledFunction();
1408 Value *A1I = nullptr;
1409 if (hasVectorInstrinsicScalarOpd(ID, 1))
1410 A1I = CI->getArgOperand(1);
1411 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1412 CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1413 if (!CI2 || CI2->getCalledFunction() != Int ||
1414 getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1415 !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1416 BS.cancelScheduling(VL);
1417 newTreeEntry(VL, false);
1418 DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
1419 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched calls:" << *
CI << "!=" << *VL[i] << "\n"; } } while (false
)
;
1420 return;
1421 }
1422 // ctlz,cttz and powi are special intrinsics whose second argument
1423 // should be same in order for them to be vectorized.
1424 if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1425 Value *A1J = CI2->getArgOperand(1);
1426 if (A1I != A1J) {
1427 BS.cancelScheduling(VL);
1428 newTreeEntry(VL, false);
1429 DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1430 << " argument "<< A1I<<"!=" << A1Jdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
1431 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched arguments in call:"
<< *CI << " argument "<< A1I<<"!=" <<
A1J << "\n"; } } while (false)
;
1432 return;
1433 }
1434 }
1435 // Verify that the bundle operands are identical between the two calls.
1436 if (CI->hasOperandBundles() &&
1437 !std::equal(CI->op_begin() + CI->getBundleOperandsStartIndex(),
1438 CI->op_begin() + CI->getBundleOperandsEndIndex(),
1439 CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1440 BS.cancelScheduling(VL);
1441 newTreeEntry(VL, false);
1442 DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
1443 << *VL[i] << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: mismatched bundle operands in calls:"
<< *CI << "!=" << *VL[i] << '\n'; } }
while (false)
;
1444 return;
1445 }
1446 }
1447
1448 newTreeEntry(VL, true);
1449 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1450 ValueList Operands;
1451 // Prepare the operand vector.
1452 for (Value *j : VL) {
1453 CallInst *CI2 = dyn_cast<CallInst>(j);
1454 Operands.push_back(CI2->getArgOperand(i));
1455 }
1456 buildTree_rec(Operands, Depth + 1);
1457 }
1458 return;
1459 }
1460 case Instruction::ShuffleVector: {
1461 // If this is not an alternate sequence of opcode like add-sub
1462 // then do not vectorize this instruction.
1463 if (!isAltShuffle) {
1464 BS.cancelScheduling(VL);
1465 newTreeEntry(VL, false);
1466 DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: ShuffleVector are not vectorized.\n"
; } } while (false)
;
1467 return;
1468 }
1469 newTreeEntry(VL, true);
1470 DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: added a ShuffleVector op.\n"
; } } while (false)
;
1471
1472 // Reorder operands if reordering would enable vectorization.
1473 if (isa<BinaryOperator>(VL0)) {
1474 ValueList Left, Right;
1475 reorderAltShuffleOperands(VL, Left, Right);
1476 buildTree_rec(Left, Depth + 1);
1477 buildTree_rec(Right, Depth + 1);
1478 return;
1479 }
1480
1481 for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1482 ValueList Operands;
1483 // Prepare the operand vector.
1484 for (Value *j : VL)
1485 Operands.push_back(cast<Instruction>(j)->getOperand(i));
1486
1487 buildTree_rec(Operands, Depth + 1);
1488 }
1489 return;
1490 }
1491 default:
1492 BS.cancelScheduling(VL);
1493 newTreeEntry(VL, false);
1494 DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Gathering unknown instruction.\n"
; } } while (false)
;
1495 return;
1496 }
1497}
1498
1499unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1500 unsigned N;
1501 Type *EltTy;
1502 auto *ST = dyn_cast<StructType>(T);
1503 if (ST) {
1504 N = ST->getNumElements();
1505 EltTy = *ST->element_begin();
1506 } else {
1507 N = cast<ArrayType>(T)->getNumElements();
1508 EltTy = cast<ArrayType>(T)->getElementType();
1509 }
1510 if (!isValidElementType(EltTy))
1511 return 0;
1512 uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1513 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1514 return 0;
1515 if (ST) {
1516 // Check that struct is homogeneous.
1517 for (const auto *Ty : ST->elements())
1518 if (Ty != EltTy)
1519 return 0;
1520 }
1521 return N;
1522}
1523
1524bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, unsigned Opcode) const {
1525 assert(Opcode == Instruction::ExtractElement ||((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1526, __PRETTY_FUNCTION__))
1526 Opcode == Instruction::ExtractValue)((Opcode == Instruction::ExtractElement || Opcode == Instruction
::ExtractValue) ? static_cast<void> (0) : __assert_fail
("Opcode == Instruction::ExtractElement || Opcode == Instruction::ExtractValue"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1526, __PRETTY_FUNCTION__))
;
1527 assert(Opcode == getSameOpcode(VL) && "Invalid opcode")((Opcode == getSameOpcode(VL) && "Invalid opcode") ? static_cast
<void> (0) : __assert_fail ("Opcode == getSameOpcode(VL) && \"Invalid opcode\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1527, __PRETTY_FUNCTION__))
;
1528 // Check if all of the extracts come from the same vector and from the
1529 // correct offset.
1530 Value *VL0 = VL[0];
1531 Instruction *E0 = cast<Instruction>(VL0);
1532 Value *Vec = E0->getOperand(0);
1533
1534 // We have to extract from a vector/aggregate with the same number of elements.
1535 unsigned NElts;
1536 if (Opcode == Instruction::ExtractValue) {
1537 const DataLayout &DL = E0->getModule()->getDataLayout();
1538 NElts = canMapToVector(Vec->getType(), DL);
1539 if (!NElts)
1540 return false;
1541 // Check if load can be rewritten as load of vector.
1542 LoadInst *LI = dyn_cast<LoadInst>(Vec);
1543 if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1544 return false;
1545 } else {
1546 NElts = Vec->getType()->getVectorNumElements();
1547 }
1548
1549 if (NElts != VL.size())
1550 return false;
1551
1552 // Check that all of the indices extract from the correct offset.
1553 if (!matchExtractIndex(E0, 0, Opcode))
1554 return false;
1555
1556 for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1557 Instruction *E = cast<Instruction>(VL[i]);
1558 if (!matchExtractIndex(E, i, Opcode))
1559 return false;
1560 if (E->getOperand(0) != Vec)
1561 return false;
1562 }
1563
1564 return true;
1565}
1566
1567int BoUpSLP::getEntryCost(TreeEntry *E) {
1568 ArrayRef<Value*> VL = E->Scalars;
1569
1570 Type *ScalarTy = VL[0]->getType();
1571 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1572 ScalarTy = SI->getValueOperand()->getType();
1573 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1574
1575 // If we have computed a smaller type for the expression, update VecTy so
1576 // that the costs will be accurate.
1577 if (MinBWs.count(VL[0]))
1578 VecTy = VectorType::get(
1579 IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1580
1581 if (E->NeedToGather) {
1582 if (allConstant(VL))
1583 return 0;
1584 if (isSplat(VL)) {
1585 return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1586 }
1587 return getGatherCost(E->Scalars);
1588 }
1589 unsigned Opcode = getSameOpcode(VL);
1590 assert(Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL")((Opcode && allSameType(VL) && allSameBlock(VL
) && "Invalid VL") ? static_cast<void> (0) : __assert_fail
("Opcode && allSameType(VL) && allSameBlock(VL) && \"Invalid VL\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1590, __PRETTY_FUNCTION__))
;
1591 Instruction *VL0 = cast<Instruction>(VL[0]);
1592 switch (Opcode) {
1593 case Instruction::PHI: {
1594 return 0;
1595 }
1596 case Instruction::ExtractValue:
1597 case Instruction::ExtractElement: {
1598 if (canReuseExtract(VL, Opcode)) {
1599 int DeadCost = 0;
1600 for (unsigned i = 0, e = VL.size(); i < e; ++i) {
1601 Instruction *E = cast<Instruction>(VL[i]);
1602 if (E->hasOneUse())
1603 // Take credit for instruction that will become dead.
1604 DeadCost +=
1605 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
1606 }
1607 return -DeadCost;
1608 }
1609 return getGatherCost(VecTy);
1610 }
1611 case Instruction::ZExt:
1612 case Instruction::SExt:
1613 case Instruction::FPToUI:
1614 case Instruction::FPToSI:
1615 case Instruction::FPExt:
1616 case Instruction::PtrToInt:
1617 case Instruction::IntToPtr:
1618 case Instruction::SIToFP:
1619 case Instruction::UIToFP:
1620 case Instruction::Trunc:
1621 case Instruction::FPTrunc:
1622 case Instruction::BitCast: {
1623 Type *SrcTy = VL0->getOperand(0)->getType();
1624
1625 // Calculate the cost of this instruction.
1626 int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
1627 VL0->getType(), SrcTy);
1628
1629 VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
1630 int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy);
1631 return VecCost - ScalarCost;
1632 }
1633 case Instruction::FCmp:
1634 case Instruction::ICmp:
1635 case Instruction::Select: {
1636 // Calculate the cost of this instruction.
1637 VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
1638 int ScalarCost = VecTy->getNumElements() *
1639 TTI->getCmpSelInstrCost(Opcode, ScalarTy, Builder.getInt1Ty());
1640 int VecCost = TTI->getCmpSelInstrCost(Opcode, VecTy, MaskTy);
1641 return VecCost - ScalarCost;
1642 }
1643 case Instruction::Add:
1644 case Instruction::FAdd:
1645 case Instruction::Sub:
1646 case Instruction::FSub:
1647 case Instruction::Mul:
1648 case Instruction::FMul:
1649 case Instruction::UDiv:
1650 case Instruction::SDiv:
1651 case Instruction::FDiv:
1652 case Instruction::URem:
1653 case Instruction::SRem:
1654 case Instruction::FRem:
1655 case Instruction::Shl:
1656 case Instruction::LShr:
1657 case Instruction::AShr:
1658 case Instruction::And:
1659 case Instruction::Or:
1660 case Instruction::Xor: {
1661 // Certain instructions can be cheaper to vectorize if they have a
1662 // constant second vector operand.
1663 TargetTransformInfo::OperandValueKind Op1VK =
1664 TargetTransformInfo::OK_AnyValue;
1665 TargetTransformInfo::OperandValueKind Op2VK =
1666 TargetTransformInfo::OK_UniformConstantValue;
1667 TargetTransformInfo::OperandValueProperties Op1VP =
1668 TargetTransformInfo::OP_None;
1669 TargetTransformInfo::OperandValueProperties Op2VP =
1670 TargetTransformInfo::OP_None;
1671
1672 // If all operands are exactly the same ConstantInt then set the
1673 // operand kind to OK_UniformConstantValue.
1674 // If instead not all operands are constants, then set the operand kind
1675 // to OK_AnyValue. If all operands are constants but not the same,
1676 // then set the operand kind to OK_NonUniformConstantValue.
1677 ConstantInt *CInt = nullptr;
1678 for (unsigned i = 0; i < VL.size(); ++i) {
1679 const Instruction *I = cast<Instruction>(VL[i]);
1680 if (!isa<ConstantInt>(I->getOperand(1))) {
1681 Op2VK = TargetTransformInfo::OK_AnyValue;
1682 break;
1683 }
1684 if (i == 0) {
1685 CInt = cast<ConstantInt>(I->getOperand(1));
1686 continue;
1687 }
1688 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue &&
1689 CInt != cast<ConstantInt>(I->getOperand(1)))
1690 Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
1691 }
1692 // FIXME: Currently cost of model modification for division by power of
1693 // 2 is handled for X86 and AArch64. Add support for other targets.
1694 if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
1695 CInt->getValue().isPowerOf2())
1696 Op2VP = TargetTransformInfo::OP_PowerOf2;
1697
1698 int ScalarCost = VecTy->getNumElements() *
1699 TTI->getArithmeticInstrCost(Opcode, ScalarTy, Op1VK,
1700 Op2VK, Op1VP, Op2VP);
1701 int VecCost = TTI->getArithmeticInstrCost(Opcode, VecTy, Op1VK, Op2VK,
1702 Op1VP, Op2VP);
1703 return VecCost - ScalarCost;
1704 }
1705 case Instruction::GetElementPtr: {
1706 TargetTransformInfo::OperandValueKind Op1VK =
1707 TargetTransformInfo::OK_AnyValue;
1708 TargetTransformInfo::OperandValueKind Op2VK =
1709 TargetTransformInfo::OK_UniformConstantValue;
1710
1711 int ScalarCost =
1712 VecTy->getNumElements() *
1713 TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
1714 int VecCost =
1715 TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
1716
1717 return VecCost - ScalarCost;
1718 }
1719 case Instruction::Load: {
1720 // Cost of wide load - cost of scalar loads.
1721 unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
1722 int ScalarLdCost = VecTy->getNumElements() *
1723 TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0);
1724 int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
1725 VecTy, alignment, 0);
1726 return VecLdCost - ScalarLdCost;
1727 }
1728 case Instruction::Store: {
1729 // We know that we can merge the stores. Calculate the cost.
1730 unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
1731 int ScalarStCost = VecTy->getNumElements() *
1732 TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0);
1733 int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
1734 VecTy, alignment, 0);
1735 return VecStCost - ScalarStCost;
1736 }
1737 case Instruction::Call: {
1738 CallInst *CI = cast<CallInst>(VL0);
1739 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1740
1741 // Calculate the cost of the scalar and vector calls.
1742 SmallVector<Type*, 4> ScalarTys, VecTys;
1743 for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op) {
1744 ScalarTys.push_back(CI->getArgOperand(op)->getType());
1745 VecTys.push_back(VectorType::get(CI->getArgOperand(op)->getType(),
1746 VecTy->getNumElements()));
1747 }
1748
1749 FastMathFlags FMF;
1750 if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
1751 FMF = FPMO->getFastMathFlags();
1752
1753 int ScalarCallCost = VecTy->getNumElements() *
1754 TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
1755
1756 int VecCallCost = TTI->getIntrinsicInstrCost(ID, VecTy, VecTys, FMF);
1757
1758 DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
1759 << " (" << VecCallCost << "-" << ScalarCallCost << ")"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
1760 << " for " << *CI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Call cost "<< VecCallCost
- ScalarCallCost << " (" << VecCallCost <<
"-" << ScalarCallCost << ")" << " for " <<
*CI << "\n"; } } while (false)
;
1761
1762 return VecCallCost - ScalarCallCost;
1763 }
1764 case Instruction::ShuffleVector: {
1765 TargetTransformInfo::OperandValueKind Op1VK =
1766 TargetTransformInfo::OK_AnyValue;
1767 TargetTransformInfo::OperandValueKind Op2VK =
1768 TargetTransformInfo::OK_AnyValue;
1769 int ScalarCost = 0;
1770 int VecCost = 0;
1771 for (Value *i : VL) {
1772 Instruction *I = cast<Instruction>(i);
1773 if (!I)
1774 break;
1775 ScalarCost +=
1776 TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
1777 }
1778 // VecCost is equal to sum of the cost of creating 2 vectors
1779 // and the cost of creating shuffle.
1780 Instruction *I0 = cast<Instruction>(VL[0]);
1781 VecCost =
1782 TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
1783 Instruction *I1 = cast<Instruction>(VL[1]);
1784 VecCost +=
1785 TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
1786 VecCost +=
1787 TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
1788 return VecCost - ScalarCost;
1789 }
1790 default:
1791 llvm_unreachable("Unknown instruction")::llvm::llvm_unreachable_internal("Unknown instruction", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1791)
;
1792 }
1793}
1794
1795bool BoUpSLP::isFullyVectorizableTinyTree() {
1796 DEBUG(dbgs() << "SLP: Check whether the tree with height " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
1797 VectorizableTree.size() << " is fully vectorizable .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Check whether the tree with height "
<< VectorizableTree.size() << " is fully vectorizable .\n"
; } } while (false)
;
1798
1799 // We only handle trees of heights 1 and 2.
1800 if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
1801 return true;
1802
1803 if (VectorizableTree.size() != 2)
1804 return false;
1805
1806 // Handle splat and all-constants stores.
1807 if (!VectorizableTree[0].NeedToGather &&
1808 (allConstant(VectorizableTree[1].Scalars) ||
1809 isSplat(VectorizableTree[1].Scalars)))
1810 return true;
1811
1812 // Gathering cost would be too much for tiny trees.
1813 if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
1814 return false;
1815
1816 return true;
1817}
1818
1819bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
1820
1821 // We can vectorize the tree if its size is greater than or equal to the
1822 // minimum size specified by the MinTreeSize command line option.
1823 if (VectorizableTree.size() >= MinTreeSize)
1824 return false;
1825
1826 // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
1827 // can vectorize it if we can prove it fully vectorizable.
1828 if (isFullyVectorizableTinyTree())
1829 return false;
1830
1831 assert(VectorizableTree.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1833, __PRETTY_FUNCTION__))
1832 ? ExternalUses.empty()((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1833, __PRETTY_FUNCTION__))
1833 : true && "We shouldn't have any external users")((VectorizableTree.empty() ? ExternalUses.empty() : true &&
"We shouldn't have any external users") ? static_cast<void
> (0) : __assert_fail ("VectorizableTree.empty() ? ExternalUses.empty() : true && \"We shouldn't have any external users\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 1833, __PRETTY_FUNCTION__))
;
1834
1835 // Otherwise, we can't vectorize the tree. It is both tiny and not fully
1836 // vectorizable.
1837 return true;
1838}
1839
1840int BoUpSLP::getSpillCost() {
1841 // Walk from the bottom of the tree to the top, tracking which values are
1842 // live. When we see a call instruction that is not part of our tree,
1843 // query TTI to see if there is a cost to keeping values live over it
1844 // (for example, if spills and fills are required).
1845 unsigned BundleWidth = VectorizableTree.front().Scalars.size();
1846 int Cost = 0;
1847
1848 SmallPtrSet<Instruction*, 4> LiveValues;
1849 Instruction *PrevInst = nullptr;
1850
1851 for (const auto &N : VectorizableTree) {
1852 Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
1853 if (!Inst)
1854 continue;
1855
1856 if (!PrevInst) {
1857 PrevInst = Inst;
1858 continue;
1859 }
1860
1861 // Update LiveValues.
1862 LiveValues.erase(PrevInst);
1863 for (auto &J : PrevInst->operands()) {
1864 if (isa<Instruction>(&*J) && ScalarToTreeEntry.count(&*J))
1865 LiveValues.insert(cast<Instruction>(&*J));
1866 }
1867
1868 DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1869 dbgs() << "SLP: #LV: " << LiveValues.size();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1870 for (auto *X : LiveValues)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1871 dbgs() << " " << X->getName();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1872 dbgs() << ", Looking at ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1873 Inst->dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
1874 )do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: #LV: " << LiveValues.size
(); for (auto *X : LiveValues) dbgs() << " " << X
->getName(); dbgs() << ", Looking at "; Inst->dump
();; } } while (false)
;
1875
1876 // Now find the sequence of instructions between PrevInst and Inst.
1877 BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
1878 PrevInstIt =
1879 PrevInst->getIterator().getReverse();
1880 while (InstIt != PrevInstIt) {
1881 if (PrevInstIt == PrevInst->getParent()->rend()) {
1882 PrevInstIt = Inst->getParent()->rbegin();
1883 continue;
1884 }
1885
1886 if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
1887 SmallVector<Type*, 4> V;
1888 for (auto *II : LiveValues)
1889 V.push_back(VectorType::get(II->getType(), BundleWidth));
1890 Cost += TTI->getCostOfKeepingLiveOverCall(V);
1891 }
1892
1893 ++PrevInstIt;
1894 }
1895
1896 PrevInst = Inst;
1897 }
1898
1899 return Cost;
1900}
1901
1902int BoUpSLP::getTreeCost() {
1903 int Cost = 0;
1904 DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
1905 VectorizableTree.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Calculating cost for tree of size "
<< VectorizableTree.size() << ".\n"; } } while (
false)
;
1906
1907 unsigned BundleWidth = VectorizableTree[0].Scalars.size();
1908
1909 for (TreeEntry &TE : VectorizableTree) {
1910 int C = getEntryCost(&TE);
1911 DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
1912 << *TE.Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << C <<
" for bundle that starts with " << *TE.Scalars[0] <<
".\n"; } } while (false)
;
1913 Cost += C;
1914 }
1915
1916 SmallSet<Value *, 16> ExtractCostCalculated;
1917 int ExtractCost = 0;
1918 for (ExternalUser &EU : ExternalUses) {
1919 // We only add extract cost once for the same scalar.
1920 if (!ExtractCostCalculated.insert(EU.Scalar).second)
1921 continue;
1922
1923 // Uses by ephemeral values are free (because the ephemeral value will be
1924 // removed prior to code generation, and so the extraction will be
1925 // removed as well).
1926 if (EphValues.count(EU.User))
1927 continue;
1928
1929 // If we plan to rewrite the tree in a smaller type, we will need to sign
1930 // extend the extracted value back to the original type. Here, we account
1931 // for the extract and the added cost of the sign extend if needed.
1932 auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
1933 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
1934 if (MinBWs.count(ScalarRoot)) {
1935 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
1936 auto Extend =
1937 MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
1938 VecTy = VectorType::get(MinTy, BundleWidth);
1939 ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
1940 VecTy, EU.Lane);
1941 } else {
1942 ExtractCost +=
1943 TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
1944 }
1945 }
1946
1947 int SpillCost = getSpillCost();
1948 Cost += SpillCost + ExtractCost;
1949
1950 DEBUG(dbgs() << "SLP: Spill Cost = " << SpillCost << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Spill Cost = " << SpillCost
<< ".\n" << "SLP: Extract Cost = " << ExtractCost
<< ".\n" << "SLP: Total Cost = " << Cost <<
".\n"; } } while (false)
1951 << "SLP: Extract Cost = " << ExtractCost << ".\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Spill Cost = " << SpillCost
<< ".\n" << "SLP: Extract Cost = " << ExtractCost
<< ".\n" << "SLP: Total Cost = " << Cost <<
".\n"; } } while (false)
1952 << "SLP: Total Cost = " << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Spill Cost = " << SpillCost
<< ".\n" << "SLP: Extract Cost = " << ExtractCost
<< ".\n" << "SLP: Total Cost = " << Cost <<
".\n"; } } while (false)
;
1953 return Cost;
1954}
1955
1956int BoUpSLP::getGatherCost(Type *Ty) {
1957 int Cost = 0;
1958 for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
1959 Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
1960 return Cost;
1961}
1962
1963int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
1964 // Find the type of the operands in VL.
1965 Type *ScalarTy = VL[0]->getType();
1966 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1967 ScalarTy = SI->getValueOperand()->getType();
1968 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1969 // Find the cost of inserting/extracting values from the vector.
1970 return getGatherCost(VecTy);
1971}
1972
1973// Reorder commutative operations in alternate shuffle if the resulting vectors
1974// are consecutive loads. This would allow us to vectorize the tree.
1975// If we have something like-
1976// load a[0] - load b[0]
1977// load b[1] + load a[1]
1978// load a[2] - load b[2]
1979// load a[3] + load b[3]
1980// Reordering the second load b[1] load a[1] would allow us to vectorize this
1981// code.
1982void BoUpSLP::reorderAltShuffleOperands(ArrayRef<Value *> VL,
1983 SmallVectorImpl<Value *> &Left,
1984 SmallVectorImpl<Value *> &Right) {
1985 // Push left and right operands of binary operation into Left and Right
1986 for (Value *i : VL) {
1987 Left.push_back(cast<Instruction>(i)->getOperand(0));
1988 Right.push_back(cast<Instruction>(i)->getOperand(1));
1989 }
1990
1991 // Reorder if we have a commutative operation and consecutive access
1992 // are on either side of the alternate instructions.
1993 for (unsigned j = 0; j < VL.size() - 1; ++j) {
1994 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
1995 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
1996 Instruction *VL1 = cast<Instruction>(VL[j]);
1997 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
1998 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
1999 std::swap(Left[j], Right[j]);
2000 continue;
2001 } else if (VL2->isCommutative() &&
2002 isConsecutiveAccess(L, L1, *DL, *SE)) {
2003 std::swap(Left[j + 1], Right[j + 1]);
2004 continue;
2005 }
2006 // else unchanged
2007 }
2008 }
2009 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2010 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2011 Instruction *VL1 = cast<Instruction>(VL[j]);
2012 Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2013 if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2014 std::swap(Left[j], Right[j]);
2015 continue;
2016 } else if (VL2->isCommutative() &&
2017 isConsecutiveAccess(L, L1, *DL, *SE)) {
2018 std::swap(Left[j + 1], Right[j + 1]);
2019 continue;
2020 }
2021 // else unchanged
2022 }
2023 }
2024 }
2025}
2026
2027// Return true if I should be commuted before adding it's left and right
2028// operands to the arrays Left and Right.
2029//
2030// The vectorizer is trying to either have all elements one side being
2031// instruction with the same opcode to enable further vectorization, or having
2032// a splat to lower the vectorizing cost.
2033static bool shouldReorderOperands(int i, Instruction &I,
2034 SmallVectorImpl<Value *> &Left,
2035 SmallVectorImpl<Value *> &Right,
2036 bool AllSameOpcodeLeft,
2037 bool AllSameOpcodeRight, bool SplatLeft,
2038 bool SplatRight) {
2039 Value *VLeft = I.getOperand(0);
2040 Value *VRight = I.getOperand(1);
2041 // If we have "SplatRight", try to see if commuting is needed to preserve it.
2042 if (SplatRight) {
2043 if (VRight == Right[i - 1])
2044 // Preserve SplatRight
2045 return false;
2046 if (VLeft == Right[i - 1]) {
2047 // Commuting would preserve SplatRight, but we don't want to break
2048 // SplatLeft either, i.e. preserve the original order if possible.
2049 // (FIXME: why do we care?)
2050 if (SplatLeft && VLeft == Left[i - 1])
2051 return false;
2052 return true;
2053 }
2054 }
2055 // Symmetrically handle Right side.
2056 if (SplatLeft) {
2057 if (VLeft == Left[i - 1])
2058 // Preserve SplatLeft
2059 return false;
2060 if (VRight == Left[i - 1])
2061 return true;
2062 }
2063
2064 Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2065 Instruction *IRight = dyn_cast<Instruction>(VRight);
2066
2067 // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2068 // it and not the right, in this case we want to commute.
2069 if (AllSameOpcodeRight) {
2070 unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2071 if (IRight && RightPrevOpcode == IRight->getOpcode())
2072 // Do not commute, a match on the right preserves AllSameOpcodeRight
2073 return false;
2074 if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2075 // We have a match and may want to commute, but first check if there is
2076 // not also a match on the existing operands on the Left to preserve
2077 // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2078 // (FIXME: why do we care?)
2079 if (AllSameOpcodeLeft && ILeft &&
2080 cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2081 return false;
2082 return true;
2083 }
2084 }
2085 // Symmetrically handle Left side.
2086 if (AllSameOpcodeLeft) {
2087 unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2088 if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2089 return false;
2090 if (IRight && LeftPrevOpcode == IRight->getOpcode())
2091 return true;
2092 }
2093 return false;
2094}
2095
2096void BoUpSLP::reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
2097 SmallVectorImpl<Value *> &Left,
2098 SmallVectorImpl<Value *> &Right) {
2099
2100 if (VL.size()) {
2101 // Peel the first iteration out of the loop since there's nothing
2102 // interesting to do anyway and it simplifies the checks in the loop.
2103 auto VLeft = cast<Instruction>(VL[0])->getOperand(0);
2104 auto VRight = cast<Instruction>(VL[0])->getOperand(1);
2105 if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2106 // Favor having instruction to the right. FIXME: why?
2107 std::swap(VLeft, VRight);
2108 Left.push_back(VLeft);
2109 Right.push_back(VRight);
2110 }
2111
2112 // Keep track if we have instructions with all the same opcode on one side.
2113 bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2114 bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2115 // Keep track if we have one side with all the same value (broadcast).
2116 bool SplatLeft = true;
2117 bool SplatRight = true;
2118
2119 for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2120 Instruction *I = cast<Instruction>(VL[i]);
2121 assert(I->isCommutative() && "Can only process commutative instruction")((I->isCommutative() && "Can only process commutative instruction"
) ? static_cast<void> (0) : __assert_fail ("I->isCommutative() && \"Can only process commutative instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2121, __PRETTY_FUNCTION__))
;
2122 // Commute to favor either a splat or maximizing having the same opcodes on
2123 // one side.
2124 if (shouldReorderOperands(i, *I, Left, Right, AllSameOpcodeLeft,
2125 AllSameOpcodeRight, SplatLeft, SplatRight)) {
2126 Left.push_back(I->getOperand(1));
2127 Right.push_back(I->getOperand(0));
2128 } else {
2129 Left.push_back(I->getOperand(0));
2130 Right.push_back(I->getOperand(1));
2131 }
2132 // Update Splat* and AllSameOpcode* after the insertion.
2133 SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2134 SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2135 AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2136 (cast<Instruction>(Left[i - 1])->getOpcode() ==
2137 cast<Instruction>(Left[i])->getOpcode());
2138 AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2139 (cast<Instruction>(Right[i - 1])->getOpcode() ==
2140 cast<Instruction>(Right[i])->getOpcode());
2141 }
2142
2143 // If one operand end up being broadcast, return this operand order.
2144 if (SplatRight || SplatLeft)
2145 return;
2146
2147 // Finally check if we can get longer vectorizable chain by reordering
2148 // without breaking the good operand order detected above.
2149 // E.g. If we have something like-
2150 // load a[0] load b[0]
2151 // load b[1] load a[1]
2152 // load a[2] load b[2]
2153 // load a[3] load b[3]
2154 // Reordering the second load b[1] load a[1] would allow us to vectorize
2155 // this code and we still retain AllSameOpcode property.
2156 // FIXME: This load reordering might break AllSameOpcode in some rare cases
2157 // such as-
2158 // add a[0],c[0] load b[0]
2159 // add a[1],c[2] load b[1]
2160 // b[2] load b[2]
2161 // add a[3],c[3] load b[3]
2162 for (unsigned j = 0; j < VL.size() - 1; ++j) {
2163 if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2164 if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2165 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2166 std::swap(Left[j + 1], Right[j + 1]);
2167 continue;
2168 }
2169 }
2170 }
2171 if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2172 if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2173 if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2174 std::swap(Left[j + 1], Right[j + 1]);
2175 continue;
2176 }
2177 }
2178 }
2179 // else unchanged
2180 }
2181}
2182
2183void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL) {
2184
2185 // Get the basic block this bundle is in. All instructions in the bundle
2186 // should be in this block.
2187 auto *Front = cast<Instruction>(VL.front());
2188 auto *BB = Front->getParent();
2189 assert(all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool {((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2191, __PRETTY_FUNCTION__))
2190 return cast<Instruction>(V)->getParent() == BB;((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2191, __PRETTY_FUNCTION__))
2191 }))((all_of(make_range(VL.begin(), VL.end()), [&](Value *V) ->
bool { return cast<Instruction>(V)->getParent() == BB
; })) ? static_cast<void> (0) : __assert_fail ("all_of(make_range(VL.begin(), VL.end()), [&](Value *V) -> bool { return cast<Instruction>(V)->getParent() == BB; })"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2191, __PRETTY_FUNCTION__))
;
2192
2193 // The last instruction in the bundle in program order.
2194 Instruction *LastInst = nullptr;
2195
2196 // Find the last instruction. The common case should be that BB has been
2197 // scheduled, and the last instruction is VL.back(). So we start with
2198 // VL.back() and iterate over schedule data until we reach the end of the
2199 // bundle. The end of the bundle is marked by null ScheduleData.
2200 if (BlocksSchedules.count(BB)) {
2201 auto *Bundle = BlocksSchedules[BB]->getScheduleData(VL.back());
2202 if (Bundle && Bundle->isPartOfBundle())
2203 for (; Bundle; Bundle = Bundle->NextInBundle)
2204 LastInst = Bundle->Inst;
2205 }
2206
2207 // LastInst can still be null at this point if there's either not an entry
2208 // for BB in BlocksSchedules or there's no ScheduleData available for
2209 // VL.back(). This can be the case if buildTree_rec aborts for various
2210 // reasons (e.g., the maximum recursion depth is reached, the maximum region
2211 // size is reached, etc.). ScheduleData is initialized in the scheduling
2212 // "dry-run".
2213 //
2214 // If this happens, we can still find the last instruction by brute force. We
2215 // iterate forwards from Front (inclusive) until we either see all
2216 // instructions in the bundle or reach the end of the block. If Front is the
2217 // last instruction in program order, LastInst will be set to Front, and we
2218 // will visit all the remaining instructions in the block.
2219 //
2220 // One of the reasons we exit early from buildTree_rec is to place an upper
2221 // bound on compile-time. Thus, taking an additional compile-time hit here is
2222 // not ideal. However, this should be exceedingly rare since it requires that
2223 // we both exit early from buildTree_rec and that the bundle be out-of-order
2224 // (causing us to iterate all the way to the end of the block).
2225 if (!LastInst) {
2226 SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2227 for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2228 if (Bundle.erase(&I))
2229 LastInst = &I;
2230 if (Bundle.empty())
2231 break;
2232 }
2233 }
2234
2235 // Set the insertion point after the last instruction in the bundle. Set the
2236 // debug location to Front.
2237 Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2238 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2239}
2240
2241Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2242 Value *Vec = UndefValue::get(Ty);
2243 // Generate the 'InsertElement' instruction.
2244 for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2245 Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2246 if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2247 GatherSeq.insert(Insrt);
2248 CSEBlocks.insert(Insrt->getParent());
2249
2250 // Add to our 'need-to-extract' list.
2251 if (ScalarToTreeEntry.count(VL[i])) {
2252 int Idx = ScalarToTreeEntry[VL[i]];
2253 TreeEntry *E = &VectorizableTree[Idx];
2254 // Find which lane we need to extract.
2255 int FoundLane = -1;
2256 for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2257 // Is this the lane of the scalar that we are looking for ?
2258 if (E->Scalars[Lane] == VL[i]) {
2259 FoundLane = Lane;
2260 break;
2261 }
2262 }
2263 assert(FoundLane >= 0 && "Could not find the correct lane")((FoundLane >= 0 && "Could not find the correct lane"
) ? static_cast<void> (0) : __assert_fail ("FoundLane >= 0 && \"Could not find the correct lane\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2263, __PRETTY_FUNCTION__))
;
2264 ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2265 }
2266 }
2267 }
2268
2269 return Vec;
2270}
2271
2272Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL) const {
2273 SmallDenseMap<Value*, int>::const_iterator Entry
2274 = ScalarToTreeEntry.find(VL[0]);
2275 if (Entry != ScalarToTreeEntry.end()) {
2276 int Idx = Entry->second;
2277 const TreeEntry *En = &VectorizableTree[Idx];
2278 if (En->isSame(VL) && En->VectorizedValue)
2279 return En->VectorizedValue;
2280 }
2281 return nullptr;
2282}
2283
2284Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2285 if (ScalarToTreeEntry.count(VL[0])) {
2286 int Idx = ScalarToTreeEntry[VL[0]];
2287 TreeEntry *E = &VectorizableTree[Idx];
2288 if (E->isSame(VL))
2289 return vectorizeTree(E);
2290 }
2291
2292 Type *ScalarTy = VL[0]->getType();
2293 if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2294 ScalarTy = SI->getValueOperand()->getType();
2295 VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2296
2297 return Gather(VL, VecTy);
2298}
2299
2300Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2301 IRBuilder<>::InsertPointGuard Guard(Builder);
2302
2303 if (E->VectorizedValue) {
2304 DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Diamond merged for " <<
*E->Scalars[0] << ".\n"; } } while (false)
;
2305 return E->VectorizedValue;
2306 }
2307
2308 Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2309 Type *ScalarTy = VL0->getType();
2310 if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2311 ScalarTy = SI->getValueOperand()->getType();
2312 VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2313
2314 if (E->NeedToGather) {
2315 setInsertPointAfterBundle(E->Scalars);
2316 auto *V = Gather(E->Scalars, VecTy);
2317 E->VectorizedValue = V;
2318 return V;
2319 }
2320
2321 unsigned Opcode = getSameOpcode(E->Scalars);
2322
2323 switch (Opcode) {
2324 case Instruction::PHI: {
2325 PHINode *PH = dyn_cast<PHINode>(VL0);
2326 Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2327 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2328 PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2329 E->VectorizedValue = NewPhi;
2330
2331 // PHINodes may have multiple entries from the same block. We want to
2332 // visit every block once.
2333 SmallSet<BasicBlock*, 4> VisitedBBs;
2334
2335 for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2336 ValueList Operands;
2337 BasicBlock *IBB = PH->getIncomingBlock(i);
2338
2339 if (!VisitedBBs.insert(IBB).second) {
2340 NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2341 continue;
2342 }
2343
2344 // Prepare the operand vector.
2345 for (Value *V : E->Scalars)
2346 Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2347
2348 Builder.SetInsertPoint(IBB->getTerminator());
2349 Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2350 Value *Vec = vectorizeTree(Operands);
2351 NewPhi->addIncoming(Vec, IBB);
2352 }
2353
2354 assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2355, __PRETTY_FUNCTION__))
2355 "Invalid number of incoming values")((NewPhi->getNumIncomingValues() == PH->getNumIncomingValues
() && "Invalid number of incoming values") ? static_cast
<void> (0) : __assert_fail ("NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() && \"Invalid number of incoming values\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2355, __PRETTY_FUNCTION__))
;
2356 return NewPhi;
2357 }
2358
2359 case Instruction::ExtractElement: {
2360 if (canReuseExtract(E->Scalars, Instruction::ExtractElement)) {
2361 Value *V = VL0->getOperand(0);
2362 E->VectorizedValue = V;
2363 return V;
2364 }
2365 setInsertPointAfterBundle(E->Scalars);
2366 auto *V = Gather(E->Scalars, VecTy);
2367 E->VectorizedValue = V;
2368 return V;
2369 }
2370 case Instruction::ExtractValue: {
2371 if (canReuseExtract(E->Scalars, Instruction::ExtractValue)) {
2372 LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2373 Builder.SetInsertPoint(LI);
2374 PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2375 Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2376 LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2377 E->VectorizedValue = V;
2378 return propagateMetadata(V, E->Scalars);
2379 }
2380 setInsertPointAfterBundle(E->Scalars);
2381 auto *V = Gather(E->Scalars, VecTy);
2382 E->VectorizedValue = V;
2383 return V;
2384 }
2385 case Instruction::ZExt:
2386 case Instruction::SExt:
2387 case Instruction::FPToUI:
2388 case Instruction::FPToSI:
2389 case Instruction::FPExt:
2390 case Instruction::PtrToInt:
2391 case Instruction::IntToPtr:
2392 case Instruction::SIToFP:
2393 case Instruction::UIToFP:
2394 case Instruction::Trunc:
2395 case Instruction::FPTrunc:
2396 case Instruction::BitCast: {
2397 ValueList INVL;
2398 for (Value *V : E->Scalars)
2399 INVL.push_back(cast<Instruction>(V)->getOperand(0));
2400
2401 setInsertPointAfterBundle(E->Scalars);
2402
2403 Value *InVec = vectorizeTree(INVL);
2404
2405 if (Value *V = alreadyVectorized(E->Scalars))
2406 return V;
2407
2408 CastInst *CI = dyn_cast<CastInst>(VL0);
2409 Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2410 E->VectorizedValue = V;
2411 ++NumVectorInstructions;
2412 return V;
2413 }
2414 case Instruction::FCmp:
2415 case Instruction::ICmp: {
2416 ValueList LHSV, RHSV;
2417 for (Value *V : E->Scalars) {
2418 LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2419 RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2420 }
2421
2422 setInsertPointAfterBundle(E->Scalars);
2423
2424 Value *L = vectorizeTree(LHSV);
2425 Value *R = vectorizeTree(RHSV);
2426
2427 if (Value *V = alreadyVectorized(E->Scalars))
2428 return V;
2429
2430 CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2431 Value *V;
2432 if (Opcode == Instruction::FCmp)
2433 V = Builder.CreateFCmp(P0, L, R);
2434 else
2435 V = Builder.CreateICmp(P0, L, R);
2436
2437 E->VectorizedValue = V;
2438 propagateIRFlags(E->VectorizedValue, E->Scalars);
2439 ++NumVectorInstructions;
2440 return V;
2441 }
2442 case Instruction::Select: {
2443 ValueList TrueVec, FalseVec, CondVec;
2444 for (Value *V : E->Scalars) {
2445 CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2446 TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2447 FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2448 }
2449
2450 setInsertPointAfterBundle(E->Scalars);
2451
2452 Value *Cond = vectorizeTree(CondVec);
2453 Value *True = vectorizeTree(TrueVec);
2454 Value *False = vectorizeTree(FalseVec);
2455
2456 if (Value *V = alreadyVectorized(E->Scalars))
2457 return V;
2458
2459 Value *V = Builder.CreateSelect(Cond, True, False);
2460 E->VectorizedValue = V;
2461 ++NumVectorInstructions;
2462 return V;
2463 }
2464 case Instruction::Add:
2465 case Instruction::FAdd:
2466 case Instruction::Sub:
2467 case Instruction::FSub:
2468 case Instruction::Mul:
2469 case Instruction::FMul:
2470 case Instruction::UDiv:
2471 case Instruction::SDiv:
2472 case Instruction::FDiv:
2473 case Instruction::URem:
2474 case Instruction::SRem:
2475 case Instruction::FRem:
2476 case Instruction::Shl:
2477 case Instruction::LShr:
2478 case Instruction::AShr:
2479 case Instruction::And:
2480 case Instruction::Or:
2481 case Instruction::Xor: {
2482 ValueList LHSVL, RHSVL;
2483 if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2484 reorderInputsAccordingToOpcode(E->Scalars, LHSVL, RHSVL);
2485 else
2486 for (Value *V : E->Scalars) {
2487 LHSVL.push_back(cast<Instruction>(V)->getOperand(0));
2488 RHSVL.push_back(cast<Instruction>(V)->getOperand(1));
2489 }
2490
2491 setInsertPointAfterBundle(E->Scalars);
2492
2493 Value *LHS = vectorizeTree(LHSVL);
2494 Value *RHS = vectorizeTree(RHSVL);
2495
2496 if (LHS == RHS && isa<Instruction>(LHS)) {
2497 assert((VL0->getOperand(0) == VL0->getOperand(1)) && "Invalid order")(((VL0->getOperand(0) == VL0->getOperand(1)) &&
"Invalid order") ? static_cast<void> (0) : __assert_fail
("(VL0->getOperand(0) == VL0->getOperand(1)) && \"Invalid order\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2497, __PRETTY_FUNCTION__))
;
2498 }
2499
2500 if (Value *V = alreadyVectorized(E->Scalars))
2501 return V;
2502
2503 BinaryOperator *BinOp = cast<BinaryOperator>(VL0);
2504 Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS);
2505 E->VectorizedValue = V;
2506 propagateIRFlags(E->VectorizedValue, E->Scalars);
2507 ++NumVectorInstructions;
2508
2509 if (Instruction *I = dyn_cast<Instruction>(V))
2510 return propagateMetadata(I, E->Scalars);
2511
2512 return V;
2513 }
2514 case Instruction::Load: {
2515 // Loads are inserted at the head of the tree because we don't want to
2516 // sink them all the way down past store instructions.
2517 setInsertPointAfterBundle(E->Scalars);
2518
2519 LoadInst *LI = cast<LoadInst>(VL0);
2520 Type *ScalarLoadTy = LI->getType();
2521 unsigned AS = LI->getPointerAddressSpace();
2522
2523 Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2524 VecTy->getPointerTo(AS));
2525
2526 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2527 // ExternalUses list to make sure that an extract will be generated in the
2528 // future.
2529 if (ScalarToTreeEntry.count(LI->getPointerOperand()))
2530 ExternalUses.push_back(
2531 ExternalUser(LI->getPointerOperand(), cast<User>(VecPtr), 0));
2532
2533 unsigned Alignment = LI->getAlignment();
2534 LI = Builder.CreateLoad(VecPtr);
2535 if (!Alignment) {
2536 Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2537 }
2538 LI->setAlignment(Alignment);
2539 E->VectorizedValue = LI;
2540 ++NumVectorInstructions;
2541 return propagateMetadata(LI, E->Scalars);
2542 }
2543 case Instruction::Store: {
2544 StoreInst *SI = cast<StoreInst>(VL0);
2545 unsigned Alignment = SI->getAlignment();
2546 unsigned AS = SI->getPointerAddressSpace();
2547
2548 ValueList ValueOp;
2549 for (Value *V : E->Scalars)
2550 ValueOp.push_back(cast<StoreInst>(V)->getValueOperand());
2551
2552 setInsertPointAfterBundle(E->Scalars);
2553
2554 Value *VecValue = vectorizeTree(ValueOp);
2555 Value *VecPtr = Builder.CreateBitCast(SI->getPointerOperand(),
2556 VecTy->getPointerTo(AS));
2557 StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
2558
2559 // The pointer operand uses an in-tree scalar so we add the new BitCast to
2560 // ExternalUses list to make sure that an extract will be generated in the
2561 // future.
2562 if (ScalarToTreeEntry.count(SI->getPointerOperand()))
2563 ExternalUses.push_back(
2564 ExternalUser(SI->getPointerOperand(), cast<User>(VecPtr), 0));
2565
2566 if (!Alignment) {
2567 Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
2568 }
2569 S->setAlignment(Alignment);
2570 E->VectorizedValue = S;
2571 ++NumVectorInstructions;
2572 return propagateMetadata(S, E->Scalars);
2573 }
2574 case Instruction::GetElementPtr: {
2575 setInsertPointAfterBundle(E->Scalars);
2576
2577 ValueList Op0VL;
2578 for (Value *V : E->Scalars)
2579 Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
2580
2581 Value *Op0 = vectorizeTree(Op0VL);
2582
2583 std::vector<Value *> OpVecs;
2584 for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
2585 ++j) {
2586 ValueList OpVL;
2587 for (Value *V : E->Scalars)
2588 OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
2589
2590 Value *OpVec = vectorizeTree(OpVL);
2591 OpVecs.push_back(OpVec);
2592 }
2593
2594 Value *V = Builder.CreateGEP(
2595 cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
2596 E->VectorizedValue = V;
2597 ++NumVectorInstructions;
2598
2599 if (Instruction *I = dyn_cast<Instruction>(V))
2600 return propagateMetadata(I, E->Scalars);
2601
2602 return V;
2603 }
2604 case Instruction::Call: {
2605 CallInst *CI = cast<CallInst>(VL0);
2606 setInsertPointAfterBundle(E->Scalars);
2607 Function *FI;
2608 Intrinsic::ID IID = Intrinsic::not_intrinsic;
2609 Value *ScalarArg = nullptr;
2610 if (CI && (FI = CI->getCalledFunction())) {
2611 IID = FI->getIntrinsicID();
2612 }
2613 std::vector<Value *> OpVecs;
2614 for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
2615 ValueList OpVL;
2616 // ctlz,cttz and powi are special intrinsics whose second argument is
2617 // a scalar. This argument should not be vectorized.
2618 if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
2619 CallInst *CEI = cast<CallInst>(E->Scalars[0]);
2620 ScalarArg = CEI->getArgOperand(j);
2621 OpVecs.push_back(CEI->getArgOperand(j));
2622 continue;
2623 }
2624 for (Value *V : E->Scalars) {
2625 CallInst *CEI = cast<CallInst>(V);
2626 OpVL.push_back(CEI->getArgOperand(j));
2627 }
2628
2629 Value *OpVec = vectorizeTree(OpVL);
2630 DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: OpVec[" << j << "]: "
<< *OpVec << "\n"; } } while (false)
;
2631 OpVecs.push_back(OpVec);
2632 }
2633
2634 Module *M = F->getParent();
2635 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2636 Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
2637 Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
2638 SmallVector<OperandBundleDef, 1> OpBundles;
2639 CI->getOperandBundlesAsDefs(OpBundles);
2640 Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
2641
2642 // The scalar argument uses an in-tree scalar so we add the new vectorized
2643 // call to ExternalUses list to make sure that an extract will be
2644 // generated in the future.
2645 if (ScalarArg && ScalarToTreeEntry.count(ScalarArg))
2646 ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
2647
2648 E->VectorizedValue = V;
2649 propagateIRFlags(E->VectorizedValue, E->Scalars);
2650 ++NumVectorInstructions;
2651 return V;
2652 }
2653 case Instruction::ShuffleVector: {
2654 ValueList LHSVL, RHSVL;
2655 assert(isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand")((isa<BinaryOperator>(VL0) && "Invalid Shuffle Vector Operand"
) ? static_cast<void> (0) : __assert_fail ("isa<BinaryOperator>(VL0) && \"Invalid Shuffle Vector Operand\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2655, __PRETTY_FUNCTION__))
;
2656 reorderAltShuffleOperands(E->Scalars, LHSVL, RHSVL);
2657 setInsertPointAfterBundle(E->Scalars);
2658
2659 Value *LHS = vectorizeTree(LHSVL);
2660 Value *RHS = vectorizeTree(RHSVL);
2661
2662 if (Value *V = alreadyVectorized(E->Scalars))
2663 return V;
2664
2665 // Create a vector of LHS op1 RHS
2666 BinaryOperator *BinOp0 = cast<BinaryOperator>(VL0);
2667 Value *V0 = Builder.CreateBinOp(BinOp0->getOpcode(), LHS, RHS);
2668
2669 // Create a vector of LHS op2 RHS
2670 Instruction *VL1 = cast<Instruction>(E->Scalars[1]);
2671 BinaryOperator *BinOp1 = cast<BinaryOperator>(VL1);
2672 Value *V1 = Builder.CreateBinOp(BinOp1->getOpcode(), LHS, RHS);
2673
2674 // Create shuffle to take alternate operations from the vector.
2675 // Also, gather up odd and even scalar ops to propagate IR flags to
2676 // each vector operation.
2677 ValueList OddScalars, EvenScalars;
2678 unsigned e = E->Scalars.size();
2679 SmallVector<Constant *, 8> Mask(e);
2680 for (unsigned i = 0; i < e; ++i) {
2681 if (i & 1) {
2682 Mask[i] = Builder.getInt32(e + i);
2683 OddScalars.push_back(E->Scalars[i]);
2684 } else {
2685 Mask[i] = Builder.getInt32(i);
2686 EvenScalars.push_back(E->Scalars[i]);
2687 }
2688 }
2689
2690 Value *ShuffleMask = ConstantVector::get(Mask);
2691 propagateIRFlags(V0, EvenScalars);
2692 propagateIRFlags(V1, OddScalars);
2693
2694 Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2695 E->VectorizedValue = V;
2696 ++NumVectorInstructions;
2697 if (Instruction *I = dyn_cast<Instruction>(V))
2698 return propagateMetadata(I, E->Scalars);
2699
2700 return V;
2701 }
2702 default:
2703 llvm_unreachable("unknown inst")::llvm::llvm_unreachable_internal("unknown inst", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2703)
;
2704 }
2705 return nullptr;
2706}
2707
2708Value *BoUpSLP::vectorizeTree() {
2709
2710 // All blocks must be scheduled before any instructions are inserted.
2711 for (auto &BSIter : BlocksSchedules) {
2712 scheduleBlock(BSIter.second.get());
2713 }
2714
2715 Builder.SetInsertPoint(&F->getEntryBlock().front());
2716 auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
2717
2718 // If the vectorized tree can be rewritten in a smaller type, we truncate the
2719 // vectorized root. InstCombine will then rewrite the entire expression. We
2720 // sign extend the extracted values below.
2721 auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2722 if (MinBWs.count(ScalarRoot)) {
2723 if (auto *I = dyn_cast<Instruction>(VectorRoot))
2724 Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
2725 auto BundleWidth = VectorizableTree[0].Scalars.size();
2726 auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2727 auto *VecTy = VectorType::get(MinTy, BundleWidth);
2728 auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
2729 VectorizableTree[0].VectorizedValue = Trunc;
2730 }
2731
2732 DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Extracting " << ExternalUses
.size() << " values .\n"; } } while (false)
;
2733
2734 // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
2735 // specified by ScalarType.
2736 auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
2737 if (!MinBWs.count(ScalarRoot))
2738 return Ex;
2739 if (MinBWs[ScalarRoot].second)
2740 return Builder.CreateSExt(Ex, ScalarType);
2741 return Builder.CreateZExt(Ex, ScalarType);
2742 };
2743
2744 // Extract all of the elements with the external uses.
2745 for (const auto &ExternalUse : ExternalUses) {
2746 Value *Scalar = ExternalUse.Scalar;
2747 llvm::User *User = ExternalUse.User;
2748
2749 // Skip users that we already RAUW. This happens when one instruction
2750 // has multiple uses of the same value.
2751 if (!is_contained(Scalar->users(), User))
2752 continue;
2753 assert(ScalarToTreeEntry.count(Scalar) && "Invalid scalar")((ScalarToTreeEntry.count(Scalar) && "Invalid scalar"
) ? static_cast<void> (0) : __assert_fail ("ScalarToTreeEntry.count(Scalar) && \"Invalid scalar\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2753, __PRETTY_FUNCTION__))
;
2754
2755 int Idx = ScalarToTreeEntry[Scalar];
2756 TreeEntry *E = &VectorizableTree[Idx];
2757 assert(!E->NeedToGather && "Extracting from a gather list")((!E->NeedToGather && "Extracting from a gather list"
) ? static_cast<void> (0) : __assert_fail ("!E->NeedToGather && \"Extracting from a gather list\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2757, __PRETTY_FUNCTION__))
;
2758
2759 Value *Vec = E->VectorizedValue;
2760 assert(Vec && "Can't find vectorizable value")((Vec && "Can't find vectorizable value") ? static_cast
<void> (0) : __assert_fail ("Vec && \"Can't find vectorizable value\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2760, __PRETTY_FUNCTION__))
;
2761
2762 Value *Lane = Builder.getInt32(ExternalUse.Lane);
2763 // Generate extracts for out-of-tree users.
2764 // Find the insertion point for the extractelement lane.
2765 if (auto *VecI = dyn_cast<Instruction>(Vec)) {
2766 if (PHINode *PH = dyn_cast<PHINode>(User)) {
2767 for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
2768 if (PH->getIncomingValue(i) == Scalar) {
2769 TerminatorInst *IncomingTerminator =
2770 PH->getIncomingBlock(i)->getTerminator();
2771 if (isa<CatchSwitchInst>(IncomingTerminator)) {
2772 Builder.SetInsertPoint(VecI->getParent(),
2773 std::next(VecI->getIterator()));
2774 } else {
2775 Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
2776 }
2777 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2778 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2779 CSEBlocks.insert(PH->getIncomingBlock(i));
2780 PH->setOperand(i, Ex);
2781 }
2782 }
2783 } else {
2784 Builder.SetInsertPoint(cast<Instruction>(User));
2785 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2786 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2787 CSEBlocks.insert(cast<Instruction>(User)->getParent());
2788 User->replaceUsesOfWith(Scalar, Ex);
2789 }
2790 } else {
2791 Builder.SetInsertPoint(&F->getEntryBlock().front());
2792 Value *Ex = Builder.CreateExtractElement(Vec, Lane);
2793 Ex = extend(ScalarRoot, Ex, Scalar->getType());
2794 CSEBlocks.insert(&F->getEntryBlock());
2795 User->replaceUsesOfWith(Scalar, Ex);
2796 }
2797
2798 DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Replaced:" << *User <<
".\n"; } } while (false)
;
2799 }
2800
2801 // For each vectorized value:
2802 for (TreeEntry &EIdx : VectorizableTree) {
2803 TreeEntry *Entry = &EIdx;
2804
2805 // For each lane:
2806 for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
2807 Value *Scalar = Entry->Scalars[Lane];
2808 // No need to handle users of gathered values.
2809 if (Entry->NeedToGather)
2810 continue;
2811
2812 assert(Entry->VectorizedValue && "Can't find vectorizable value")((Entry->VectorizedValue && "Can't find vectorizable value"
) ? static_cast<void> (0) : __assert_fail ("Entry->VectorizedValue && \"Can't find vectorizable value\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2812, __PRETTY_FUNCTION__))
;
2813
2814 Type *Ty = Scalar->getType();
2815 if (!Ty->isVoidTy()) {
2816#ifndef NDEBUG
2817 for (User *U : Scalar->users()) {
2818 DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tvalidating user:" <<
*U << ".\n"; } } while (false)
;
2819
2820 assert((ScalarToTreeEntry.count(U) ||(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2823, __PRETTY_FUNCTION__))
2821 // It is legal to replace users in the ignorelist by undef.(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2823, __PRETTY_FUNCTION__))
2822 is_contained(UserIgnoreList, U)) &&(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2823, __PRETTY_FUNCTION__))
2823 "Replacing out-of-tree value with undef")(((ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList,
U)) && "Replacing out-of-tree value with undef") ? static_cast
<void> (0) : __assert_fail ("(ScalarToTreeEntry.count(U) || is_contained(UserIgnoreList, U)) && \"Replacing out-of-tree value with undef\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2823, __PRETTY_FUNCTION__))
;
2824 }
2825#endif
2826 Value *Undef = UndefValue::get(Ty);
2827 Scalar->replaceAllUsesWith(Undef);
2828 }
2829 DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: \tErasing scalar:" << *
Scalar << ".\n"; } } while (false)
;
2830 eraseInstruction(cast<Instruction>(Scalar));
2831 }
2832 }
2833
2834 Builder.ClearInsertionPoint();
2835
2836 return VectorizableTree[0].VectorizedValue;
2837}
2838
2839void BoUpSLP::optimizeGatherSequence() {
2840 DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
2841 << " gather sequences instructions.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Optimizing " << GatherSeq
.size() << " gather sequences instructions.\n"; } } while
(false)
;
2842 // LICM InsertElementInst sequences.
2843 for (Instruction *it : GatherSeq) {
2844 InsertElementInst *Insert = dyn_cast<InsertElementInst>(it);
2845
2846 if (!Insert)
2847 continue;
2848
2849 // Check if this block is inside a loop.
2850 Loop *L = LI->getLoopFor(Insert->getParent());
2851 if (!L)
2852 continue;
2853
2854 // Check if it has a preheader.
2855 BasicBlock *PreHeader = L->getLoopPreheader();
2856 if (!PreHeader)
2857 continue;
2858
2859 // If the vector or the element that we insert into it are
2860 // instructions that are defined in this basic block then we can't
2861 // hoist this instruction.
2862 Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
2863 Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
2864 if (CurrVec && L->contains(CurrVec))
2865 continue;
2866 if (NewElem && L->contains(NewElem))
2867 continue;
2868
2869 // We can hoist this instruction. Move it to the pre-header.
2870 Insert->moveBefore(PreHeader->getTerminator());
2871 }
2872
2873 // Make a list of all reachable blocks in our CSE queue.
2874 SmallVector<const DomTreeNode *, 8> CSEWorkList;
2875 CSEWorkList.reserve(CSEBlocks.size());
2876 for (BasicBlock *BB : CSEBlocks)
2877 if (DomTreeNode *N = DT->getNode(BB)) {
2878 assert(DT->isReachableFromEntry(N))((DT->isReachableFromEntry(N)) ? static_cast<void> (
0) : __assert_fail ("DT->isReachableFromEntry(N)", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2878, __PRETTY_FUNCTION__))
;
2879 CSEWorkList.push_back(N);
2880 }
2881
2882 // Sort blocks by domination. This ensures we visit a block after all blocks
2883 // dominating it are visited.
2884 std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
2885 [this](const DomTreeNode *A, const DomTreeNode *B) {
2886 return DT->properlyDominates(A, B);
2887 });
2888
2889 // Perform O(N^2) search over the gather sequences and merge identical
2890 // instructions. TODO: We can further optimize this scan if we split the
2891 // instructions into different buckets based on the insert lane.
2892 SmallVector<Instruction *, 16> Visited;
2893 for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
1
Assuming 'I' is not equal to 'E'
2
Loop condition is true. Entering loop body
4
Loop condition is true. Entering loop body
6
Loop condition is true. Entering loop body
8
Loop condition is true. Entering loop body
2894 assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2895, __PRETTY_FUNCTION__))
2895 "Worklist not sorted properly!")(((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev
(I))) && "Worklist not sorted properly!") ? static_cast
<void> (0) : __assert_fail ("(I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) && \"Worklist not sorted properly!\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2895, __PRETTY_FUNCTION__))
;
2896 BasicBlock *BB = (*I)->getBlock();
9
Called C++ object pointer is null
2897 // For all instructions in blocks containing gather sequences:
2898 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3
Loop condition is false. Execution continues on line 2893
5
Loop condition is false. Execution continues on line 2893
7
Loop condition is false. Execution continues on line 2893
2899 Instruction *In = &*it++;
2900 if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
2901 continue;
2902
2903 // Check if we can replace this instruction with any of the
2904 // visited instructions.
2905 for (Instruction *v : Visited) {
2906 if (In->isIdenticalTo(v) &&
2907 DT->dominates(v->getParent(), In->getParent())) {
2908 In->replaceAllUsesWith(v);
2909 eraseInstruction(In);
2910 In = nullptr;
2911 break;
2912 }
2913 }
2914 if (In) {
2915 assert(!is_contained(Visited, In))((!is_contained(Visited, In)) ? static_cast<void> (0) :
__assert_fail ("!is_contained(Visited, In)", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2915, __PRETTY_FUNCTION__))
;
2916 Visited.push_back(In);
2917 }
2918 }
2919 }
2920 CSEBlocks.clear();
2921 GatherSeq.clear();
2922}
2923
2924// Groups the instructions to a bundle (which is then a single scheduling entity)
2925// and schedules instructions until the bundle gets ready.
2926bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
2927 BoUpSLP *SLP) {
2928 if (isa<PHINode>(VL[0]))
2929 return true;
2930
2931 // Initialize the instruction bundle.
2932 Instruction *OldScheduleEnd = ScheduleEnd;
2933 ScheduleData *PrevInBundle = nullptr;
2934 ScheduleData *Bundle = nullptr;
2935 bool ReSchedule = false;
2936 DEBUG(dbgs() << "SLP: bundle: " << *VL[0] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: bundle: " << *VL[0] <<
"\n"; } } while (false)
;
2937
2938 // Make sure that the scheduling region contains all
2939 // instructions of the bundle.
2940 for (Value *V : VL) {
2941 if (!extendSchedulingRegion(V))
2942 return false;
2943 }
2944
2945 for (Value *V : VL) {
2946 ScheduleData *BundleMember = getScheduleData(V);
2947 assert(BundleMember &&((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2948, __PRETTY_FUNCTION__))
2948 "no ScheduleData for bundle member (maybe not in same basic block)")((BundleMember && "no ScheduleData for bundle member (maybe not in same basic block)"
) ? static_cast<void> (0) : __assert_fail ("BundleMember && \"no ScheduleData for bundle member (maybe not in same basic block)\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2948, __PRETTY_FUNCTION__))
;
2949 if (BundleMember->IsScheduled) {
2950 // A bundle member was scheduled as single instruction before and now
2951 // needs to be scheduled as part of the bundle. We just get rid of the
2952 // existing schedule.
2953 DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMemberdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
2954 << " was already scheduled\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: reset schedule because " <<
*BundleMember << " was already scheduled\n"; } } while
(false)
;
2955 ReSchedule = true;
2956 }
2957 assert(BundleMember->isSchedulingEntity() &&((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2958, __PRETTY_FUNCTION__))
2958 "bundle member already part of other bundle")((BundleMember->isSchedulingEntity() && "bundle member already part of other bundle"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->isSchedulingEntity() && \"bundle member already part of other bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 2958, __PRETTY_FUNCTION__))
;
2959 if (PrevInBundle) {
2960 PrevInBundle->NextInBundle = BundleMember;
2961 } else {
2962 Bundle = BundleMember;
2963 }
2964 BundleMember->UnscheduledDepsInBundle = 0;
2965 Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
2966
2967 // Group the instructions to a bundle.
2968 BundleMember->FirstInBundle = Bundle;
2969 PrevInBundle = BundleMember;
2970 }
2971 if (ScheduleEnd != OldScheduleEnd) {
2972 // The scheduling region got new instructions at the lower end (or it is a
2973 // new region for the first bundle). This makes it necessary to
2974 // recalculate all dependencies.
2975 // It is seldom that this needs to be done a second time after adding the
2976 // initial bundle to the region.
2977 for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
2978 ScheduleData *SD = getScheduleData(I);
2979 SD->clearDependencies();
2980 }
2981 ReSchedule = true;
2982 }
2983 if (ReSchedule) {
2984 resetSchedule();
2985 initialFillReadyList(ReadyInsts);
2986 }
2987
2988 DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
2989 << BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: try schedule bundle " <<
*Bundle << " in block " << BB->getName() <<
"\n"; } } while (false)
;
2990
2991 calculateDependencies(Bundle, true, SLP);
2992
2993 // Now try to schedule the new bundle. As soon as the bundle is "ready" it
2994 // means that there are no cyclic dependencies and we can schedule it.
2995 // Note that's important that we don't "schedule" the bundle yet (see
2996 // cancelScheduling).
2997 while (!Bundle->isReady() && !ReadyInsts.empty()) {
2998
2999 ScheduleData *pickedSD = ReadyInsts.back();
3000 ReadyInsts.pop_back();
3001
3002 if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3003 schedule(pickedSD, ReadyInsts);
3004 }
3005 }
3006 if (!Bundle->isReady()) {
3007 cancelScheduling(VL);
3008 return false;
3009 }
3010 return true;
3011}
3012
3013void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL) {
3014 if (isa<PHINode>(VL[0]))
3015 return;
3016
3017 ScheduleData *Bundle = getScheduleData(VL[0]);
3018 DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: cancel scheduling of " <<
*Bundle << "\n"; } } while (false)
;
3019 assert(!Bundle->IsScheduled &&((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3020, __PRETTY_FUNCTION__))
3020 "Can't cancel bundle which is already scheduled")((!Bundle->IsScheduled && "Can't cancel bundle which is already scheduled"
) ? static_cast<void> (0) : __assert_fail ("!Bundle->IsScheduled && \"Can't cancel bundle which is already scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3020, __PRETTY_FUNCTION__))
;
3021 assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3022, __PRETTY_FUNCTION__))
3022 "tried to unbundle something which is not a bundle")((Bundle->isSchedulingEntity() && Bundle->isPartOfBundle
() && "tried to unbundle something which is not a bundle"
) ? static_cast<void> (0) : __assert_fail ("Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() && \"tried to unbundle something which is not a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3022, __PRETTY_FUNCTION__))
;
3023
3024 // Un-bundle: make single instructions out of the bundle.
3025 ScheduleData *BundleMember = Bundle;
3026 while (BundleMember) {
3027 assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links")((BundleMember->FirstInBundle == Bundle && "corrupt bundle links"
) ? static_cast<void> (0) : __assert_fail ("BundleMember->FirstInBundle == Bundle && \"corrupt bundle links\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3027, __PRETTY_FUNCTION__))
;
3028 BundleMember->FirstInBundle = BundleMember;
3029 ScheduleData *Next = BundleMember->NextInBundle;
3030 BundleMember->NextInBundle = nullptr;
3031 BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3032 if (BundleMember->UnscheduledDepsInBundle == 0) {
3033 ReadyInsts.insert(BundleMember);
3034 }
3035 BundleMember = Next;
3036 }
3037}
3038
3039bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V) {
3040 if (getScheduleData(V))
3041 return true;
3042 Instruction *I = dyn_cast<Instruction>(V);
3043 assert(I && "bundle member must be an instruction")((I && "bundle member must be an instruction") ? static_cast
<void> (0) : __assert_fail ("I && \"bundle member must be an instruction\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3043, __PRETTY_FUNCTION__))
;
3044 assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled")((!isa<PHINode>(I) && "phi nodes don't need to be scheduled"
) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(I) && \"phi nodes don't need to be scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3044, __PRETTY_FUNCTION__))
;
3045 if (!ScheduleStart) {
3046 // It's the first instruction in the new region.
3047 initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3048 ScheduleStart = I;
3049 ScheduleEnd = I->getNextNode();
3050 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3050, __PRETTY_FUNCTION__))
;
3051 DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: initialize schedule region to "
<< *I << "\n"; } } while (false)
;
3052 return true;
3053 }
3054 // Search up and down at the same time, because we don't know if the new
3055 // instruction is above or below the existing scheduling region.
3056 BasicBlock::reverse_iterator UpIter =
3057 ++ScheduleStart->getIterator().getReverse();
3058 BasicBlock::reverse_iterator UpperEnd = BB->rend();
3059 BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3060 BasicBlock::iterator LowerEnd = BB->end();
3061 for (;;) {
3062 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3063 DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: exceeded schedule region size limit\n"
; } } while (false)
;
3064 return false;
3065 }
3066
3067 if (UpIter != UpperEnd) {
3068 if (&*UpIter == I) {
3069 initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3070 ScheduleStart = I;
3071 DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region start to "
<< *I << "\n"; } } while (false)
;
3072 return true;
3073 }
3074 UpIter++;
3075 }
3076 if (DownIter != LowerEnd) {
3077 if (&*DownIter == I) {
3078 initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3079 nullptr);
3080 ScheduleEnd = I->getNextNode();
3081 assert(ScheduleEnd && "tried to vectorize a TerminatorInst?")((ScheduleEnd && "tried to vectorize a TerminatorInst?"
) ? static_cast<void> (0) : __assert_fail ("ScheduleEnd && \"tried to vectorize a TerminatorInst?\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3081, __PRETTY_FUNCTION__))
;
3082 DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: extend schedule region end to "
<< *I << "\n"; } } while (false)
;
3083 return true;
3084 }
3085 DownIter++;
3086 }
3087 assert((UpIter != UpperEnd || DownIter != LowerEnd) &&(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3088, __PRETTY_FUNCTION__))
3088 "instruction not found in block")(((UpIter != UpperEnd || DownIter != LowerEnd) && "instruction not found in block"
) ? static_cast<void> (0) : __assert_fail ("(UpIter != UpperEnd || DownIter != LowerEnd) && \"instruction not found in block\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3088, __PRETTY_FUNCTION__))
;
3089 }
3090 return true;
3091}
3092
3093void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3094 Instruction *ToI,
3095 ScheduleData *PrevLoadStore,
3096 ScheduleData *NextLoadStore) {
3097 ScheduleData *CurrentLoadStore = PrevLoadStore;
3098 for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3099 ScheduleData *SD = ScheduleDataMap[I];
3100 if (!SD) {
3101 // Allocate a new ScheduleData for the instruction.
3102 if (ChunkPos >= ChunkSize) {
3103 ScheduleDataChunks.push_back(
3104 llvm::make_unique<ScheduleData[]>(ChunkSize));
3105 ChunkPos = 0;
3106 }
3107 SD = &(ScheduleDataChunks.back()[ChunkPos++]);
3108 ScheduleDataMap[I] = SD;
3109 SD->Inst = I;
3110 }
3111 assert(!isInSchedulingRegion(SD) &&((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3112, __PRETTY_FUNCTION__))
3112 "new ScheduleData already in scheduling region")((!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"
) ? static_cast<void> (0) : __assert_fail ("!isInSchedulingRegion(SD) && \"new ScheduleData already in scheduling region\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3112, __PRETTY_FUNCTION__))
;
3113 SD->init(SchedulingRegionID);
3114
3115 if (I->mayReadOrWriteMemory()) {
3116 // Update the linked list of memory accessing instructions.
3117 if (CurrentLoadStore) {
3118 CurrentLoadStore->NextLoadStore = SD;
3119 } else {
3120 FirstLoadStoreInRegion = SD;
3121 }
3122 CurrentLoadStore = SD;
3123 }
3124 }
3125 if (NextLoadStore) {
3126 if (CurrentLoadStore)
3127 CurrentLoadStore->NextLoadStore = NextLoadStore;
3128 } else {
3129 LastLoadStoreInRegion = CurrentLoadStore;
3130 }
3131}
3132
3133void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3134 bool InsertInReadyList,
3135 BoUpSLP *SLP) {
3136 assert(SD->isSchedulingEntity())((SD->isSchedulingEntity()) ? static_cast<void> (0) :
__assert_fail ("SD->isSchedulingEntity()", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3136, __PRETTY_FUNCTION__))
;
3137
3138 SmallVector<ScheduleData *, 10> WorkList;
3139 WorkList.push_back(SD);
3140
3141 while (!WorkList.empty()) {
3142 ScheduleData *SD = WorkList.back();
3143 WorkList.pop_back();
3144
3145 ScheduleData *BundleMember = SD;
3146 while (BundleMember) {
3147 assert(isInSchedulingRegion(BundleMember))((isInSchedulingRegion(BundleMember)) ? static_cast<void>
(0) : __assert_fail ("isInSchedulingRegion(BundleMember)", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3147, __PRETTY_FUNCTION__))
;
3148 if (!BundleMember->hasValidDependencies()) {
3149
3150 DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: update deps of " <<
*BundleMember << "\n"; } } while (false)
;
3151 BundleMember->Dependencies = 0;
3152 BundleMember->resetUnscheduledDeps();
3153
3154 // Handle def-use chain dependencies.
3155 for (User *U : BundleMember->Inst->users()) {
3156 if (isa<Instruction>(U)) {
3157 ScheduleData *UseSD = getScheduleData(U);
3158 if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3159 BundleMember->Dependencies++;
3160 ScheduleData *DestBundle = UseSD->FirstInBundle;
3161 if (!DestBundle->IsScheduled) {
3162 BundleMember->incrementUnscheduledDeps(1);
3163 }
3164 if (!DestBundle->hasValidDependencies()) {
3165 WorkList.push_back(DestBundle);
3166 }
3167 }
3168 } else {
3169 // I'm not sure if this can ever happen. But we need to be safe.
3170 // This lets the instruction/bundle never be scheduled and
3171 // eventually disable vectorization.
3172 BundleMember->Dependencies++;
3173 BundleMember->incrementUnscheduledDeps(1);
3174 }
3175 }
3176
3177 // Handle the memory dependencies.
3178 ScheduleData *DepDest = BundleMember->NextLoadStore;
3179 if (DepDest) {
3180 Instruction *SrcInst = BundleMember->Inst;
3181 MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3182 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3183 unsigned numAliased = 0;
3184 unsigned DistToSrc = 1;
3185
3186 while (DepDest) {
3187 assert(isInSchedulingRegion(DepDest))((isInSchedulingRegion(DepDest)) ? static_cast<void> (0
) : __assert_fail ("isInSchedulingRegion(DepDest)", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3187, __PRETTY_FUNCTION__))
;
3188
3189 // We have two limits to reduce the complexity:
3190 // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3191 // SLP->isAliased (which is the expensive part in this loop).
3192 // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3193 // the whole loop (even if the loop is fast, it's quadratic).
3194 // It's important for the loop break condition (see below) to
3195 // check this limit even between two read-only instructions.
3196 if (DistToSrc >= MaxMemDepDistance ||
3197 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3198 (numAliased >= AliasedCheckLimit ||
3199 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3200
3201 // We increment the counter only if the locations are aliased
3202 // (instead of counting all alias checks). This gives a better
3203 // balance between reduced runtime and accurate dependencies.
3204 numAliased++;
3205
3206 DepDest->MemoryDependencies.push_back(BundleMember);
3207 BundleMember->Dependencies++;
3208 ScheduleData *DestBundle = DepDest->FirstInBundle;
3209 if (!DestBundle->IsScheduled) {
3210 BundleMember->incrementUnscheduledDeps(1);
3211 }
3212 if (!DestBundle->hasValidDependencies()) {
3213 WorkList.push_back(DestBundle);
3214 }
3215 }
3216 DepDest = DepDest->NextLoadStore;
3217
3218 // Example, explaining the loop break condition: Let's assume our
3219 // starting instruction is i0 and MaxMemDepDistance = 3.
3220 //
3221 // +--------v--v--v
3222 // i0,i1,i2,i3,i4,i5,i6,i7,i8
3223 // +--------^--^--^
3224 //
3225 // MaxMemDepDistance let us stop alias-checking at i3 and we add
3226 // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3227 // Previously we already added dependencies from i3 to i6,i7,i8
3228 // (because of MaxMemDepDistance). As we added a dependency from
3229 // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3230 // and we can abort this loop at i6.
3231 if (DistToSrc >= 2 * MaxMemDepDistance)
3232 break;
3233 DistToSrc++;
3234 }
3235 }
3236 }
3237 BundleMember = BundleMember->NextInBundle;
3238 }
3239 if (InsertInReadyList && SD->isReady()) {
3240 ReadyInsts.push_back(SD);
3241 DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: gets ready on update: " <<
*SD->Inst << "\n"; } } while (false)
;
3242 }
3243 }
3244}
3245
3246void BoUpSLP::BlockScheduling::resetSchedule() {
3247 assert(ScheduleStart &&((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3248, __PRETTY_FUNCTION__))
3248 "tried to reset schedule on block which has not been scheduled")((ScheduleStart && "tried to reset schedule on block which has not been scheduled"
) ? static_cast<void> (0) : __assert_fail ("ScheduleStart && \"tried to reset schedule on block which has not been scheduled\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3248, __PRETTY_FUNCTION__))
;
3249 for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3250 ScheduleData *SD = getScheduleData(I);
3251 assert(isInSchedulingRegion(SD))((isInSchedulingRegion(SD)) ? static_cast<void> (0) : __assert_fail
("isInSchedulingRegion(SD)", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3251, __PRETTY_FUNCTION__))
;
3252 SD->IsScheduled = false;
3253 SD->resetUnscheduledDeps();
3254 }
3255 ReadyInsts.clear();
3256}
3257
3258void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3259
3260 if (!BS->ScheduleStart)
3261 return;
3262
3263 DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: schedule block " << BS
->BB->getName() << "\n"; } } while (false)
;
3264
3265 BS->resetSchedule();
3266
3267 // For the real scheduling we use a more sophisticated ready-list: it is
3268 // sorted by the original instruction location. This lets the final schedule
3269 // be as close as possible to the original instruction order.
3270 struct ScheduleDataCompare {
3271 bool operator()(ScheduleData *SD1, ScheduleData *SD2) {
3272 return SD2->SchedulingPriority < SD1->SchedulingPriority;
3273 }
3274 };
3275 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3276
3277 // Ensure that all dependency data is updated and fill the ready-list with
3278 // initial instructions.
3279 int Idx = 0;
3280 int NumToSchedule = 0;
3281 for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3282 I = I->getNextNode()) {
3283 ScheduleData *SD = BS->getScheduleData(I);
3284 assert(((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3286, __PRETTY_FUNCTION__))
3285 SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) &&((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3286, __PRETTY_FUNCTION__))
3286 "scheduler and vectorizer have different opinion on what is a bundle")((SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->
Inst) != 0) && "scheduler and vectorizer have different opinion on what is a bundle"
) ? static_cast<void> (0) : __assert_fail ("SD->isPartOfBundle() == (ScalarToTreeEntry.count(SD->Inst) != 0) && \"scheduler and vectorizer have different opinion on what is a bundle\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3286, __PRETTY_FUNCTION__))
;
3287 SD->FirstInBundle->SchedulingPriority = Idx++;
3288 if (SD->isSchedulingEntity()) {
3289 BS->calculateDependencies(SD, false, this);
3290 NumToSchedule++;
3291 }
3292 }
3293 BS->initialFillReadyList(ReadyInsts);
3294
3295 Instruction *LastScheduledInst = BS->ScheduleEnd;
3296
3297 // Do the "real" scheduling.
3298 while (!ReadyInsts.empty()) {
3299 ScheduleData *picked = *ReadyInsts.begin();
3300 ReadyInsts.erase(ReadyInsts.begin());
3301
3302 // Move the scheduled instruction(s) to their dedicated places, if not
3303 // there yet.
3304 ScheduleData *BundleMember = picked;
3305 while (BundleMember) {
3306 Instruction *pickedInst = BundleMember->Inst;
3307 if (LastScheduledInst->getNextNode() != pickedInst) {
3308 BS->BB->getInstList().remove(pickedInst);
3309 BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3310 pickedInst);
3311 }
3312 LastScheduledInst = pickedInst;
3313 BundleMember = BundleMember->NextInBundle;
3314 }
3315
3316 BS->schedule(picked, ReadyInsts);
3317 NumToSchedule--;
3318 }
3319 assert(NumToSchedule == 0 && "could not schedule all instructions")((NumToSchedule == 0 && "could not schedule all instructions"
) ? static_cast<void> (0) : __assert_fail ("NumToSchedule == 0 && \"could not schedule all instructions\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3319, __PRETTY_FUNCTION__))
;
3320
3321 // Avoid duplicate scheduling of the block.
3322 BS->ScheduleStart = nullptr;
3323}
3324
3325unsigned BoUpSLP::getVectorElementSize(Value *V) {
3326 // If V is a store, just return the width of the stored value without
3327 // traversing the expression tree. This is the common case.
3328 if (auto *Store = dyn_cast<StoreInst>(V))
3329 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3330
3331 // If V is not a store, we can traverse the expression tree to find loads
3332 // that feed it. The type of the loaded value may indicate a more suitable
3333 // width than V's type. We want to base the vector element size on the width
3334 // of memory operations where possible.
3335 SmallVector<Instruction *, 16> Worklist;
3336 SmallPtrSet<Instruction *, 16> Visited;
3337 if (auto *I = dyn_cast<Instruction>(V))
3338 Worklist.push_back(I);
3339
3340 // Traverse the expression tree in bottom-up order looking for loads. If we
3341 // encounter an instruciton we don't yet handle, we give up.
3342 auto MaxWidth = 0u;
3343 auto FoundUnknownInst = false;
3344 while (!Worklist.empty() && !FoundUnknownInst) {
3345 auto *I = Worklist.pop_back_val();
3346 Visited.insert(I);
3347
3348 // We should only be looking at scalar instructions here. If the current
3349 // instruction has a vector type, give up.
3350 auto *Ty = I->getType();
3351 if (isa<VectorType>(Ty))
3352 FoundUnknownInst = true;
3353
3354 // If the current instruction is a load, update MaxWidth to reflect the
3355 // width of the loaded value.
3356 else if (isa<LoadInst>(I))
3357 MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3358
3359 // Otherwise, we need to visit the operands of the instruction. We only
3360 // handle the interesting cases from buildTree here. If an operand is an
3361 // instruction we haven't yet visited, we add it to the worklist.
3362 else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3363 isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3364 for (Use &U : I->operands())
3365 if (auto *J = dyn_cast<Instruction>(U.get()))
3366 if (!Visited.count(J))
3367 Worklist.push_back(J);
3368 }
3369
3370 // If we don't yet handle the instruction, give up.
3371 else
3372 FoundUnknownInst = true;
3373 }
3374
3375 // If we didn't encounter a memory access in the expression tree, or if we
3376 // gave up for some reason, just return the width of V.
3377 if (!MaxWidth || FoundUnknownInst)
3378 return DL->getTypeSizeInBits(V->getType());
3379
3380 // Otherwise, return the maximum width we found.
3381 return MaxWidth;
3382}
3383
3384// Determine if a value V in a vectorizable expression Expr can be demoted to a
3385// smaller type with a truncation. We collect the values that will be demoted
3386// in ToDemote and additional roots that require investigating in Roots.
3387static bool collectValuesToDemote(Value *V, SmallPtrSetImpl<Value *> &Expr,
3388 SmallVectorImpl<Value *> &ToDemote,
3389 SmallVectorImpl<Value *> &Roots) {
3390
3391 // We can always demote constants.
3392 if (isa<Constant>(V)) {
3393 ToDemote.push_back(V);
3394 return true;
3395 }
3396
3397 // If the value is not an instruction in the expression with only one use, it
3398 // cannot be demoted.
3399 auto *I = dyn_cast<Instruction>(V);
3400 if (!I || !I->hasOneUse() || !Expr.count(I))
3401 return false;
3402
3403 switch (I->getOpcode()) {
3404
3405 // We can always demote truncations and extensions. Since truncations can
3406 // seed additional demotion, we save the truncated value.
3407 case Instruction::Trunc:
3408 Roots.push_back(I->getOperand(0));
3409 case Instruction::ZExt:
3410 case Instruction::SExt:
3411 break;
3412
3413 // We can demote certain binary operations if we can demote both of their
3414 // operands.
3415 case Instruction::Add:
3416 case Instruction::Sub:
3417 case Instruction::Mul:
3418 case Instruction::And:
3419 case Instruction::Or:
3420 case Instruction::Xor:
3421 if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3422 !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3423 return false;
3424 break;
3425
3426 // We can demote selects if we can demote their true and false values.
3427 case Instruction::Select: {
3428 SelectInst *SI = cast<SelectInst>(I);
3429 if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3430 !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3431 return false;
3432 break;
3433 }
3434
3435 // We can demote phis if we can demote all their incoming operands. Note that
3436 // we don't need to worry about cycles since we ensure single use above.
3437 case Instruction::PHI: {
3438 PHINode *PN = cast<PHINode>(I);
3439 for (Value *IncValue : PN->incoming_values())
3440 if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3441 return false;
3442 break;
3443 }
3444
3445 // Otherwise, conservatively give up.
3446 default:
3447 return false;
3448 }
3449
3450 // Record the value that we can demote.
3451 ToDemote.push_back(V);
3452 return true;
3453}
3454
3455void BoUpSLP::computeMinimumValueSizes() {
3456 // If there are no external uses, the expression tree must be rooted by a
3457 // store. We can't demote in-memory values, so there is nothing to do here.
3458 if (ExternalUses.empty())
3459 return;
3460
3461 // We only attempt to truncate integer expressions.
3462 auto &TreeRoot = VectorizableTree[0].Scalars;
3463 auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3464 if (!TreeRootIT)
3465 return;
3466
3467 // If the expression is not rooted by a store, these roots should have
3468 // external uses. We will rely on InstCombine to rewrite the expression in
3469 // the narrower type. However, InstCombine only rewrites single-use values.
3470 // This means that if a tree entry other than a root is used externally, it
3471 // must have multiple uses and InstCombine will not rewrite it. The code
3472 // below ensures that only the roots are used externally.
3473 SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3474 for (auto &EU : ExternalUses)
3475 if (!Expr.erase(EU.Scalar))
3476 return;
3477 if (!Expr.empty())
3478 return;
3479
3480 // Collect the scalar values of the vectorizable expression. We will use this
3481 // context to determine which values can be demoted. If we see a truncation,
3482 // we mark it as seeding another demotion.
3483 for (auto &Entry : VectorizableTree)
3484 Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3485
3486 // Ensure the roots of the vectorizable tree don't form a cycle. They must
3487 // have a single external user that is not in the vectorizable tree.
3488 for (auto *Root : TreeRoot)
3489 if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
3490 return;
3491
3492 // Conservatively determine if we can actually truncate the roots of the
3493 // expression. Collect the values that can be demoted in ToDemote and
3494 // additional roots that require investigating in Roots.
3495 SmallVector<Value *, 32> ToDemote;
3496 SmallVector<Value *, 4> Roots;
3497 for (auto *Root : TreeRoot)
3498 if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
3499 return;
3500
3501 // The maximum bit width required to represent all the values that can be
3502 // demoted without loss of precision. It would be safe to truncate the roots
3503 // of the expression to this width.
3504 auto MaxBitWidth = 8u;
3505
3506 // We first check if all the bits of the roots are demanded. If they're not,
3507 // we can truncate the roots to this narrower type.
3508 for (auto *Root : TreeRoot) {
3509 auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
3510 MaxBitWidth = std::max<unsigned>(
3511 Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
3512 }
3513
3514 // True if the roots can be zero-extended back to their original type, rather
3515 // than sign-extended. We know that if the leading bits are not demanded, we
3516 // can safely zero-extend. So we initialize IsKnownPositive to True.
3517 bool IsKnownPositive = true;
3518
3519 // If all the bits of the roots are demanded, we can try a little harder to
3520 // compute a narrower type. This can happen, for example, if the roots are
3521 // getelementptr indices. InstCombine promotes these indices to the pointer
3522 // width. Thus, all their bits are technically demanded even though the
3523 // address computation might be vectorized in a smaller type.
3524 //
3525 // We start by looking at each entry that can be demoted. We compute the
3526 // maximum bit width required to store the scalar by using ValueTracking to
3527 // compute the number of high-order bits we can truncate.
3528 if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
3529 MaxBitWidth = 8u;
3530
3531 // Determine if the sign bit of all the roots is known to be zero. If not,
3532 // IsKnownPositive is set to False.
3533 IsKnownPositive = all_of(TreeRoot, [&](Value *R) {
3534 bool KnownZero = false;
3535 bool KnownOne = false;
3536 ComputeSignBit(R, KnownZero, KnownOne, *DL);
3537 return KnownZero;
3538 });
3539
3540 // Determine the maximum number of bits required to store the scalar
3541 // values.
3542 for (auto *Scalar : ToDemote) {
3543 auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, 0, DT);
3544 auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
3545 MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
3546 }
3547
3548 // If we can't prove that the sign bit is zero, we must add one to the
3549 // maximum bit width to account for the unknown sign bit. This preserves
3550 // the existing sign bit so we can safely sign-extend the root back to the
3551 // original type. Otherwise, if we know the sign bit is zero, we will
3552 // zero-extend the root instead.
3553 //
3554 // FIXME: This is somewhat suboptimal, as there will be cases where adding
3555 // one to the maximum bit width will yield a larger-than-necessary
3556 // type. In general, we need to add an extra bit only if we can't
3557 // prove that the upper bit of the original type is equal to the
3558 // upper bit of the proposed smaller type. If these two bits are the
3559 // same (either zero or one) we know that sign-extending from the
3560 // smaller type will result in the same value. Here, since we can't
3561 // yet prove this, we are just making the proposed smaller type
3562 // larger to ensure correctness.
3563 if (!IsKnownPositive)
3564 ++MaxBitWidth;
3565 }
3566
3567 // Round MaxBitWidth up to the next power-of-two.
3568 if (!isPowerOf2_64(MaxBitWidth))
3569 MaxBitWidth = NextPowerOf2(MaxBitWidth);
3570
3571 // If the maximum bit width we compute is less than the with of the roots'
3572 // type, we can proceed with the narrowing. Otherwise, do nothing.
3573 if (MaxBitWidth >= TreeRootIT->getBitWidth())
3574 return;
3575
3576 // If we can truncate the root, we must collect additional values that might
3577 // be demoted as a result. That is, those seeded by truncations we will
3578 // modify.
3579 while (!Roots.empty())
3580 collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
3581
3582 // Finally, map the values we can demote to the maximum bit with we computed.
3583 for (auto *Scalar : ToDemote)
3584 MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
3585}
3586
3587namespace {
3588/// The SLPVectorizer Pass.
3589struct SLPVectorizer : public FunctionPass {
3590 SLPVectorizerPass Impl;
3591
3592 /// Pass identification, replacement for typeid
3593 static char ID;
3594
3595 explicit SLPVectorizer() : FunctionPass(ID) {
3596 initializeSLPVectorizerPass(*PassRegistry::getPassRegistry());
3597 }
3598
3599
3600 bool doInitialization(Module &M) override {
3601 return false;
3602 }
3603
3604 bool runOnFunction(Function &F) override {
3605 if (skipFunction(F))
3606 return false;
3607
3608 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
3609 auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
3610 auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
3611 auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
3612 auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3613 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
3614 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
3615 auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
3616 auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
3617
3618 return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB);
3619 }
3620
3621 void getAnalysisUsage(AnalysisUsage &AU) const override {
3622 FunctionPass::getAnalysisUsage(AU);
3623 AU.addRequired<AssumptionCacheTracker>();
3624 AU.addRequired<ScalarEvolutionWrapperPass>();
3625 AU.addRequired<AAResultsWrapperPass>();
3626 AU.addRequired<TargetTransformInfoWrapperPass>();
3627 AU.addRequired<LoopInfoWrapperPass>();
3628 AU.addRequired<DominatorTreeWrapperPass>();
3629 AU.addRequired<DemandedBitsWrapperPass>();
3630 AU.addPreserved<LoopInfoWrapperPass>();
3631 AU.addPreserved<DominatorTreeWrapperPass>();
3632 AU.addPreserved<AAResultsWrapperPass>();
3633 AU.addPreserved<GlobalsAAWrapperPass>();
3634 AU.setPreservesCFG();
3635 }
3636};
3637} // end anonymous namespace
3638
3639PreservedAnalyses SLPVectorizerPass::run(Function &F, FunctionAnalysisManager &AM) {
3640 auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
3641 auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
3642 auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
3643 auto *AA = &AM.getResult<AAManager>(F);
3644 auto *LI = &AM.getResult<LoopAnalysis>(F);
3645 auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
3646 auto *AC = &AM.getResult<AssumptionAnalysis>(F);
3647 auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
3648
3649 bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB);
3650 if (!Changed)
3651 return PreservedAnalyses::all();
3652 PreservedAnalyses PA;
3653 PA.preserve<LoopAnalysis>();
3654 PA.preserve<DominatorTreeAnalysis>();
3655 PA.preserve<AAManager>();
3656 PA.preserve<GlobalsAA>();
3657 return PA;
3658}
3659
3660bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
3661 TargetTransformInfo *TTI_,
3662 TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
3663 LoopInfo *LI_, DominatorTree *DT_,
3664 AssumptionCache *AC_, DemandedBits *DB_) {
3665 SE = SE_;
3666 TTI = TTI_;
3667 TLI = TLI_;
3668 AA = AA_;
3669 LI = LI_;
3670 DT = DT_;
3671 AC = AC_;
3672 DB = DB_;
3673 DL = &F.getParent()->getDataLayout();
3674
3675 Stores.clear();
3676 GEPs.clear();
3677 bool Changed = false;
3678
3679 // If the target claims to have no vector registers don't attempt
3680 // vectorization.
3681 if (!TTI->getNumberOfRegisters(true))
3682 return false;
3683
3684 // Don't vectorize when the attribute NoImplicitFloat is used.
3685 if (F.hasFnAttribute(Attribute::NoImplicitFloat))
3686 return false;
3687
3688 DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing blocks in " <<
F.getName() << ".\n"; } } while (false)
;
3689
3690 // Use the bottom up slp vectorizer to construct chains that start with
3691 // store instructions.
3692 BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL);
3693
3694 // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
3695 // delete instructions.
3696
3697 // Scan the blocks in the function in post order.
3698 for (auto BB : post_order(&F.getEntryBlock())) {
3699 collectSeedInstructions(BB);
3700
3701 // Vectorize trees that end at stores.
3702 if (!Stores.empty()) {
3703 DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
3704 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found stores for " << Stores
.size() << " underlying objects.\n"; } } while (false)
;
3705 Changed |= vectorizeStoreChains(R);
3706 }
3707
3708 // Vectorize trees that end at reductions.
3709 Changed |= vectorizeChainsInBlock(BB, R);
3710
3711 // Vectorize the index computations of getelementptr instructions. This
3712 // is primarily intended to catch gather-like idioms ending at
3713 // non-consecutive loads.
3714 if (!GEPs.empty()) {
3715 DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
3716 << " underlying objects.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found GEPs for " << GEPs
.size() << " underlying objects.\n"; } } while (false)
;
3717 Changed |= vectorizeGEPIndices(BB, R);
3718 }
3719 }
3720
3721 if (Changed) {
3722 R.optimizeGatherSequence();
3723 DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: vectorized \"" << F.getName
() << "\"\n"; } } while (false)
;
3724 DEBUG(verifyFunction(F))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { verifyFunction(F); } } while (false)
;
3725 }
3726 return Changed;
3727}
3728
3729/// \brief Check that the Values in the slice in VL array are still existent in
3730/// the WeakVH array.
3731/// Vectorization of part of the VL array may cause later values in the VL array
3732/// to become invalid. We track when this has happened in the WeakVH array.
3733static bool hasValueBeenRAUWed(ArrayRef<Value *> VL, ArrayRef<WeakVH> VH,
3734 unsigned SliceBegin, unsigned SliceSize) {
3735 VL = VL.slice(SliceBegin, SliceSize);
3736 VH = VH.slice(SliceBegin, SliceSize);
3737 return !std::equal(VL.begin(), VL.end(), VH.begin());
3738}
3739
3740bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
3741 unsigned VecRegSize) {
3742 unsigned ChainLen = Chain.size();
3743 DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLendo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
3744 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< ChainLen << "\n"; } } while (false)
;
3745 unsigned Sz = R.getVectorElementSize(Chain[0]);
3746 unsigned VF = VecRegSize / Sz;
3747
3748 if (!isPowerOf2_32(Sz) || VF < 2)
3749 return false;
3750
3751 // Keep track of values that were deleted by vectorizing in the loop below.
3752 SmallVector<WeakVH, 8> TrackValues(Chain.begin(), Chain.end());
3753
3754 bool Changed = false;
3755 // Look for profitable vectorizable trees at all offsets, starting at zero.
3756 for (unsigned i = 0, e = ChainLen; i < e; ++i) {
3757 if (i + VF > e)
3758 break;
3759
3760 // Check that a previous iteration of this loop did not delete the Value.
3761 if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
3762 continue;
3763
3764 DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << ido { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
3765 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << VF <<
" stores at offset " << i << "\n"; } } while (false
)
;
3766 ArrayRef<Value *> Operands = Chain.slice(i, VF);
3767
3768 R.buildTree(Operands);
3769 if (R.isTreeTinyAndNotFullyVectorizable())
3770 continue;
3771
3772 R.computeMinimumValueSizes();
3773
3774 int Cost = R.getTreeCost();
3775
3776 DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Found cost=" << Cost <<
" for VF=" << VF << "\n"; } } while (false)
;
3777 if (Cost < -SLPCostThreshold) {
3778 DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Decided to vectorize cost=" <<
Cost << "\n"; } } while (false)
;
3779 R.vectorizeTree();
3780
3781 // Move to the next bundle.
3782 i += VF - 1;
3783 Changed = true;
3784 }
3785 }
3786
3787 return Changed;
3788}
3789
3790bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
3791 BoUpSLP &R) {
3792 SetVector<StoreInst *> Heads, Tails;
3793 SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
3794
3795 // We may run into multiple chains that merge into a single chain. We mark the
3796 // stores that we vectorized so that we don't visit the same store twice.
3797 BoUpSLP::ValueSet VectorizedStores;
3798 bool Changed = false;
3799
3800 // Do a quadratic search on all of the given stores and find
3801 // all of the pairs of stores that follow each other.
3802 SmallVector<unsigned, 16> IndexQueue;
3803 for (unsigned i = 0, e = Stores.size(); i < e; ++i) {
3804 IndexQueue.clear();
3805 // If a store has multiple consecutive store candidates, search Stores
3806 // array according to the sequence: from i+1 to e, then from i-1 to 0.
3807 // This is because usually pairing with immediate succeeding or preceding
3808 // candidate create the best chance to find slp vectorization opportunity.
3809 unsigned j = 0;
3810 for (j = i + 1; j < e; ++j)
3811 IndexQueue.push_back(j);
3812 for (j = i; j > 0; --j)
3813 IndexQueue.push_back(j - 1);
3814
3815 for (auto &k : IndexQueue) {
3816 if (isConsecutiveAccess(Stores[i], Stores[k], *DL, *SE)) {
3817 Tails.insert(Stores[k]);
3818 Heads.insert(Stores[i]);
3819 ConsecutiveChain[Stores[i]] = Stores[k];
3820 break;
3821 }
3822 }
3823 }
3824
3825 // For stores that start but don't end a link in the chain:
3826 for (SetVector<StoreInst *>::iterator it = Heads.begin(), e = Heads.end();
3827 it != e; ++it) {
3828 if (Tails.count(*it))
3829 continue;
3830
3831 // We found a store instr that starts a chain. Now follow the chain and try
3832 // to vectorize it.
3833 BoUpSLP::ValueList Operands;
3834 StoreInst *I = *it;
3835 // Collect the chain into a list.
3836 while (Tails.count(I) || Heads.count(I)) {
3837 if (VectorizedStores.count(I))
3838 break;
3839 Operands.push_back(I);
3840 // Move to the next value in the chain.
3841 I = ConsecutiveChain[I];
3842 }
3843
3844 // FIXME: Is division-by-2 the correct step? Should we assert that the
3845 // register size is a power-of-2?
3846 for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
3847 Size /= 2) {
3848 if (vectorizeStoreChain(Operands, R, Size)) {
3849 // Mark the vectorized stores so that we don't vectorize them again.
3850 VectorizedStores.insert(Operands.begin(), Operands.end());
3851 Changed = true;
3852 break;
3853 }
3854 }
3855 }
3856
3857 return Changed;
3858}
3859
3860void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
3861
3862 // Initialize the collections. We will make a single pass over the block.
3863 Stores.clear();
3864 GEPs.clear();
3865
3866 // Visit the store and getelementptr instructions in BB and organize them in
3867 // Stores and GEPs according to the underlying objects of their pointer
3868 // operands.
3869 for (Instruction &I : *BB) {
3870
3871 // Ignore store instructions that are volatile or have a pointer operand
3872 // that doesn't point to a scalar type.
3873 if (auto *SI = dyn_cast<StoreInst>(&I)) {
3874 if (!SI->isSimple())
3875 continue;
3876 if (!isValidElementType(SI->getValueOperand()->getType()))
3877 continue;
3878 Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
3879 }
3880
3881 // Ignore getelementptr instructions that have more than one index, a
3882 // constant index, or a pointer operand that doesn't point to a scalar
3883 // type.
3884 else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
3885 auto Idx = GEP->idx_begin()->get();
3886 if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
3887 continue;
3888 if (!isValidElementType(Idx->getType()))
3889 continue;
3890 if (GEP->getType()->isVectorTy())
3891 continue;
3892 GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
3893 }
3894 }
3895}
3896
3897bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
3898 if (!A || !B)
3899 return false;
3900 Value *VL[] = { A, B };
3901 return tryToVectorizeList(VL, R, None, true);
3902}
3903
3904bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
3905 ArrayRef<Value *> BuildVector,
3906 bool AllowReorder) {
3907 if (VL.size() < 2)
3908 return false;
3909
3910 DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
3911 << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Trying to vectorize a list of length = "
<< VL.size() << ".\n"; } } while (false)
;
3912
3913 // Check that all of the parts are scalar instructions of the same type.
3914 Instruction *I0 = dyn_cast<Instruction>(VL[0]);
3915 if (!I0)
3916 return false;
3917
3918 unsigned Opcode0 = I0->getOpcode();
3919
3920 unsigned Sz = R.getVectorElementSize(I0);
3921 unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
3922 unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
3923 if (MaxVF < 2)
3924 return false;
3925
3926 for (Value *V : VL) {
3927 Type *Ty = V->getType();
3928 if (!isValidElementType(Ty))
3929 return false;
3930 Instruction *Inst = dyn_cast<Instruction>(V);
3931 if (!Inst || Inst->getOpcode() != Opcode0)
3932 return false;
3933 }
3934
3935 bool Changed = false;
3936
3937 // Keep track of values that were deleted by vectorizing in the loop below.
3938 SmallVector<WeakVH, 8> TrackValues(VL.begin(), VL.end());
3939
3940 unsigned NextInst = 0, MaxInst = VL.size();
3941 for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
3942 VF /= 2) {
3943 // No actual vectorization should happen, if number of parts is the same as
3944 // provided vectorization factor (i.e. the scalar type is used for vector
3945 // code during codegen).
3946 auto *VecTy = VectorType::get(VL[0]->getType(), VF);
3947 if (TTI->getNumberOfParts(VecTy) == VF)
3948 continue;
3949 for (unsigned I = NextInst; I < MaxInst; ++I) {
3950 unsigned OpsWidth = 0;
3951
3952 if (I + VF > MaxInst)
3953 OpsWidth = MaxInst - I;
3954 else
3955 OpsWidth = VF;
3956
3957 if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
3958 break;
3959
3960 // Check that a previous iteration of this loop did not delete the Value.
3961 if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
3962 continue;
3963
3964 DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
3965 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing " << OpsWidth
<< " operations " << "\n"; } } while (false)
;
3966 ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
3967
3968 ArrayRef<Value *> BuildVectorSlice;
3969 if (!BuildVector.empty())
3970 BuildVectorSlice = BuildVector.slice(I, OpsWidth);
3971
3972 R.buildTree(Ops, BuildVectorSlice);
3973 // TODO: check if we can allow reordering for more cases.
3974 if (AllowReorder && R.shouldReorder()) {
3975 // Conceptually, there is nothing actually preventing us from trying to
3976 // reorder a larger list. In fact, we do exactly this when vectorizing
3977 // reductions. However, at this point, we only expect to get here from
3978 // tryToVectorizePair().
3979 assert(Ops.size() == 2)((Ops.size() == 2) ? static_cast<void> (0) : __assert_fail
("Ops.size() == 2", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3979, __PRETTY_FUNCTION__))
;
3980 assert(BuildVectorSlice.empty())((BuildVectorSlice.empty()) ? static_cast<void> (0) : __assert_fail
("BuildVectorSlice.empty()", "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 3980, __PRETTY_FUNCTION__))
;
3981 Value *ReorderedOps[] = {Ops[1], Ops[0]};
3982 R.buildTree(ReorderedOps, None);
3983 }
3984 if (R.isTreeTinyAndNotFullyVectorizable())
3985 continue;
3986
3987 R.computeMinimumValueSizes();
3988 int Cost = R.getTreeCost();
3989
3990 if (Cost < -SLPCostThreshold) {
3991 DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing list at cost:" <<
Cost << ".\n"; } } while (false)
;
3992 Value *VectorizedRoot = R.vectorizeTree();
3993
3994 // Reconstruct the build vector by extracting the vectorized root. This
3995 // way we handle the case where some elements of the vector are
3996 // undefined.
3997 // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
3998 if (!BuildVectorSlice.empty()) {
3999 // The insert point is the last build vector instruction. The
4000 // vectorized root will precede it. This guarantees that we get an
4001 // instruction. The vectorized tree could have been constant folded.
4002 Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4003 unsigned VecIdx = 0;
4004 for (auto &V : BuildVectorSlice) {
4005 IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4006 ++BasicBlock::iterator(InsertAfter));
4007 Instruction *I = cast<Instruction>(V);
4008 assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I))((isa<InsertElementInst>(I) || isa<InsertValueInst>
(I)) ? static_cast<void> (0) : __assert_fail ("isa<InsertElementInst>(I) || isa<InsertValueInst>(I)"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4008, __PRETTY_FUNCTION__))
;
4009 Instruction *Extract =
4010 cast<Instruction>(Builder.CreateExtractElement(
4011 VectorizedRoot, Builder.getInt32(VecIdx++)));
4012 I->setOperand(1, Extract);
4013 I->removeFromParent();
4014 I->insertAfter(Extract);
4015 InsertAfter = I;
4016 }
4017 }
4018 // Move to the next bundle.
4019 I += VF - 1;
4020 NextInst = I + 1;
4021 Changed = true;
4022 }
4023 }
4024 }
4025
4026 return Changed;
4027}
4028
4029bool SLPVectorizerPass::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
4030 if (!V)
4031 return false;
4032
4033 Value *P = V->getParent();
4034
4035 // Vectorize in current basic block only.
4036 auto *Op0 = dyn_cast<Instruction>(V->getOperand(0));
4037 auto *Op1 = dyn_cast<Instruction>(V->getOperand(1));
4038 if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4039 return false;
4040
4041 // Try to vectorize V.
4042 if (tryToVectorizePair(Op0, Op1, R))
4043 return true;
4044
4045 auto *A = dyn_cast<BinaryOperator>(Op0);
4046 auto *B = dyn_cast<BinaryOperator>(Op1);
4047 // Try to skip B.
4048 if (B && B->hasOneUse()) {
4049 auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4050 auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4051 if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4052 return true;
4053 if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4054 return true;
4055 }
4056
4057 // Try to skip A.
4058 if (A && A->hasOneUse()) {
4059 auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4060 auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4061 if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4062 return true;
4063 if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4064 return true;
4065 }
4066 return false;
4067}
4068
4069/// \brief Generate a shuffle mask to be used in a reduction tree.
4070///
4071/// \param VecLen The length of the vector to be reduced.
4072/// \param NumEltsToRdx The number of elements that should be reduced in the
4073/// vector.
4074/// \param IsPairwise Whether the reduction is a pairwise or splitting
4075/// reduction. A pairwise reduction will generate a mask of
4076/// <0,2,...> or <1,3,..> while a splitting reduction will generate
4077/// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4078/// \param IsLeft True will generate a mask of even elements, odd otherwise.
4079static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4080 bool IsPairwise, bool IsLeft,
4081 IRBuilder<> &Builder) {
4082 assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask")(((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask"
) ? static_cast<void> (0) : __assert_fail ("(IsPairwise || !IsLeft) && \"Don't support a <0,1,undef,...> mask\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4082, __PRETTY_FUNCTION__))
;
4083
4084 SmallVector<Constant *, 32> ShuffleMask(
4085 VecLen, UndefValue::get(Builder.getInt32Ty()));
4086
4087 if (IsPairwise)
4088 // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4089 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4090 ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4091 else
4092 // Move the upper half of the vector to the lower half.
4093 for (unsigned i = 0; i != NumEltsToRdx; ++i)
4094 ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4095
4096 return ConstantVector::get(ShuffleMask);
4097}
4098
4099namespace {
4100/// Model horizontal reductions.
4101///
4102/// A horizontal reduction is a tree of reduction operations (currently add and
4103/// fadd) that has operations that can be put into a vector as its leaf.
4104/// For example, this tree:
4105///
4106/// mul mul mul mul
4107/// \ / \ /
4108/// + +
4109/// \ /
4110/// +
4111/// This tree has "mul" as its reduced values and "+" as its reduction
4112/// operations. A reduction might be feeding into a store or a binary operation
4113/// feeding a phi.
4114/// ...
4115/// \ /
4116/// +
4117/// |
4118/// phi +=
4119///
4120/// Or:
4121/// ...
4122/// \ /
4123/// +
4124/// |
4125/// *p =
4126///
4127class HorizontalReduction {
4128 SmallVector<Value *, 16> ReductionOps;
4129 SmallVector<Value *, 32> ReducedVals;
4130
4131 BinaryOperator *ReductionRoot;
4132 // After successfull horizontal reduction vectorization attempt for PHI node
4133 // vectorizer tries to update root binary op by combining vectorized tree and
4134 // the ReductionPHI node. But during vectorization this ReductionPHI can be
4135 // vectorized itself and replaced by the undef value, while the instruction
4136 // itself is marked for deletion. This 'marked for deletion' PHI node then can
4137 // be used in new binary operation, causing "Use still stuck around after Def
4138 // is destroyed" crash upon PHI node deletion.
4139 WeakVH ReductionPHI;
4140
4141 /// The opcode of the reduction.
4142 unsigned ReductionOpcode;
4143 /// The opcode of the values we perform a reduction on.
4144 unsigned ReducedValueOpcode;
4145 /// Should we model this reduction as a pairwise reduction tree or a tree that
4146 /// splits the vector in halves and adds those halves.
4147 bool IsPairwiseReduction;
4148
4149public:
4150 /// The width of one full horizontal reduction operation.
4151 unsigned ReduxWidth;
4152
4153 /// Minimal width of available vector registers. It's used to determine
4154 /// ReduxWidth.
4155 unsigned MinVecRegSize;
4156
4157 HorizontalReduction(unsigned MinVecRegSize)
4158 : ReductionRoot(nullptr), ReductionOpcode(0), ReducedValueOpcode(0),
4159 IsPairwiseReduction(false), ReduxWidth(0),
4160 MinVecRegSize(MinVecRegSize) {}
4161
4162 /// \brief Try to find a reduction tree.
4163 bool matchAssociativeReduction(PHINode *Phi, BinaryOperator *B) {
4164 assert((!Phi || is_contained(Phi->operands(), B)) &&(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4165, __PRETTY_FUNCTION__))
4165 "Thi phi needs to use the binary operator")(((!Phi || is_contained(Phi->operands(), B)) && "Thi phi needs to use the binary operator"
) ? static_cast<void> (0) : __assert_fail ("(!Phi || is_contained(Phi->operands(), B)) && \"Thi phi needs to use the binary operator\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4165, __PRETTY_FUNCTION__))
;
4166
4167 // We could have a initial reductions that is not an add.
4168 // r *= v1 + v2 + v3 + v4
4169 // In such a case start looking for a tree rooted in the first '+'.
4170 if (Phi) {
4171 if (B->getOperand(0) == Phi) {
4172 Phi = nullptr;
4173 B = dyn_cast<BinaryOperator>(B->getOperand(1));
4174 } else if (B->getOperand(1) == Phi) {
4175 Phi = nullptr;
4176 B = dyn_cast<BinaryOperator>(B->getOperand(0));
4177 }
4178 }
4179
4180 if (!B)
4181 return false;
4182
4183 Type *Ty = B->getType();
4184 if (!isValidElementType(Ty))
4185 return false;
4186
4187 const DataLayout &DL = B->getModule()->getDataLayout();
4188 ReductionOpcode = B->getOpcode();
4189 ReducedValueOpcode = 0;
4190 // FIXME: Register size should be a parameter to this function, so we can
4191 // try different vectorization factors.
4192 ReduxWidth = MinVecRegSize / DL.getTypeSizeInBits(Ty);
4193 ReductionRoot = B;
4194 ReductionPHI = Phi;
4195
4196 if (ReduxWidth < 4)
4197 return false;
4198
4199 // We currently only support adds.
4200 if (ReductionOpcode != Instruction::Add &&
4201 ReductionOpcode != Instruction::FAdd)
4202 return false;
4203
4204 // Post order traverse the reduction tree starting at B. We only handle true
4205 // trees containing only binary operators or selects.
4206 SmallVector<std::pair<Instruction *, unsigned>, 32> Stack;
4207 Stack.push_back(std::make_pair(B, 0));
4208 while (!Stack.empty()) {
4209 Instruction *TreeN = Stack.back().first;
4210 unsigned EdgeToVist = Stack.back().second++;
4211 bool IsReducedValue = TreeN->getOpcode() != ReductionOpcode;
4212
4213 // Only handle trees in the current basic block.
4214 if (TreeN->getParent() != B->getParent())
4215 return false;
4216
4217 // Each tree node needs to have one user except for the ultimate
4218 // reduction.
4219 if (!TreeN->hasOneUse() && TreeN != B)
4220 return false;
4221
4222 // Postorder vist.
4223 if (EdgeToVist == 2 || IsReducedValue) {
4224 if (IsReducedValue) {
4225 // Make sure that the opcodes of the operations that we are going to
4226 // reduce match.
4227 if (!ReducedValueOpcode)
4228 ReducedValueOpcode = TreeN->getOpcode();
4229 else if (ReducedValueOpcode != TreeN->getOpcode())
4230 return false;
4231 ReducedVals.push_back(TreeN);
4232 } else {
4233 // We need to be able to reassociate the adds.
4234 if (!TreeN->isAssociative())
4235 return false;
4236 ReductionOps.push_back(TreeN);
4237 }
4238 // Retract.
4239 Stack.pop_back();
4240 continue;
4241 }
4242
4243 // Visit left or right.
4244 Value *NextV = TreeN->getOperand(EdgeToVist);
4245 if (NextV != Phi) {
4246 auto *I = dyn_cast<Instruction>(NextV);
4247 // Continue analysis if the next operand is a reduction operation or
4248 // (possibly) a reduced value. If the reduced value opcode is not set,
4249 // the first met operation != reduction operation is considered as the
4250 // reduced value class.
4251 if (I && (!ReducedValueOpcode || I->getOpcode() == ReducedValueOpcode ||
4252 I->getOpcode() == ReductionOpcode)) {
4253 if (!ReducedValueOpcode && I->getOpcode() != ReductionOpcode)
4254 ReducedValueOpcode = I->getOpcode();
4255 Stack.push_back(std::make_pair(I, 0));
4256 continue;
4257 }
4258 return false;
4259 }
4260 }
4261 return true;
4262 }
4263
4264 /// \brief Attempt to vectorize the tree found by
4265 /// matchAssociativeReduction.
4266 bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
4267 if (ReducedVals.empty())
4268 return false;
4269
4270 unsigned NumReducedVals = ReducedVals.size();
4271 if (NumReducedVals < ReduxWidth)
4272 return false;
4273
4274 Value *VectorizedTree = nullptr;
4275 IRBuilder<> Builder(ReductionRoot);
4276 FastMathFlags Unsafe;
4277 Unsafe.setUnsafeAlgebra();
4278 Builder.setFastMathFlags(Unsafe);
4279 unsigned i = 0;
4280
4281 for (; i < NumReducedVals - ReduxWidth + 1; i += ReduxWidth) {
4282 auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
4283 V.buildTree(VL, ReductionOps);
4284 if (V.shouldReorder()) {
4285 SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
4286 V.buildTree(Reversed, ReductionOps);
4287 }
4288 if (V.isTreeTinyAndNotFullyVectorizable())
4289 continue;
4290
4291 V.computeMinimumValueSizes();
4292
4293 // Estimate cost.
4294 int Cost = V.getTreeCost() + getReductionCost(TTI, ReducedVals[i]);
4295 if (Cost >= -SLPCostThreshold)
4296 break;
4297
4298 DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
4299 << ". (HorRdx)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
<< Cost << ". (HorRdx)\n"; } } while (false)
;
4300
4301 // Vectorize a tree.
4302 DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
4303 Value *VectorizedRoot = V.vectorizeTree();
4304
4305 // Emit a reduction.
4306 Value *ReducedSubTree = emitReduction(VectorizedRoot, Builder);
4307 if (VectorizedTree) {
4308 Builder.SetCurrentDebugLocation(Loc);
4309 VectorizedTree = createBinOp(Builder, ReductionOpcode, VectorizedTree,
4310 ReducedSubTree, "bin.rdx");
4311 } else
4312 VectorizedTree = ReducedSubTree;
4313 }
4314
4315 if (VectorizedTree) {
4316 // Finish the reduction.
4317 for (; i < NumReducedVals; ++i) {
4318 Builder.SetCurrentDebugLocation(
4319 cast<Instruction>(ReducedVals[i])->getDebugLoc());
4320 VectorizedTree = createBinOp(Builder, ReductionOpcode, VectorizedTree,
4321 ReducedVals[i]);
4322 }
4323 // Update users.
4324 if (ReductionPHI && !isa<UndefValue>(ReductionPHI)) {
4325 assert(ReductionRoot && "Need a reduction operation")((ReductionRoot && "Need a reduction operation") ? static_cast
<void> (0) : __assert_fail ("ReductionRoot && \"Need a reduction operation\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4325, __PRETTY_FUNCTION__))
;
4326 ReductionRoot->setOperand(0, VectorizedTree);
4327 ReductionRoot->setOperand(1, ReductionPHI);
4328 } else
4329 ReductionRoot->replaceAllUsesWith(VectorizedTree);
4330 }
4331 return VectorizedTree != nullptr;
4332 }
4333
4334 unsigned numReductionValues() const {
4335 return ReducedVals.size();
4336 }
4337
4338private:
4339 /// \brief Calculate the cost of a reduction.
4340 int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal) {
4341 Type *ScalarTy = FirstReducedVal->getType();
4342 Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
4343
4344 int PairwiseRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, true);
4345 int SplittingRdxCost = TTI->getReductionCost(ReductionOpcode, VecTy, false);
4346
4347 IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
4348 int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
4349
4350 int ScalarReduxCost =
4351 (ReduxWidth - 1) *
4352 TTI->getArithmeticInstrCost(ReductionOpcode, ScalarTy);
4353
4354 DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4355 << " for reduction that starts with " << *FirstReducedValdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4356 << " (It is a "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4357 << (IsPairwiseReduction ? "pairwise" : "splitting")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
4358 << " reduction)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Adding cost " << VecReduxCost
- ScalarReduxCost << " for reduction that starts with "
<< *FirstReducedVal << " (It is a " << (IsPairwiseReduction
? "pairwise" : "splitting") << " reduction)\n"; } } while
(false)
;
4359
4360 return VecReduxCost - ScalarReduxCost;
4361 }
4362
4363 static Value *createBinOp(IRBuilder<> &Builder, unsigned Opcode, Value *L,
4364 Value *R, const Twine &Name = "") {
4365 if (Opcode == Instruction::FAdd)
4366 return Builder.CreateFAdd(L, R, Name);
4367 return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, L, R, Name);
4368 }
4369
4370 /// \brief Emit a horizontal reduction of the vectorized value.
4371 Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder) {
4372 assert(VectorizedValue && "Need to have a vectorized tree node")((VectorizedValue && "Need to have a vectorized tree node"
) ? static_cast<void> (0) : __assert_fail ("VectorizedValue && \"Need to have a vectorized tree node\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4372, __PRETTY_FUNCTION__))
;
4373 assert(isPowerOf2_32(ReduxWidth) &&((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4374, __PRETTY_FUNCTION__))
4374 "We only handle power-of-two reductions for now")((isPowerOf2_32(ReduxWidth) && "We only handle power-of-two reductions for now"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ReduxWidth) && \"We only handle power-of-two reductions for now\""
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4374, __PRETTY_FUNCTION__))
;
4375
4376 Value *TmpVec = VectorizedValue;
4377 for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
4378 if (IsPairwiseReduction) {
4379 Value *LeftMask =
4380 createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
4381 Value *RightMask =
4382 createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
4383
4384 Value *LeftShuf = Builder.CreateShuffleVector(
4385 TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
4386 Value *RightShuf = Builder.CreateShuffleVector(
4387 TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
4388 "rdx.shuf.r");
4389 TmpVec = createBinOp(Builder, ReductionOpcode, LeftShuf, RightShuf,
4390 "bin.rdx");
4391 } else {
4392 Value *UpperHalf =
4393 createRdxShuffleMask(ReduxWidth, i, false, false, Builder);
4394 Value *Shuf = Builder.CreateShuffleVector(
4395 TmpVec, UndefValue::get(TmpVec->getType()), UpperHalf, "rdx.shuf");
4396 TmpVec = createBinOp(Builder, ReductionOpcode, TmpVec, Shuf, "bin.rdx");
4397 }
4398 }
4399
4400 // The result is in the first element of the vector.
4401 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
4402 }
4403};
4404} // end anonymous namespace
4405
4406/// \brief Recognize construction of vectors like
4407/// %ra = insertelement <4 x float> undef, float %s0, i32 0
4408/// %rb = insertelement <4 x float> %ra, float %s1, i32 1
4409/// %rc = insertelement <4 x float> %rb, float %s2, i32 2
4410/// %rd = insertelement <4 x float> %rc, float %s3, i32 3
4411///
4412/// Returns true if it matches
4413///
4414static bool findBuildVector(InsertElementInst *FirstInsertElem,
4415 SmallVectorImpl<Value *> &BuildVector,
4416 SmallVectorImpl<Value *> &BuildVectorOpds) {
4417 if (!isa<UndefValue>(FirstInsertElem->getOperand(0)))
4418 return false;
4419
4420 InsertElementInst *IE = FirstInsertElem;
4421 while (true) {
4422 BuildVector.push_back(IE);
4423 BuildVectorOpds.push_back(IE->getOperand(1));
4424
4425 if (IE->use_empty())
4426 return false;
4427
4428 InsertElementInst *NextUse = dyn_cast<InsertElementInst>(IE->user_back());
4429 if (!NextUse)
4430 return true;
4431
4432 // If this isn't the final use, make sure the next insertelement is the only
4433 // use. It's OK if the final constructed vector is used multiple times
4434 if (!IE->hasOneUse())
4435 return false;
4436
4437 IE = NextUse;
4438 }
4439
4440 return false;
4441}
4442
4443/// \brief Like findBuildVector, but looks backwards for construction of aggregate.
4444///
4445/// \return true if it matches.
4446static bool findBuildAggregate(InsertValueInst *IV,
4447 SmallVectorImpl<Value *> &BuildVector,
4448 SmallVectorImpl<Value *> &BuildVectorOpds) {
4449 if (!IV->hasOneUse())
4450 return false;
4451 Value *V = IV->getAggregateOperand();
4452 if (!isa<UndefValue>(V)) {
4453 InsertValueInst *I = dyn_cast<InsertValueInst>(V);
4454 if (!I || !findBuildAggregate(I, BuildVector, BuildVectorOpds))
4455 return false;
4456 }
4457 BuildVector.push_back(IV);
4458 BuildVectorOpds.push_back(IV->getInsertedValueOperand());
4459 return true;
4460}
4461
4462static bool PhiTypeSorterFunc(Value *V, Value *V2) {
4463 return V->getType() < V2->getType();
4464}
4465
4466/// \brief Try and get a reduction value from a phi node.
4467///
4468/// Given a phi node \p P in a block \p ParentBB, consider possible reductions
4469/// if they come from either \p ParentBB or a containing loop latch.
4470///
4471/// \returns A candidate reduction value if possible, or \code nullptr \endcode
4472/// if not possible.
4473static Value *getReductionValue(const DominatorTree *DT, PHINode *P,
4474 BasicBlock *ParentBB, LoopInfo *LI) {
4475 // There are situations where the reduction value is not dominated by the
4476 // reduction phi. Vectorizing such cases has been reported to cause
4477 // miscompiles. See PR25787.
4478 auto DominatedReduxValue = [&](Value *R) {
4479 return (
4480 dyn_cast<Instruction>(R) &&
4481 DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
4482 };
4483
4484 Value *Rdx = nullptr;
4485
4486 // Return the incoming value if it comes from the same BB as the phi node.
4487 if (P->getIncomingBlock(0) == ParentBB) {
4488 Rdx = P->getIncomingValue(0);
4489 } else if (P->getIncomingBlock(1) == ParentBB) {
4490 Rdx = P->getIncomingValue(1);
4491 }
4492
4493 if (Rdx && DominatedReduxValue(Rdx))
4494 return Rdx;
4495
4496 // Otherwise, check whether we have a loop latch to look at.
4497 Loop *BBL = LI->getLoopFor(ParentBB);
4498 if (!BBL)
4499 return nullptr;
4500 BasicBlock *BBLatch = BBL->getLoopLatch();
4501 if (!BBLatch)
4502 return nullptr;
4503
4504 // There is a loop latch, return the incoming value if it comes from
4505 // that. This reduction pattern occasionally turns up.
4506 if (P->getIncomingBlock(0) == BBLatch) {
4507 Rdx = P->getIncomingValue(0);
4508 } else if (P->getIncomingBlock(1) == BBLatch) {
4509 Rdx = P->getIncomingValue(1);
4510 }
4511
4512 if (Rdx && DominatedReduxValue(Rdx))
4513 return Rdx;
4514
4515 return nullptr;
4516}
4517
4518namespace {
4519/// Tracks instructons and its children.
4520class WeakVHWithLevel final : public CallbackVH {
4521 /// Operand index of the instruction currently beeing analized.
4522 unsigned Level = 0;
4523 /// Is this the instruction that should be vectorized, or are we now
4524 /// processing children (i.e. operands of this instruction) for potential
4525 /// vectorization?
4526 bool IsInitial = true;
4527
4528public:
4529 explicit WeakVHWithLevel() = default;
4530 WeakVHWithLevel(Value *V) : CallbackVH(V){};
4531 /// Restart children analysis each time it is repaced by the new instruction.
4532 void allUsesReplacedWith(Value *New) override {
4533 setValPtr(New);
4534 Level = 0;
4535 IsInitial = true;
4536 }
4537 /// Check if the instruction was not deleted during vectorization.
4538 bool isValid() const { return !getValPtr(); }
4539 /// Is the istruction itself must be vectorized?
4540 bool isInitial() const { return IsInitial; }
4541 /// Try to vectorize children.
4542 void clearInitial() { IsInitial = false; }
4543 /// Are all children processed already?
4544 bool isFinal() const {
4545 assert(getValPtr() &&((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4547, __PRETTY_FUNCTION__))
4546 (isa<Instruction>(getValPtr()) &&((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4547, __PRETTY_FUNCTION__))
4547 cast<Instruction>(getValPtr())->getNumOperands() >= Level))((getValPtr() && (isa<Instruction>(getValPtr())
&& cast<Instruction>(getValPtr())->getNumOperands
() >= Level)) ? static_cast<void> (0) : __assert_fail
("getValPtr() && (isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() >= Level)"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4547, __PRETTY_FUNCTION__))
;
4548 return getValPtr() &&
4549 cast<Instruction>(getValPtr())->getNumOperands() == Level;
4550 }
4551 /// Get next child operation.
4552 Value *nextOperand() {
4553 assert(getValPtr() && isa<Instruction>(getValPtr()) &&((getValPtr() && isa<Instruction>(getValPtr()) &&
cast<Instruction>(getValPtr())->getNumOperands() >
Level) ? static_cast<void> (0) : __assert_fail ("getValPtr() && isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() > Level"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4554, __PRETTY_FUNCTION__))
4554 cast<Instruction>(getValPtr())->getNumOperands() > Level)((getValPtr() && isa<Instruction>(getValPtr()) &&
cast<Instruction>(getValPtr())->getNumOperands() >
Level) ? static_cast<void> (0) : __assert_fail ("getValPtr() && isa<Instruction>(getValPtr()) && cast<Instruction>(getValPtr())->getNumOperands() > Level"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4554, __PRETTY_FUNCTION__))
;
4555 return cast<Instruction>(getValPtr())->getOperand(Level++);
4556 }
4557 virtual ~WeakVHWithLevel() = default;
4558};
4559} // namespace
4560
4561/// \brief Attempt to reduce a horizontal reduction.
4562/// If it is legal to match a horizontal reduction feeding
4563/// the phi node P with reduction operators Root in a basic block BB, then check
4564/// if it can be done.
4565/// \returns true if a horizontal reduction was matched and reduced.
4566/// \returns false if a horizontal reduction was not matched.
4567static bool canBeVectorized(
4568 PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
4569 TargetTransformInfo *TTI,
4570 const function_ref<bool(BinaryOperator *, BoUpSLP &)> Vectorize) {
4571 if (!ShouldVectorizeHor)
4572 return false;
4573
4574 if (!Root)
4575 return false;
4576
4577 if (Root->getParent() != BB)
4578 return false;
4579 SmallVector<WeakVHWithLevel, 8> Stack(1, Root);
4580 SmallSet<Value *, 8> VisitedInstrs;
4581 bool Res = false;
4582 while (!Stack.empty()) {
4583 Value *V = Stack.back();
4584 if (!V) {
4585 Stack.pop_back();
4586 continue;
4587 }
4588 auto *Inst = dyn_cast<Instruction>(V);
4589 if (!Inst || isa<PHINode>(Inst)) {
4590 Stack.pop_back();
4591 continue;
4592 }
4593 if (Stack.back().isInitial()) {
4594 Stack.back().clearInitial();
4595 if (auto *BI = dyn_cast<BinaryOperator>(Inst)) {
4596 HorizontalReduction HorRdx(R.getMinVecRegSize());
4597 if (HorRdx.matchAssociativeReduction(P, BI)) {
4598 // If there is a sufficient number of reduction values, reduce
4599 // to a nearby power-of-2. Can safely generate oversized
4600 // vectors and rely on the backend to split them to legal sizes.
4601 HorRdx.ReduxWidth =
4602 std::max((uint64_t)4, PowerOf2Floor(HorRdx.numReductionValues()));
4603
4604 if (HorRdx.tryToReduce(R, TTI)) {
4605 Res = true;
4606 P = nullptr;
4607 continue;
4608 }
4609 }
4610 if (P) {
4611 Inst = dyn_cast<Instruction>(BI->getOperand(0));
4612 if (Inst == P)
4613 Inst = dyn_cast<Instruction>(BI->getOperand(1));
4614 if (!Inst) {
4615 P = nullptr;
4616 continue;
4617 }
4618 }
4619 }
4620 P = nullptr;
4621 if (Vectorize(dyn_cast<BinaryOperator>(Inst), R)) {
4622 Res = true;
4623 continue;
4624 }
4625 }
4626 if (Stack.back().isFinal()) {
4627 Stack.pop_back();
4628 continue;
4629 }
4630
4631 if (auto *NextV = dyn_cast<Instruction>(Stack.back().nextOperand()))
4632 if (NextV->getParent() == BB && VisitedInstrs.insert(NextV).second &&
4633 Stack.size() < RecursionMaxDepth)
4634 Stack.push_back(NextV);
4635 }
4636 return Res;
4637}
4638
4639bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
4640 BasicBlock *BB, BoUpSLP &R,
4641 TargetTransformInfo *TTI) {
4642 if (!V)
4643 return false;
4644 auto *I = dyn_cast<Instruction>(V);
4645 if (!I)
4646 return false;
4647
4648 if (!isa<BinaryOperator>(I))
4649 P = nullptr;
4650 // Try to match and vectorize a horizontal reduction.
4651 return canBeVectorized(P, I, BB, R, TTI,
4652 [this](BinaryOperator *BI, BoUpSLP &R) -> bool {
4653 return tryToVectorize(BI, R);
4654 });
4655}
4656
4657bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
4658 bool Changed = false;
4659 SmallVector<Value *, 4> Incoming;
4660 SmallSet<Value *, 16> VisitedInstrs;
4661
4662 bool HaveVectorizedPhiNodes = true;
4663 while (HaveVectorizedPhiNodes) {
4664 HaveVectorizedPhiNodes = false;
4665
4666 // Collect the incoming values from the PHIs.
4667 Incoming.clear();
4668 for (Instruction &I : *BB) {
4669 PHINode *P = dyn_cast<PHINode>(&I);
4670 if (!P)
4671 break;
4672
4673 if (!VisitedInstrs.count(P))
4674 Incoming.push_back(P);
4675 }
4676
4677 // Sort by type.
4678 std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
4679
4680 // Try to vectorize elements base on their type.
4681 for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
4682 E = Incoming.end();
4683 IncIt != E;) {
4684
4685 // Look for the next elements with the same type.
4686 SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
4687 while (SameTypeIt != E &&
4688 (*SameTypeIt)->getType() == (*IncIt)->getType()) {
4689 VisitedInstrs.insert(*SameTypeIt);
4690 ++SameTypeIt;
4691 }
4692
4693 // Try to vectorize them.
4694 unsigned NumElts = (SameTypeIt - IncIt);
4695 DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { errs() << "SLP: Trying to vectorize starting at PHIs ("
<< NumElts << ")\n"; } } while (false)
;
4696 if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R)) {
4697 // Success start over because instructions might have been changed.
4698 HaveVectorizedPhiNodes = true;
4699 Changed = true;
4700 break;
4701 }
4702
4703 // Start over at the next instruction of a different type (or the end).
4704 IncIt = SameTypeIt;
4705 }
4706 }
4707
4708 VisitedInstrs.clear();
4709
4710 for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
4711 // We may go through BB multiple times so skip the one we have checked.
4712 if (!VisitedInstrs.insert(&*it).second)
4713 continue;
4714
4715 if (isa<DbgInfoIntrinsic>(it))
4716 continue;
4717
4718 // Try to vectorize reductions that use PHINodes.
4719 if (PHINode *P = dyn_cast<PHINode>(it)) {
4720 // Check that the PHI is a reduction PHI.
4721 if (P->getNumIncomingValues() != 2)
4722 return Changed;
4723
4724 // Try to match and vectorize a horizontal reduction.
4725 if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
4726 TTI)) {
4727 Changed = true;
4728 it = BB->begin();
4729 e = BB->end();
4730 continue;
4731 }
4732 continue;
4733 }
4734
4735 if (ShouldStartVectorizeHorAtStore) {
4736 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
4737 // Try to match and vectorize a horizontal reduction.
4738 if (vectorizeRootInstruction(nullptr, SI->getValueOperand(), BB, R,
4739 TTI)) {
4740 Changed = true;
4741 it = BB->begin();
4742 e = BB->end();
4743 continue;
4744 }
4745 }
4746 }
4747
4748 // Try to vectorize horizontal reductions feeding into a return.
4749 if (ReturnInst *RI = dyn_cast<ReturnInst>(it)) {
4750 if (RI->getNumOperands() != 0) {
4751 // Try to match and vectorize a horizontal reduction.
4752 if (vectorizeRootInstruction(nullptr, RI->getOperand(0), BB, R, TTI)) {
4753 Changed = true;
4754 it = BB->begin();
4755 e = BB->end();
4756 continue;
4757 }
4758 }
4759 }
4760
4761 // Try to vectorize trees that start at compare instructions.
4762 if (CmpInst *CI = dyn_cast<CmpInst>(it)) {
4763 if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R)) {
4764 Changed = true;
4765 // We would like to start over since some instructions are deleted
4766 // and the iterator may become invalid value.
4767 it = BB->begin();
4768 e = BB->end();
4769 continue;
4770 }
4771
4772 for (int I = 0; I < 2; ++I) {
4773 if (vectorizeRootInstruction(nullptr, CI->getOperand(I), BB, R, TTI)) {
4774 Changed = true;
4775 // We would like to start over since some instructions are deleted
4776 // and the iterator may become invalid value.
4777 it = BB->begin();
4778 e = BB->end();
4779 break;
4780 }
4781 }
4782 continue;
4783 }
4784
4785 // Try to vectorize trees that start at insertelement instructions.
4786 if (InsertElementInst *FirstInsertElem = dyn_cast<InsertElementInst>(it)) {
4787 SmallVector<Value *, 16> BuildVector;
4788 SmallVector<Value *, 16> BuildVectorOpds;
4789 if (!findBuildVector(FirstInsertElem, BuildVector, BuildVectorOpds))
4790 continue;
4791
4792 // Vectorize starting with the build vector operands ignoring the
4793 // BuildVector instructions for the purpose of scheduling and user
4794 // extraction.
4795 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector)) {
4796 Changed = true;
4797 it = BB->begin();
4798 e = BB->end();
4799 }
4800
4801 continue;
4802 }
4803
4804 // Try to vectorize trees that start at insertvalue instructions feeding into
4805 // a store.
4806 if (StoreInst *SI = dyn_cast<StoreInst>(it)) {
4807 if (InsertValueInst *LastInsertValue = dyn_cast<InsertValueInst>(SI->getValueOperand())) {
4808 const DataLayout &DL = BB->getModule()->getDataLayout();
4809 if (R.canMapToVector(SI->getValueOperand()->getType(), DL)) {
4810 SmallVector<Value *, 16> BuildVector;
4811 SmallVector<Value *, 16> BuildVectorOpds;
4812 if (!findBuildAggregate(LastInsertValue, BuildVector, BuildVectorOpds))
4813 continue;
4814
4815 DEBUG(dbgs() << "SLP: store of array mappable to vector: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: store of array mappable to vector: "
<< *SI << "\n"; } } while (false)
;
4816 if (tryToVectorizeList(BuildVectorOpds, R, BuildVector, false)) {
4817 Changed = true;
4818 it = BB->begin();
4819 e = BB->end();
4820 }
4821 continue;
4822 }
4823 }
4824 }
4825 }
4826
4827 return Changed;
4828}
4829
4830bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
4831 auto Changed = false;
4832 for (auto &Entry : GEPs) {
4833
4834 // If the getelementptr list has fewer than two elements, there's nothing
4835 // to do.
4836 if (Entry.second.size() < 2)
4837 continue;
4838
4839 DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
4840 << Entry.second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a getelementptr list of length "
<< Entry.second.size() << ".\n"; } } while (false
)
;
4841
4842 // We process the getelementptr list in chunks of 16 (like we do for
4843 // stores) to minimize compile-time.
4844 for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
4845 auto Len = std::min<unsigned>(BE - BI, 16);
4846 auto GEPList = makeArrayRef(&Entry.second[BI], Len);
4847
4848 // Initialize a set a candidate getelementptrs. Note that we use a
4849 // SetVector here to preserve program order. If the index computations
4850 // are vectorizable and begin with loads, we want to minimize the chance
4851 // of having to reorder them later.
4852 SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
4853
4854 // Some of the candidates may have already been vectorized after we
4855 // initially collected them. If so, the WeakVHs will have nullified the
4856 // values, so remove them from the set of candidates.
4857 Candidates.remove(nullptr);
4858
4859 // Remove from the set of candidates all pairs of getelementptrs with
4860 // constant differences. Such getelementptrs are likely not good
4861 // candidates for vectorization in a bottom-up phase since one can be
4862 // computed from the other. We also ensure all candidate getelementptr
4863 // indices are unique.
4864 for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
4865 auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
4866 if (!Candidates.count(GEPI))
4867 continue;
4868 auto *SCEVI = SE->getSCEV(GEPList[I]);
4869 for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
4870 auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
4871 auto *SCEVJ = SE->getSCEV(GEPList[J]);
4872 if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
4873 Candidates.remove(GEPList[I]);
4874 Candidates.remove(GEPList[J]);
4875 } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
4876 Candidates.remove(GEPList[J]);
4877 }
4878 }
4879 }
4880
4881 // We break out of the above computation as soon as we know there are
4882 // fewer than two candidates remaining.
4883 if (Candidates.size() < 2)
4884 continue;
4885
4886 // Add the single, non-constant index of each candidate to the bundle. We
4887 // ensured the indices met these constraints when we originally collected
4888 // the getelementptrs.
4889 SmallVector<Value *, 16> Bundle(Candidates.size());
4890 auto BundleIndex = 0u;
4891 for (auto *V : Candidates) {
4892 auto *GEP = cast<GetElementPtrInst>(V);
4893 auto *GEPIdx = GEP->idx_begin()->get();
4894 assert(GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx))((GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx
)) ? static_cast<void> (0) : __assert_fail ("GEP->getNumIndices() == 1 || !isa<Constant>(GEPIdx)"
, "/tmp/buildd/llvm-toolchain-snapshot-4.0~svn290870/lib/Transforms/Vectorize/SLPVectorizer.cpp"
, 4894, __PRETTY_FUNCTION__))
;
4895 Bundle[BundleIndex++] = GEPIdx;
4896 }
4897
4898 // Try and vectorize the indices. We are currently only interested in
4899 // gather-like cases of the form:
4900 //
4901 // ... = g[a[0] - b[0]] + g[a[1] - b[1]] + ...
4902 //
4903 // where the loads of "a", the loads of "b", and the subtractions can be
4904 // performed in parallel. It's likely that detecting this pattern in a
4905 // bottom-up phase will be simpler and less costly than building a
4906 // full-blown top-down phase beginning at the consecutive loads.
4907 Changed |= tryToVectorizeList(Bundle, R);
4908 }
4909 }
4910 return Changed;
4911}
4912
4913bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
4914 bool Changed = false;
4915 // Attempt to sort and vectorize each of the store-groups.
4916 for (StoreListMap::iterator it = Stores.begin(), e = Stores.end(); it != e;
4917 ++it) {
4918 if (it->second.size() < 2)
4919 continue;
4920
4921 DEBUG(dbgs() << "SLP: Analyzing a store chain of length "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
4922 << it->second.size() << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("SLP")) { dbgs() << "SLP: Analyzing a store chain of length "
<< it->second.size() << ".\n"; } } while (false
)
;
4923
4924 // Process the stores in chunks of 16.
4925 // TODO: The limit of 16 inhibits greater vectorization factors.
4926 // For example, AVX2 supports v32i8. Increasing this limit, however,
4927 // may cause a significant compile-time increase.
4928 for (unsigned CI = 0, CE = it->second.size(); CI < CE; CI+=16) {
4929 unsigned Len = std::min<unsigned>(CE - CI, 16);
4930 Changed |= vectorizeStores(makeArrayRef(&it->second[CI], Len), R);
4931 }
4932 }
4933 return Changed;
4934}
4935
4936char SLPVectorizer::ID = 0;
4937static const char lv_name[] = "SLP Vectorizer";
4938INITIALIZE_PASS_BEGIN(SLPVectorizer, SV_NAME, lv_name, false, false)static void *initializeSLPVectorizerPassOnce(PassRegistry &
Registry) {
4939INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
4940INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
4941INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry);
4942INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)initializeScalarEvolutionWrapperPassPass(Registry);
4943INITIALIZE_PASS_DEPENDENCY(LoopSimplify)initializeLoopSimplifyPass(Registry);
4944INITIALIZE_PASS_DEPENDENCY(DemandedBitsWrapperPass)initializeDemandedBitsWrapperPassPass(Registry);
4945INITIALIZE_PASS_END(SLPVectorizer, SV_NAME, lv_name, false, false)PassInfo *PI = new PassInfo( lv_name, "slp-vectorizer", &
SLPVectorizer::ID, PassInfo::NormalCtor_t(callDefaultCtor<
SLPVectorizer>), false, false); Registry.registerPass(*PI,
true); return PI; } static once_flag InitializeSLPVectorizerPassFlag
; void llvm::initializeSLPVectorizerPass(PassRegistry &Registry
) { llvm::call_once(InitializeSLPVectorizerPassFlag, initializeSLPVectorizerPassOnce
, std::ref(Registry)); }
4946
4947namespace llvm {
4948Pass *createSLPVectorizerPass() { return new SLPVectorizer(); }
4949}