LLVM  9.0.0svn
SLPVectorizer.cpp
Go to the documentation of this file.
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10 // stores that can be put together into vector-stores. Next, it attempts to
11 // construct vectorizable tree using the use-def chains. If a profitable tree
12 // was found, the SLP vectorizer performs vectorization on the tree.
13 //
14 // The pass is inspired by the work described in the paper:
15 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16 //
17 //===----------------------------------------------------------------------===//
18 
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/DenseSet.h"
23 #include "llvm/ADT/MapVector.h"
24 #include "llvm/ADT/None.h"
25 #include "llvm/ADT/Optional.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/SetVector.h"
29 #include "llvm/ADT/SmallPtrSet.h"
30 #include "llvm/ADT/SmallSet.h"
31 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/ADT/Statistic.h"
33 #include "llvm/ADT/iterator.h"
40 #include "llvm/Analysis/LoopInfo.h"
49 #include "llvm/IR/Attributes.h"
50 #include "llvm/IR/BasicBlock.h"
51 #include "llvm/IR/Constant.h"
52 #include "llvm/IR/Constants.h"
53 #include "llvm/IR/DataLayout.h"
54 #include "llvm/IR/DebugLoc.h"
55 #include "llvm/IR/DerivedTypes.h"
56 #include "llvm/IR/Dominators.h"
57 #include "llvm/IR/Function.h"
58 #include "llvm/IR/IRBuilder.h"
59 #include "llvm/IR/InstrTypes.h"
60 #include "llvm/IR/Instruction.h"
61 #include "llvm/IR/Instructions.h"
62 #include "llvm/IR/IntrinsicInst.h"
63 #include "llvm/IR/Intrinsics.h"
64 #include "llvm/IR/Module.h"
65 #include "llvm/IR/NoFolder.h"
66 #include "llvm/IR/Operator.h"
67 #include "llvm/IR/PassManager.h"
68 #include "llvm/IR/PatternMatch.h"
69 #include "llvm/IR/Type.h"
70 #include "llvm/IR/Use.h"
71 #include "llvm/IR/User.h"
72 #include "llvm/IR/Value.h"
73 #include "llvm/IR/ValueHandle.h"
74 #include "llvm/IR/Verifier.h"
75 #include "llvm/Pass.h"
76 #include "llvm/Support/Casting.h"
78 #include "llvm/Support/Compiler.h"
80 #include "llvm/Support/Debug.h"
83 #include "llvm/Support/KnownBits.h"
88 #include <algorithm>
89 #include <cassert>
90 #include <cstdint>
91 #include <iterator>
92 #include <memory>
93 #include <set>
94 #include <string>
95 #include <tuple>
96 #include <utility>
97 #include <vector>
98 
99 using namespace llvm;
100 using namespace llvm::PatternMatch;
101 using namespace slpvectorizer;
102 
103 #define SV_NAME "slp-vectorizer"
104 #define DEBUG_TYPE "SLP"
105 
106 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
107 
108 static cl::opt<int>
109  SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
110  cl::desc("Only vectorize if you gain more than this "
111  "number "));
112 
113 static cl::opt<bool>
114 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
115  cl::desc("Attempt to vectorize horizontal reductions"));
116 
118  "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
119  cl::desc(
120  "Attempt to vectorize horizontal reductions feeding into a store"));
121 
122 static cl::opt<int>
123 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
124  cl::desc("Attempt to vectorize for this register size in bits"));
125 
126 /// Limits the size of scheduling regions in a block.
127 /// It avoid long compile times for _very_ large blocks where vector
128 /// instructions are spread over a wide range.
129 /// This limit is way higher than needed by real-world functions.
130 static cl::opt<int>
131 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
132  cl::desc("Limit the size of the SLP scheduling region per block"));
133 
135  "slp-min-reg-size", cl::init(128), cl::Hidden,
136  cl::desc("Attempt to vectorize for this register size in bits"));
137 
139  "slp-recursion-max-depth", cl::init(12), cl::Hidden,
140  cl::desc("Limit the recursion depth when building a vectorizable tree"));
141 
143  "slp-min-tree-size", cl::init(3), cl::Hidden,
144  cl::desc("Only vectorize small trees if they are fully vectorizable"));
145 
146 static cl::opt<bool>
147  ViewSLPTree("view-slp-tree", cl::Hidden,
148  cl::desc("Display the SLP trees with Graphviz"));
149 
150 // Limit the number of alias checks. The limit is chosen so that
151 // it has no negative effect on the llvm benchmarks.
152 static const unsigned AliasedCheckLimit = 10;
153 
154 // Another limit for the alias checks: The maximum distance between load/store
155 // instructions where alias checks are done.
156 // This limit is useful for very large basic blocks.
157 static const unsigned MaxMemDepDistance = 160;
158 
159 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
160 /// regions to be handled.
161 static const int MinScheduleRegionSize = 16;
162 
163 /// Predicate for the element types that the SLP vectorizer supports.
164 ///
165 /// The most important thing to filter here are types which are invalid in LLVM
166 /// vectors. We also filter target specific types which have absolutely no
167 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
168 /// avoids spending time checking the cost model and realizing that they will
169 /// be inevitably scalarized.
170 static bool isValidElementType(Type *Ty) {
171  return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
172  !Ty->isPPC_FP128Ty();
173 }
174 
175 /// \returns true if all of the instructions in \p VL are in the same block or
176 /// false otherwise.
178  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
179  if (!I0)
180  return false;
181  BasicBlock *BB = I0->getParent();
182  for (int i = 1, e = VL.size(); i < e; i++) {
183  Instruction *I = dyn_cast<Instruction>(VL[i]);
184  if (!I)
185  return false;
186 
187  if (BB != I->getParent())
188  return false;
189  }
190  return true;
191 }
192 
193 /// \returns True if all of the values in \p VL are constants.
195  for (Value *i : VL)
196  if (!isa<Constant>(i))
197  return false;
198  return true;
199 }
200 
201 /// \returns True if all of the values in \p VL are identical.
202 static bool isSplat(ArrayRef<Value *> VL) {
203  for (unsigned i = 1, e = VL.size(); i < e; ++i)
204  if (VL[i] != VL[0])
205  return false;
206  return true;
207 }
208 
209 /// Checks if the vector of instructions can be represented as a shuffle, like:
210 /// %x0 = extractelement <4 x i8> %x, i32 0
211 /// %x3 = extractelement <4 x i8> %x, i32 3
212 /// %y1 = extractelement <4 x i8> %y, i32 1
213 /// %y2 = extractelement <4 x i8> %y, i32 2
214 /// %x0x0 = mul i8 %x0, %x0
215 /// %x3x3 = mul i8 %x3, %x3
216 /// %y1y1 = mul i8 %y1, %y1
217 /// %y2y2 = mul i8 %y2, %y2
218 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
219 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
220 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
221 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
222 /// ret <4 x i8> %ins4
223 /// can be transformed into:
224 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
225 /// i32 6>
226 /// %2 = mul <4 x i8> %1, %1
227 /// ret <4 x i8> %2
228 /// We convert this initially to something like:
229 /// %x0 = extractelement <4 x i8> %x, i32 0
230 /// %x3 = extractelement <4 x i8> %x, i32 3
231 /// %y1 = extractelement <4 x i8> %y, i32 1
232 /// %y2 = extractelement <4 x i8> %y, i32 2
233 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
234 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
235 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
236 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
237 /// %5 = mul <4 x i8> %4, %4
238 /// %6 = extractelement <4 x i8> %5, i32 0
239 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
240 /// %7 = extractelement <4 x i8> %5, i32 1
241 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
242 /// %8 = extractelement <4 x i8> %5, i32 2
243 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
244 /// %9 = extractelement <4 x i8> %5, i32 3
245 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
246 /// ret <4 x i8> %ins4
247 /// InstCombiner transforms this into a shuffle and vector mul
248 /// TODO: Can we split off and reuse the shuffle mask detection from
249 /// TargetTransformInfo::getInstructionThroughput?
252  auto *EI0 = cast<ExtractElementInst>(VL[0]);
253  unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
254  Value *Vec1 = nullptr;
255  Value *Vec2 = nullptr;
256  enum ShuffleMode { Unknown, Select, Permute };
257  ShuffleMode CommonShuffleMode = Unknown;
258  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
259  auto *EI = cast<ExtractElementInst>(VL[I]);
260  auto *Vec = EI->getVectorOperand();
261  // All vector operands must have the same number of vector elements.
262  if (Vec->getType()->getVectorNumElements() != Size)
263  return None;
264  auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
265  if (!Idx)
266  return None;
267  // Undefined behavior if Idx is negative or >= Size.
268  if (Idx->getValue().uge(Size))
269  continue;
270  unsigned IntIdx = Idx->getValue().getZExtValue();
271  // We can extractelement from undef vector.
272  if (isa<UndefValue>(Vec))
273  continue;
274  // For correct shuffling we have to have at most 2 different vector operands
275  // in all extractelement instructions.
276  if (!Vec1 || Vec1 == Vec)
277  Vec1 = Vec;
278  else if (!Vec2 || Vec2 == Vec)
279  Vec2 = Vec;
280  else
281  return None;
282  if (CommonShuffleMode == Permute)
283  continue;
284  // If the extract index is not the same as the operation number, it is a
285  // permutation.
286  if (IntIdx != I) {
287  CommonShuffleMode = Permute;
288  continue;
289  }
290  CommonShuffleMode = Select;
291  }
292  // If we're not crossing lanes in different vectors, consider it as blending.
293  if (CommonShuffleMode == Select && Vec2)
295  // If Vec2 was never used, we have a permutation of a single vector, otherwise
296  // we have permutation of 2 vectors.
299 }
300 
301 namespace {
302 
303 /// Main data required for vectorization of instructions.
304 struct InstructionsState {
305  /// The very first instruction in the list with the main opcode.
306  Value *OpValue = nullptr;
307 
308  /// The main/alternate instruction.
309  Instruction *MainOp = nullptr;
310  Instruction *AltOp = nullptr;
311 
312  /// The main/alternate opcodes for the list of instructions.
313  unsigned getOpcode() const {
314  return MainOp ? MainOp->getOpcode() : 0;
315  }
316 
317  unsigned getAltOpcode() const {
318  return AltOp ? AltOp->getOpcode() : 0;
319  }
320 
321  /// Some of the instructions in the list have alternate opcodes.
322  bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
323 
324  bool isOpcodeOrAlt(Instruction *I) const {
325  unsigned CheckedOpcode = I->getOpcode();
326  return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
327  }
328 
329  InstructionsState() = delete;
330  InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
331  : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
332 };
333 
334 } // end anonymous namespace
335 
336 /// Chooses the correct key for scheduling data. If \p Op has the same (or
337 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
338 /// OpValue.
339 static Value *isOneOf(const InstructionsState &S, Value *Op) {
340  auto *I = dyn_cast<Instruction>(Op);
341  if (I && S.isOpcodeOrAlt(I))
342  return Op;
343  return S.OpValue;
344 }
345 
346 /// \returns analysis of the Instructions in \p VL described in
347 /// InstructionsState, the Opcode that we suppose the whole list
348 /// could be vectorized even if its structure is diverse.
349 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
350  unsigned BaseIndex = 0) {
351  // Make sure these are all Instructions.
352  if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
353  return InstructionsState(VL[BaseIndex], nullptr, nullptr);
354 
355  bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
356  bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
357  unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
358  unsigned AltOpcode = Opcode;
359  unsigned AltIndex = BaseIndex;
360 
361  // Check for one alternate opcode from another BinaryOperator.
362  // TODO - generalize to support all operators (types, calls etc.).
363  for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
364  unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
365  if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
366  if (InstOpcode == Opcode || InstOpcode == AltOpcode)
367  continue;
368  if (Opcode == AltOpcode) {
369  AltOpcode = InstOpcode;
370  AltIndex = Cnt;
371  continue;
372  }
373  } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
374  Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
375  Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
376  if (Ty0 == Ty1) {
377  if (InstOpcode == Opcode || InstOpcode == AltOpcode)
378  continue;
379  if (Opcode == AltOpcode) {
380  AltOpcode = InstOpcode;
381  AltIndex = Cnt;
382  continue;
383  }
384  }
385  } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
386  continue;
387  return InstructionsState(VL[BaseIndex], nullptr, nullptr);
388  }
389 
390  return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
391  cast<Instruction>(VL[AltIndex]));
392 }
393 
394 /// \returns true if all of the values in \p VL have the same type or false
395 /// otherwise.
397  Type *Ty = VL[0]->getType();
398  for (int i = 1, e = VL.size(); i < e; i++)
399  if (VL[i]->getType() != Ty)
400  return false;
401 
402  return true;
403 }
404 
405 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
407  unsigned Opcode = E->getOpcode();
408  assert((Opcode == Instruction::ExtractElement ||
409  Opcode == Instruction::ExtractValue) &&
410  "Expected extractelement or extractvalue instruction.");
411  if (Opcode == Instruction::ExtractElement) {
412  auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
413  if (!CI)
414  return None;
415  return CI->getZExtValue();
416  }
417  ExtractValueInst *EI = cast<ExtractValueInst>(E);
418  if (EI->getNumIndices() != 1)
419  return None;
420  return *EI->idx_begin();
421 }
422 
423 /// \returns True if in-tree use also needs extract. This refers to
424 /// possible scalar operand in vectorized instruction.
426  TargetLibraryInfo *TLI) {
427  unsigned Opcode = UserInst->getOpcode();
428  switch (Opcode) {
429  case Instruction::Load: {
430  LoadInst *LI = cast<LoadInst>(UserInst);
431  return (LI->getPointerOperand() == Scalar);
432  }
433  case Instruction::Store: {
434  StoreInst *SI = cast<StoreInst>(UserInst);
435  return (SI->getPointerOperand() == Scalar);
436  }
437  case Instruction::Call: {
438  CallInst *CI = cast<CallInst>(UserInst);
440  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
441  if (hasVectorInstrinsicScalarOpd(ID, i))
442  return (CI->getArgOperand(i) == Scalar);
443  }
445  }
446  default:
447  return false;
448  }
449 }
450 
451 /// \returns the AA location that is being access by the instruction.
453  if (StoreInst *SI = dyn_cast<StoreInst>(I))
454  return MemoryLocation::get(SI);
455  if (LoadInst *LI = dyn_cast<LoadInst>(I))
456  return MemoryLocation::get(LI);
457  return MemoryLocation();
458 }
459 
460 /// \returns True if the instruction is not a volatile or atomic load/store.
461 static bool isSimple(Instruction *I) {
462  if (LoadInst *LI = dyn_cast<LoadInst>(I))
463  return LI->isSimple();
464  if (StoreInst *SI = dyn_cast<StoreInst>(I))
465  return SI->isSimple();
466  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
467  return !MI->isVolatile();
468  return true;
469 }
470 
471 namespace llvm {
472 
473 namespace slpvectorizer {
474 
475 /// Bottom Up SLP Vectorizer.
476 class BoUpSLP {
477 public:
484 
488  const DataLayout *DL, OptimizationRemarkEmitter *ORE)
489  : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
490  DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
491  CodeMetrics::collectEphemeralValues(F, AC, EphValues);
492  // Use the vector register size specified by the target unless overridden
493  // by a command-line option.
494  // TODO: It would be better to limit the vectorization factor based on
495  // data type rather than just register size. For example, x86 AVX has
496  // 256-bit registers, but it does not support integer operations
497  // at that width (that requires AVX2).
498  if (MaxVectorRegSizeOption.getNumOccurrences())
499  MaxVecRegSize = MaxVectorRegSizeOption;
500  else
501  MaxVecRegSize = TTI->getRegisterBitWidth(true);
502 
503  if (MinVectorRegSizeOption.getNumOccurrences())
504  MinVecRegSize = MinVectorRegSizeOption;
505  else
506  MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
507  }
508 
509  /// Vectorize the tree that starts with the elements in \p VL.
510  /// Returns the vectorized root.
511  Value *vectorizeTree();
512 
513  /// Vectorize the tree but with the list of externally used values \p
514  /// ExternallyUsedValues. Values in this MapVector can be replaced but the
515  /// generated extractvalue instructions.
516  Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
517 
518  /// \returns the cost incurred by unwanted spills and fills, caused by
519  /// holding live values over call sites.
520  int getSpillCost();
521 
522  /// \returns the vectorization cost of the subtree that starts at \p VL.
523  /// A negative number means that this is profitable.
524  int getTreeCost();
525 
526  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
527  /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
528  void buildTree(ArrayRef<Value *> Roots,
529  ArrayRef<Value *> UserIgnoreLst = None);
530 
531  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
532  /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
533  /// into account (anf updating it, if required) list of externally used
534  /// values stored in \p ExternallyUsedValues.
535  void buildTree(ArrayRef<Value *> Roots,
536  ExtraValueToDebugLocsMap &ExternallyUsedValues,
537  ArrayRef<Value *> UserIgnoreLst = None);
538 
539  /// Clear the internal data structures that are created by 'buildTree'.
540  void deleteTree() {
541  VectorizableTree.clear();
542  ScalarToTreeEntry.clear();
543  MustGather.clear();
544  ExternalUses.clear();
545  NumOpsWantToKeepOrder.clear();
546  NumOpsWantToKeepOriginalOrder = 0;
547  for (auto &Iter : BlocksSchedules) {
548  BlockScheduling *BS = Iter.second.get();
549  BS->clear();
550  }
551  MinBWs.clear();
552  }
553 
554  unsigned getTreeSize() const { return VectorizableTree.size(); }
555 
556  /// Perform LICM and CSE on the newly generated gather sequences.
557  void optimizeGatherSequence();
558 
559  /// \returns The best order of instructions for vectorization.
561  auto I = std::max_element(
562  NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
563  [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
564  const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
565  return D1.second < D2.second;
566  });
567  if (I == NumOpsWantToKeepOrder.end() ||
568  I->getSecond() <= NumOpsWantToKeepOriginalOrder)
569  return None;
570 
571  return makeArrayRef(I->getFirst());
572  }
573 
574  /// \return The vector element size in bits to use when vectorizing the
575  /// expression tree ending at \p V. If V is a store, the size is the width of
576  /// the stored value. Otherwise, the size is the width of the largest loaded
577  /// value reaching V. This method is used by the vectorizer to calculate
578  /// vectorization factors.
579  unsigned getVectorElementSize(Value *V);
580 
581  /// Compute the minimum type sizes required to represent the entries in a
582  /// vectorizable tree.
584 
585  // \returns maximum vector register size as set by TTI or overridden by cl::opt.
586  unsigned getMaxVecRegSize() const {
587  return MaxVecRegSize;
588  }
589 
590  // \returns minimum vector register size as set by cl::opt.
591  unsigned getMinVecRegSize() const {
592  return MinVecRegSize;
593  }
594 
595  /// Check if ArrayType or StructType is isomorphic to some VectorType.
596  ///
597  /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
598  unsigned canMapToVector(Type *T, const DataLayout &DL) const;
599 
600  /// \returns True if the VectorizableTree is both tiny and not fully
601  /// vectorizable. We do not vectorize such trees.
602  bool isTreeTinyAndNotFullyVectorizable();
603 
605 
606 private:
607  struct TreeEntry;
608 
609  /// Checks if all users of \p I are the part of the vectorization tree.
610  bool areAllUsersVectorized(Instruction *I) const;
611 
612  /// \returns the cost of the vectorizable entry.
613  int getEntryCost(TreeEntry *E);
614 
615  /// This is the recursive part of buildTree.
616  void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
617 
618  /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
619  /// be vectorized to use the original vector (or aggregate "bitcast" to a
620  /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
621  /// returns false, setting \p CurrentOrder to either an empty vector or a
622  /// non-identity permutation that allows to reuse extract instructions.
623  bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
624  SmallVectorImpl<unsigned> &CurrentOrder) const;
625 
626  /// Vectorize a single entry in the tree.
627  Value *vectorizeTree(TreeEntry *E);
628 
629  /// Vectorize a single entry in the tree, starting in \p VL.
630  Value *vectorizeTree(ArrayRef<Value *> VL);
631 
632  /// \returns the scalarization cost for this type. Scalarization in this
633  /// context means the creation of vectors from a group of scalars.
634  int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
635 
636  /// \returns the scalarization cost for this list of values. Assuming that
637  /// this subtree gets vectorized, we may need to extract the values from the
638  /// roots. This method calculates the cost of extracting the values.
639  int getGatherCost(ArrayRef<Value *> VL);
640 
641  /// Set the Builder insert point to one after the last instruction in
642  /// the bundle
643  void setInsertPointAfterBundle(ArrayRef<Value *> VL,
644  const InstructionsState &S);
645 
646  /// \returns a vector from a collection of scalars in \p VL.
647  Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
648 
649  /// \returns whether the VectorizableTree is fully vectorizable and will
650  /// be beneficial even the tree height is tiny.
651  bool isFullyVectorizableTinyTree();
652 
653  /// \reorder commutative operands in alt shuffle if they result in
654  /// vectorized code.
655  void reorderAltShuffleOperands(const InstructionsState &S,
659 
660  /// \reorder commutative operands to get better probability of
661  /// generating vectorized code.
662  void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
664  SmallVectorImpl<Value *> &Right);
665  struct TreeEntry {
666  TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
667 
668  /// \returns true if the scalars in VL are equal to this entry.
669  bool isSame(ArrayRef<Value *> VL) const {
670  if (VL.size() == Scalars.size())
671  return std::equal(VL.begin(), VL.end(), Scalars.begin());
672  return VL.size() == ReuseShuffleIndices.size() &&
673  std::equal(
674  VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
675  [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
676  }
677 
678  /// A vector of scalars.
679  ValueList Scalars;
680 
681  /// The Scalars are vectorized into this value. It is initialized to Null.
682  Value *VectorizedValue = nullptr;
683 
684  /// Do we need to gather this sequence ?
685  bool NeedToGather = false;
686 
687  /// Does this sequence require some shuffling?
688  SmallVector<unsigned, 4> ReuseShuffleIndices;
689 
690  /// Does this entry require reordering?
691  ArrayRef<unsigned> ReorderIndices;
692 
693  /// Points back to the VectorizableTree.
694  ///
695  /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
696  /// to be a pointer and needs to be able to initialize the child iterator.
697  /// Thus we need a reference back to the container to translate the indices
698  /// to entries.
699  std::vector<TreeEntry> &Container;
700 
701  /// The TreeEntry index containing the user of this entry. We can actually
702  /// have multiple users so the data structure is not truly a tree.
703  SmallVector<int, 1> UserTreeIndices;
704  };
705 
706  /// Create a new VectorizableTree entry.
707  void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
708  ArrayRef<unsigned> ReuseShuffleIndices = None,
709  ArrayRef<unsigned> ReorderIndices = None) {
710  VectorizableTree.emplace_back(VectorizableTree);
711  int idx = VectorizableTree.size() - 1;
712  TreeEntry *Last = &VectorizableTree[idx];
713  Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
714  Last->NeedToGather = !Vectorized;
715  Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
716  ReuseShuffleIndices.end());
717  Last->ReorderIndices = ReorderIndices;
718  if (Vectorized) {
719  for (int i = 0, e = VL.size(); i != e; ++i) {
720  assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
721  ScalarToTreeEntry[VL[i]] = idx;
722  }
723  } else {
724  MustGather.insert(VL.begin(), VL.end());
725  }
726 
727  if (UserTreeIdx >= 0)
728  Last->UserTreeIndices.push_back(UserTreeIdx);
729  UserTreeIdx = idx;
730  }
731 
732  /// -- Vectorization State --
733  /// Holds all of the tree entries.
734  std::vector<TreeEntry> VectorizableTree;
735 
736  TreeEntry *getTreeEntry(Value *V) {
737  auto I = ScalarToTreeEntry.find(V);
738  if (I != ScalarToTreeEntry.end())
739  return &VectorizableTree[I->second];
740  return nullptr;
741  }
742 
743  /// Maps a specific scalar to its tree entry.
744  SmallDenseMap<Value*, int> ScalarToTreeEntry;
745 
746  /// A list of scalars that we found that we need to keep as scalars.
747  ValueSet MustGather;
748 
749  /// This POD struct describes one external user in the vectorized tree.
750  struct ExternalUser {
751  ExternalUser(Value *S, llvm::User *U, int L)
752  : Scalar(S), User(U), Lane(L) {}
753 
754  // Which scalar in our function.
755  Value *Scalar;
756 
757  // Which user that uses the scalar.
758  llvm::User *User;
759 
760  // Which lane does the scalar belong to.
761  int Lane;
762  };
764 
765  /// Checks if two instructions may access the same memory.
766  ///
767  /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
768  /// is invariant in the calling loop.
769  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
770  Instruction *Inst2) {
771  // First check if the result is already in the cache.
772  AliasCacheKey key = std::make_pair(Inst1, Inst2);
773  Optional<bool> &result = AliasCache[key];
774  if (result.hasValue()) {
775  return result.getValue();
776  }
777  MemoryLocation Loc2 = getLocation(Inst2, AA);
778  bool aliased = true;
779  if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
780  // Do the alias check.
781  aliased = AA->alias(Loc1, Loc2);
782  }
783  // Store the result in the cache.
784  result = aliased;
785  return aliased;
786  }
787 
788  using AliasCacheKey = std::pair<Instruction *, Instruction *>;
789 
790  /// Cache for alias results.
791  /// TODO: consider moving this to the AliasAnalysis itself.
793 
794  /// Removes an instruction from its block and eventually deletes it.
795  /// It's like Instruction::eraseFromParent() except that the actual deletion
796  /// is delayed until BoUpSLP is destructed.
797  /// This is required to ensure that there are no incorrect collisions in the
798  /// AliasCache, which can happen if a new instruction is allocated at the
799  /// same address as a previously deleted instruction.
800  void eraseInstruction(Instruction *I) {
801  I->removeFromParent();
802  I->dropAllReferences();
803  DeletedInstructions.emplace_back(I);
804  }
805 
806  /// Temporary store for deleted instructions. Instructions will be deleted
807  /// eventually when the BoUpSLP is destructed.
808  SmallVector<unique_value, 8> DeletedInstructions;
809 
810  /// A list of values that need to extracted out of the tree.
811  /// This list holds pairs of (Internal Scalar : External User). External User
812  /// can be nullptr, it means that this Internal Scalar will be used later,
813  /// after vectorization.
814  UserList ExternalUses;
815 
816  /// Values used only by @llvm.assume calls.
818 
819  /// Holds all of the instructions that we gathered.
820  SetVector<Instruction *> GatherSeq;
821 
822  /// A list of blocks that we are going to CSE.
823  SetVector<BasicBlock *> CSEBlocks;
824 
825  /// Contains all scheduling relevant data for an instruction.
826  /// A ScheduleData either represents a single instruction or a member of an
827  /// instruction bundle (= a group of instructions which is combined into a
828  /// vector instruction).
829  struct ScheduleData {
830  // The initial value for the dependency counters. It means that the
831  // dependencies are not calculated yet.
832  enum { InvalidDeps = -1 };
833 
834  ScheduleData() = default;
835 
836  void init(int BlockSchedulingRegionID, Value *OpVal) {
837  FirstInBundle = this;
838  NextInBundle = nullptr;
839  NextLoadStore = nullptr;
840  IsScheduled = false;
841  SchedulingRegionID = BlockSchedulingRegionID;
842  UnscheduledDepsInBundle = UnscheduledDeps;
843  clearDependencies();
844  OpValue = OpVal;
845  }
846 
847  /// Returns true if the dependency information has been calculated.
848  bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
849 
850  /// Returns true for single instructions and for bundle representatives
851  /// (= the head of a bundle).
852  bool isSchedulingEntity() const { return FirstInBundle == this; }
853 
854  /// Returns true if it represents an instruction bundle and not only a
855  /// single instruction.
856  bool isPartOfBundle() const {
857  return NextInBundle != nullptr || FirstInBundle != this;
858  }
859 
860  /// Returns true if it is ready for scheduling, i.e. it has no more
861  /// unscheduled depending instructions/bundles.
862  bool isReady() const {
863  assert(isSchedulingEntity() &&
864  "can't consider non-scheduling entity for ready list");
865  return UnscheduledDepsInBundle == 0 && !IsScheduled;
866  }
867 
868  /// Modifies the number of unscheduled dependencies, also updating it for
869  /// the whole bundle.
870  int incrementUnscheduledDeps(int Incr) {
871  UnscheduledDeps += Incr;
872  return FirstInBundle->UnscheduledDepsInBundle += Incr;
873  }
874 
875  /// Sets the number of unscheduled dependencies to the number of
876  /// dependencies.
877  void resetUnscheduledDeps() {
878  incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
879  }
880 
881  /// Clears all dependency information.
882  void clearDependencies() {
883  Dependencies = InvalidDeps;
884  resetUnscheduledDeps();
885  MemoryDependencies.clear();
886  }
887 
888  void dump(raw_ostream &os) const {
889  if (!isSchedulingEntity()) {
890  os << "/ " << *Inst;
891  } else if (NextInBundle) {
892  os << '[' << *Inst;
893  ScheduleData *SD = NextInBundle;
894  while (SD) {
895  os << ';' << *SD->Inst;
896  SD = SD->NextInBundle;
897  }
898  os << ']';
899  } else {
900  os << *Inst;
901  }
902  }
903 
904  Instruction *Inst = nullptr;
905 
906  /// Points to the head in an instruction bundle (and always to this for
907  /// single instructions).
908  ScheduleData *FirstInBundle = nullptr;
909 
910  /// Single linked list of all instructions in a bundle. Null if it is a
911  /// single instruction.
912  ScheduleData *NextInBundle = nullptr;
913 
914  /// Single linked list of all memory instructions (e.g. load, store, call)
915  /// in the block - until the end of the scheduling region.
916  ScheduleData *NextLoadStore = nullptr;
917 
918  /// The dependent memory instructions.
919  /// This list is derived on demand in calculateDependencies().
920  SmallVector<ScheduleData *, 4> MemoryDependencies;
921 
922  /// This ScheduleData is in the current scheduling region if this matches
923  /// the current SchedulingRegionID of BlockScheduling.
924  int SchedulingRegionID = 0;
925 
926  /// Used for getting a "good" final ordering of instructions.
927  int SchedulingPriority = 0;
928 
929  /// The number of dependencies. Constitutes of the number of users of the
930  /// instruction plus the number of dependent memory instructions (if any).
931  /// This value is calculated on demand.
932  /// If InvalidDeps, the number of dependencies is not calculated yet.
933  int Dependencies = InvalidDeps;
934 
935  /// The number of dependencies minus the number of dependencies of scheduled
936  /// instructions. As soon as this is zero, the instruction/bundle gets ready
937  /// for scheduling.
938  /// Note that this is negative as long as Dependencies is not calculated.
939  int UnscheduledDeps = InvalidDeps;
940 
941  /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
942  /// single instructions.
943  int UnscheduledDepsInBundle = InvalidDeps;
944 
945  /// True if this instruction is scheduled (or considered as scheduled in the
946  /// dry-run).
947  bool IsScheduled = false;
948 
949  /// Opcode of the current instruction in the schedule data.
950  Value *OpValue = nullptr;
951  };
952 
953 #ifndef NDEBUG
954  friend inline raw_ostream &operator<<(raw_ostream &os,
955  const BoUpSLP::ScheduleData &SD) {
956  SD.dump(os);
957  return os;
958  }
959 #endif
960 
961  friend struct GraphTraits<BoUpSLP *>;
962  friend struct DOTGraphTraits<BoUpSLP *>;
963 
964  /// Contains all scheduling data for a basic block.
965  struct BlockScheduling {
966  BlockScheduling(BasicBlock *BB)
967  : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
968 
969  void clear() {
970  ReadyInsts.clear();
971  ScheduleStart = nullptr;
972  ScheduleEnd = nullptr;
973  FirstLoadStoreInRegion = nullptr;
974  LastLoadStoreInRegion = nullptr;
975 
976  // Reduce the maximum schedule region size by the size of the
977  // previous scheduling run.
978  ScheduleRegionSizeLimit -= ScheduleRegionSize;
979  if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
980  ScheduleRegionSizeLimit = MinScheduleRegionSize;
981  ScheduleRegionSize = 0;
982 
983  // Make a new scheduling region, i.e. all existing ScheduleData is not
984  // in the new region yet.
985  ++SchedulingRegionID;
986  }
987 
988  ScheduleData *getScheduleData(Value *V) {
989  ScheduleData *SD = ScheduleDataMap[V];
990  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
991  return SD;
992  return nullptr;
993  }
994 
995  ScheduleData *getScheduleData(Value *V, Value *Key) {
996  if (V == Key)
997  return getScheduleData(V);
998  auto I = ExtraScheduleDataMap.find(V);
999  if (I != ExtraScheduleDataMap.end()) {
1000  ScheduleData *SD = I->second[Key];
1001  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1002  return SD;
1003  }
1004  return nullptr;
1005  }
1006 
1007  bool isInSchedulingRegion(ScheduleData *SD) {
1008  return SD->SchedulingRegionID == SchedulingRegionID;
1009  }
1010 
1011  /// Marks an instruction as scheduled and puts all dependent ready
1012  /// instructions into the ready-list.
1013  template <typename ReadyListType>
1014  void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1015  SD->IsScheduled = true;
1016  LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1017 
1018  ScheduleData *BundleMember = SD;
1019  while (BundleMember) {
1020  if (BundleMember->Inst != BundleMember->OpValue) {
1021  BundleMember = BundleMember->NextInBundle;
1022  continue;
1023  }
1024  // Handle the def-use chain dependencies.
1025  for (Use &U : BundleMember->Inst->operands()) {
1026  auto *I = dyn_cast<Instruction>(U.get());
1027  if (!I)
1028  continue;
1029  doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1030  if (OpDef && OpDef->hasValidDependencies() &&
1031  OpDef->incrementUnscheduledDeps(-1) == 0) {
1032  // There are no more unscheduled dependencies after
1033  // decrementing, so we can put the dependent instruction
1034  // into the ready list.
1035  ScheduleData *DepBundle = OpDef->FirstInBundle;
1036  assert(!DepBundle->IsScheduled &&
1037  "already scheduled bundle gets ready");
1038  ReadyList.insert(DepBundle);
1039  LLVM_DEBUG(dbgs()
1040  << "SLP: gets ready (def): " << *DepBundle << "\n");
1041  }
1042  });
1043  }
1044  // Handle the memory dependencies.
1045  for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1046  if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1047  // There are no more unscheduled dependencies after decrementing,
1048  // so we can put the dependent instruction into the ready list.
1049  ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1050  assert(!DepBundle->IsScheduled &&
1051  "already scheduled bundle gets ready");
1052  ReadyList.insert(DepBundle);
1053  LLVM_DEBUG(dbgs()
1054  << "SLP: gets ready (mem): " << *DepBundle << "\n");
1055  }
1056  }
1057  BundleMember = BundleMember->NextInBundle;
1058  }
1059  }
1060 
1061  void doForAllOpcodes(Value *V,
1062  function_ref<void(ScheduleData *SD)> Action) {
1063  if (ScheduleData *SD = getScheduleData(V))
1064  Action(SD);
1065  auto I = ExtraScheduleDataMap.find(V);
1066  if (I != ExtraScheduleDataMap.end())
1067  for (auto &P : I->second)
1068  if (P.second->SchedulingRegionID == SchedulingRegionID)
1069  Action(P.second);
1070  }
1071 
1072  /// Put all instructions into the ReadyList which are ready for scheduling.
1073  template <typename ReadyListType>
1074  void initialFillReadyList(ReadyListType &ReadyList) {
1075  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1076  doForAllOpcodes(I, [&](ScheduleData *SD) {
1077  if (SD->isSchedulingEntity() && SD->isReady()) {
1078  ReadyList.insert(SD);
1079  LLVM_DEBUG(dbgs()
1080  << "SLP: initially in ready list: " << *I << "\n");
1081  }
1082  });
1083  }
1084  }
1085 
1086  /// Checks if a bundle of instructions can be scheduled, i.e. has no
1087  /// cyclic dependencies. This is only a dry-run, no instructions are
1088  /// actually moved at this stage.
1089  bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
1090  const InstructionsState &S);
1091 
1092  /// Un-bundles a group of instructions.
1093  void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1094 
1095  /// Allocates schedule data chunk.
1096  ScheduleData *allocateScheduleDataChunks();
1097 
1098  /// Extends the scheduling region so that V is inside the region.
1099  /// \returns true if the region size is within the limit.
1100  bool extendSchedulingRegion(Value *V, const InstructionsState &S);
1101 
1102  /// Initialize the ScheduleData structures for new instructions in the
1103  /// scheduling region.
1104  void initScheduleData(Instruction *FromI, Instruction *ToI,
1105  ScheduleData *PrevLoadStore,
1106  ScheduleData *NextLoadStore);
1107 
1108  /// Updates the dependency information of a bundle and of all instructions/
1109  /// bundles which depend on the original bundle.
1110  void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1111  BoUpSLP *SLP);
1112 
1113  /// Sets all instruction in the scheduling region to un-scheduled.
1114  void resetSchedule();
1115 
1116  BasicBlock *BB;
1117 
1118  /// Simple memory allocation for ScheduleData.
1119  std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1120 
1121  /// The size of a ScheduleData array in ScheduleDataChunks.
1122  int ChunkSize;
1123 
1124  /// The allocator position in the current chunk, which is the last entry
1125  /// of ScheduleDataChunks.
1126  int ChunkPos;
1127 
1128  /// Attaches ScheduleData to Instruction.
1129  /// Note that the mapping survives during all vectorization iterations, i.e.
1130  /// ScheduleData structures are recycled.
1131  DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1132 
1133  /// Attaches ScheduleData to Instruction with the leading key.
1135  ExtraScheduleDataMap;
1136 
1137  struct ReadyList : SmallVector<ScheduleData *, 8> {
1138  void insert(ScheduleData *SD) { push_back(SD); }
1139  };
1140 
1141  /// The ready-list for scheduling (only used for the dry-run).
1142  ReadyList ReadyInsts;
1143 
1144  /// The first instruction of the scheduling region.
1145  Instruction *ScheduleStart = nullptr;
1146 
1147  /// The first instruction _after_ the scheduling region.
1148  Instruction *ScheduleEnd = nullptr;
1149 
1150  /// The first memory accessing instruction in the scheduling region
1151  /// (can be null).
1152  ScheduleData *FirstLoadStoreInRegion = nullptr;
1153 
1154  /// The last memory accessing instruction in the scheduling region
1155  /// (can be null).
1156  ScheduleData *LastLoadStoreInRegion = nullptr;
1157 
1158  /// The current size of the scheduling region.
1159  int ScheduleRegionSize = 0;
1160 
1161  /// The maximum size allowed for the scheduling region.
1162  int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1163 
1164  /// The ID of the scheduling region. For a new vectorization iteration this
1165  /// is incremented which "removes" all ScheduleData from the region.
1166  // Make sure that the initial SchedulingRegionID is greater than the
1167  // initial SchedulingRegionID in ScheduleData (which is 0).
1168  int SchedulingRegionID = 1;
1169  };
1170 
1171  /// Attaches the BlockScheduling structures to basic blocks.
1173 
1174  /// Performs the "real" scheduling. Done before vectorization is actually
1175  /// performed in a basic block.
1176  void scheduleBlock(BlockScheduling *BS);
1177 
1178  /// List of users to ignore during scheduling and that don't need extracting.
1179  ArrayRef<Value *> UserIgnoreList;
1180 
1182  /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
1183  /// sorted SmallVectors of unsigned.
1184  struct OrdersTypeDenseMapInfo {
1185  static OrdersType getEmptyKey() {
1186  OrdersType V;
1187  V.push_back(~1U);
1188  return V;
1189  }
1190 
1191  static OrdersType getTombstoneKey() {
1192  OrdersType V;
1193  V.push_back(~2U);
1194  return V;
1195  }
1196 
1197  static unsigned getHashValue(const OrdersType &V) {
1198  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1199  }
1200 
1201  static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
1202  return LHS == RHS;
1203  }
1204  };
1205 
1206  /// Contains orders of operations along with the number of bundles that have
1207  /// operations in this order. It stores only those orders that require
1208  /// reordering, if reordering is not required it is counted using \a
1209  /// NumOpsWantToKeepOriginalOrder.
1211  /// Number of bundles that do not require reordering.
1212  unsigned NumOpsWantToKeepOriginalOrder = 0;
1213 
1214  // Analysis and block reference.
1215  Function *F;
1216  ScalarEvolution *SE;
1217  TargetTransformInfo *TTI;
1218  TargetLibraryInfo *TLI;
1219  AliasAnalysis *AA;
1220  LoopInfo *LI;
1221  DominatorTree *DT;
1222  AssumptionCache *AC;
1223  DemandedBits *DB;
1224  const DataLayout *DL;
1226 
1227  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1228  unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1229 
1230  /// Instruction builder to construct the vectorized tree.
1231  IRBuilder<> Builder;
1232 
1233  /// A map of scalar integer values to the smallest bit width with which they
1234  /// can legally be represented. The values map to (width, signed) pairs,
1235  /// where "width" indicates the minimum bit width and "signed" is True if the
1236  /// value must be signed-extended, rather than zero-extended, back to its
1237  /// original width.
1239 };
1240 
1241 } // end namespace slpvectorizer
1242 
1243 template <> struct GraphTraits<BoUpSLP *> {
1244  using TreeEntry = BoUpSLP::TreeEntry;
1245 
1246  /// NodeRef has to be a pointer per the GraphWriter.
1247  using NodeRef = TreeEntry *;
1248 
1249  /// Add the VectorizableTree to the index iterator to be able to return
1250  /// TreeEntry pointers.
1251  struct ChildIteratorType
1252  : public iterator_adaptor_base<ChildIteratorType,
1253  SmallVector<int, 1>::iterator> {
1254  std::vector<TreeEntry> &VectorizableTree;
1255 
1257  std::vector<TreeEntry> &VT)
1258  : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1259 
1260  NodeRef operator*() { return &VectorizableTree[*I]; }
1261  };
1262 
1263  static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1264 
1265  static ChildIteratorType child_begin(NodeRef N) {
1266  return {N->UserTreeIndices.begin(), N->Container};
1267  }
1268 
1269  static ChildIteratorType child_end(NodeRef N) {
1270  return {N->UserTreeIndices.end(), N->Container};
1271  }
1272 
1273  /// For the node iterator we just need to turn the TreeEntry iterator into a
1274  /// TreeEntry* iterator so that it dereferences to NodeRef.
1276 
1277  static nodes_iterator nodes_begin(BoUpSLP *R) {
1278  return nodes_iterator(R->VectorizableTree.begin());
1279  }
1280 
1281  static nodes_iterator nodes_end(BoUpSLP *R) {
1282  return nodes_iterator(R->VectorizableTree.end());
1283  }
1284 
1285  static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1286 };
1287 
1288 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1289  using TreeEntry = BoUpSLP::TreeEntry;
1290 
1292 
1293  std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1294  std::string Str;
1295  raw_string_ostream OS(Str);
1296  if (isSplat(Entry->Scalars)) {
1297  OS << "<splat> " << *Entry->Scalars[0];
1298  return Str;
1299  }
1300  for (auto V : Entry->Scalars) {
1301  OS << *V;
1302  if (std::any_of(
1303  R->ExternalUses.begin(), R->ExternalUses.end(),
1304  [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1305  OS << " <extract>";
1306  OS << "\n";
1307  }
1308  return Str;
1309  }
1310 
1311  static std::string getNodeAttributes(const TreeEntry *Entry,
1312  const BoUpSLP *) {
1313  if (Entry->NeedToGather)
1314  return "color=red";
1315  return "";
1316  }
1317 };
1318 
1319 } // end namespace llvm
1320 
1321 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1322  ArrayRef<Value *> UserIgnoreLst) {
1323  ExtraValueToDebugLocsMap ExternallyUsedValues;
1324  buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1325 }
1326 
1327 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1328  ExtraValueToDebugLocsMap &ExternallyUsedValues,
1329  ArrayRef<Value *> UserIgnoreLst) {
1330  deleteTree();
1331  UserIgnoreList = UserIgnoreLst;
1332  if (!allSameType(Roots))
1333  return;
1334  buildTree_rec(Roots, 0, -1);
1335 
1336  // Collect the values that we need to extract from the tree.
1337  for (TreeEntry &EIdx : VectorizableTree) {
1338  TreeEntry *Entry = &EIdx;
1339 
1340  // No need to handle users of gathered values.
1341  if (Entry->NeedToGather)
1342  continue;
1343 
1344  // For each lane:
1345  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1346  Value *Scalar = Entry->Scalars[Lane];
1347  int FoundLane = Lane;
1348  if (!Entry->ReuseShuffleIndices.empty()) {
1349  FoundLane =
1350  std::distance(Entry->ReuseShuffleIndices.begin(),
1351  llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1352  }
1353 
1354  // Check if the scalar is externally used as an extra arg.
1355  auto ExtI = ExternallyUsedValues.find(Scalar);
1356  if (ExtI != ExternallyUsedValues.end()) {
1357  LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
1358  << Lane << " from " << *Scalar << ".\n");
1359  ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1360  }
1361  for (User *U : Scalar->users()) {
1362  LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1363 
1364  Instruction *UserInst = dyn_cast<Instruction>(U);
1365  if (!UserInst)
1366  continue;
1367 
1368  // Skip in-tree scalars that become vectors
1369  if (TreeEntry *UseEntry = getTreeEntry(U)) {
1370  Value *UseScalar = UseEntry->Scalars[0];
1371  // Some in-tree scalars will remain as scalar in vectorized
1372  // instructions. If that is the case, the one in Lane 0 will
1373  // be used.
1374  if (UseScalar != U ||
1375  !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1376  LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1377  << ".\n");
1378  assert(!UseEntry->NeedToGather && "Bad state");
1379  continue;
1380  }
1381  }
1382 
1383  // Ignore users in the user ignore list.
1384  if (is_contained(UserIgnoreList, UserInst))
1385  continue;
1386 
1387  LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "
1388  << Lane << " from " << *Scalar << ".\n");
1389  ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1390  }
1391  }
1392  }
1393 }
1394 
1395 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1396  int UserTreeIdx) {
1397  assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1398 
1399  InstructionsState S = getSameOpcode(VL);
1400  if (Depth == RecursionMaxDepth) {
1401  LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1402  newTreeEntry(VL, false, UserTreeIdx);
1403  return;
1404  }
1405 
1406  // Don't handle vectors.
1407  if (S.OpValue->getType()->isVectorTy()) {
1408  LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1409  newTreeEntry(VL, false, UserTreeIdx);
1410  return;
1411  }
1412 
1413  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1414  if (SI->getValueOperand()->getType()->isVectorTy()) {
1415  LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1416  newTreeEntry(VL, false, UserTreeIdx);
1417  return;
1418  }
1419 
1420  // If all of the operands are identical or constant we have a simple solution.
1421  if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
1422  LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1423  newTreeEntry(VL, false, UserTreeIdx);
1424  return;
1425  }
1426 
1427  // We now know that this is a vector of instructions of the same type from
1428  // the same block.
1429 
1430  // Don't vectorize ephemeral values.
1431  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1432  if (EphValues.count(VL[i])) {
1433  LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]
1434  << ") is ephemeral.\n");
1435  newTreeEntry(VL, false, UserTreeIdx);
1436  return;
1437  }
1438  }
1439 
1440  // Check if this is a duplicate of another entry.
1441  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1442  LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n");
1443  if (!E->isSame(VL)) {
1444  LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1445  newTreeEntry(VL, false, UserTreeIdx);
1446  return;
1447  }
1448  // Record the reuse of the tree node. FIXME, currently this is only used to
1449  // properly draw the graph rather than for the actual vectorization.
1450  E->UserTreeIndices.push_back(UserTreeIdx);
1451  LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue
1452  << ".\n");
1453  return;
1454  }
1455 
1456  // Check that none of the instructions in the bundle are already in the tree.
1457  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1458  auto *I = dyn_cast<Instruction>(VL[i]);
1459  if (!I)
1460  continue;
1461  if (getTreeEntry(I)) {
1462  LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]
1463  << ") is already in tree.\n");
1464  newTreeEntry(VL, false, UserTreeIdx);
1465  return;
1466  }
1467  }
1468 
1469  // If any of the scalars is marked as a value that needs to stay scalar, then
1470  // we need to gather the scalars.
1471  // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
1472  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1473  if (MustGather.count(VL[i]) || is_contained(UserIgnoreList, VL[i])) {
1474  LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1475  newTreeEntry(VL, false, UserTreeIdx);
1476  return;
1477  }
1478  }
1479 
1480  // Check that all of the users of the scalars that we want to vectorize are
1481  // schedulable.
1482  auto *VL0 = cast<Instruction>(S.OpValue);
1483  BasicBlock *BB = VL0->getParent();
1484 
1485  if (!DT->isReachableFromEntry(BB)) {
1486  // Don't go into unreachable blocks. They may contain instructions with
1487  // dependency cycles which confuse the final scheduling.
1488  LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1489  newTreeEntry(VL, false, UserTreeIdx);
1490  return;
1491  }
1492 
1493  // Check that every instruction appears once in this bundle.
1494  SmallVector<unsigned, 4> ReuseShuffleIndicies;
1495  SmallVector<Value *, 4> UniqueValues;
1496  DenseMap<Value *, unsigned> UniquePositions;
1497  for (Value *V : VL) {
1498  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1499  ReuseShuffleIndicies.emplace_back(Res.first->second);
1500  if (Res.second)
1501  UniqueValues.emplace_back(V);
1502  }
1503  if (UniqueValues.size() == VL.size()) {
1504  ReuseShuffleIndicies.clear();
1505  } else {
1506  LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
1507  if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1508  LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1509  newTreeEntry(VL, false, UserTreeIdx);
1510  return;
1511  }
1512  VL = UniqueValues;
1513  }
1514 
1515  auto &BSRef = BlocksSchedules[BB];
1516  if (!BSRef)
1517  BSRef = llvm::make_unique<BlockScheduling>(BB);
1518 
1519  BlockScheduling &BS = *BSRef.get();
1520 
1521  if (!BS.tryScheduleBundle(VL, this, S)) {
1522  LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1523  assert((!BS.getScheduleData(VL0) ||
1524  !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1525  "tryScheduleBundle should cancelScheduling on failure");
1526  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1527  return;
1528  }
1529  LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1530 
1531  unsigned ShuffleOrOp = S.isAltShuffle() ?
1532  (unsigned) Instruction::ShuffleVector : S.getOpcode();
1533  switch (ShuffleOrOp) {
1534  case Instruction::PHI: {
1535  PHINode *PH = dyn_cast<PHINode>(VL0);
1536 
1537  // Check for terminator values (e.g. invoke).
1538  for (unsigned j = 0; j < VL.size(); ++j)
1539  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1540  Instruction *Term = dyn_cast<Instruction>(
1541  cast<PHINode>(VL[j])->getIncomingValueForBlock(
1542  PH->getIncomingBlock(i)));
1543  if (Term && Term->isTerminator()) {
1544  LLVM_DEBUG(dbgs()
1545  << "SLP: Need to swizzle PHINodes (terminator use).\n");
1546  BS.cancelScheduling(VL, VL0);
1547  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1548  return;
1549  }
1550  }
1551 
1552  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1553  LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1554 
1555  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1556  ValueList Operands;
1557  // Prepare the operand vector.
1558  for (Value *j : VL)
1559  Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1560  PH->getIncomingBlock(i)));
1561 
1562  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1563  }
1564  return;
1565  }
1566  case Instruction::ExtractValue:
1567  case Instruction::ExtractElement: {
1568  OrdersType CurrentOrder;
1569  bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
1570  if (Reuse) {
1571  LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
1572  ++NumOpsWantToKeepOriginalOrder;
1573  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1574  ReuseShuffleIndicies);
1575  return;
1576  }
1577  if (!CurrentOrder.empty()) {
1578  LLVM_DEBUG({
1579  dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
1580  "with order";
1581  for (unsigned Idx : CurrentOrder)
1582  dbgs() << " " << Idx;
1583  dbgs() << "\n";
1584  });
1585  // Insert new order with initial value 0, if it does not exist,
1586  // otherwise return the iterator to the existing one.
1587  auto StoredCurrentOrderAndNum =
1588  NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1589  ++StoredCurrentOrderAndNum->getSecond();
1590  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx, ReuseShuffleIndicies,
1591  StoredCurrentOrderAndNum->getFirst());
1592  return;
1593  }
1594  LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
1595  newTreeEntry(VL, /*Vectorized=*/false, UserTreeIdx, ReuseShuffleIndicies);
1596  BS.cancelScheduling(VL, VL0);
1597  return;
1598  }
1599  case Instruction::Load: {
1600  // Check that a vectorized load would load the same memory as a scalar
1601  // load. For example, we don't want to vectorize loads that are smaller
1602  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1603  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1604  // from such a struct, we read/write packed bits disagreeing with the
1605  // unvectorized version.
1606  Type *ScalarTy = VL0->getType();
1607 
1608  if (DL->getTypeSizeInBits(ScalarTy) !=
1609  DL->getTypeAllocSizeInBits(ScalarTy)) {
1610  BS.cancelScheduling(VL, VL0);
1611  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1612  LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1613  return;
1614  }
1615 
1616  // Make sure all loads in the bundle are simple - we can't vectorize
1617  // atomic or volatile loads.
1618  SmallVector<Value *, 4> PointerOps(VL.size());
1619  auto POIter = PointerOps.begin();
1620  for (Value *V : VL) {
1621  auto *L = cast<LoadInst>(V);
1622  if (!L->isSimple()) {
1623  BS.cancelScheduling(VL, VL0);
1624  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1625  LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1626  return;
1627  }
1628  *POIter = L->getPointerOperand();
1629  ++POIter;
1630  }
1631 
1632  OrdersType CurrentOrder;
1633  // Check the order of pointer operands.
1634  if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
1635  Value *Ptr0;
1636  Value *PtrN;
1637  if (CurrentOrder.empty()) {
1638  Ptr0 = PointerOps.front();
1639  PtrN = PointerOps.back();
1640  } else {
1641  Ptr0 = PointerOps[CurrentOrder.front()];
1642  PtrN = PointerOps[CurrentOrder.back()];
1643  }
1644  const SCEV *Scev0 = SE->getSCEV(Ptr0);
1645  const SCEV *ScevN = SE->getSCEV(PtrN);
1646  const auto *Diff =
1647  dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
1648  uint64_t Size = DL->getTypeAllocSize(ScalarTy);
1649  // Check that the sorted loads are consecutive.
1650  if (Diff && Diff->getAPInt().getZExtValue() == (VL.size() - 1) * Size) {
1651  if (CurrentOrder.empty()) {
1652  // Original loads are consecutive and does not require reordering.
1653  ++NumOpsWantToKeepOriginalOrder;
1654  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1655  ReuseShuffleIndicies);
1656  LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1657  } else {
1658  // Need to reorder.
1659  auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1660  ++I->getSecond();
1661  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1662  ReuseShuffleIndicies, I->getFirst());
1663  LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
1664  }
1665  return;
1666  }
1667  }
1668 
1669  LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1670  BS.cancelScheduling(VL, VL0);
1671  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1672  return;
1673  }
1674  case Instruction::ZExt:
1675  case Instruction::SExt:
1676  case Instruction::FPToUI:
1677  case Instruction::FPToSI:
1678  case Instruction::FPExt:
1679  case Instruction::PtrToInt:
1680  case Instruction::IntToPtr:
1681  case Instruction::SIToFP:
1682  case Instruction::UIToFP:
1683  case Instruction::Trunc:
1684  case Instruction::FPTrunc:
1685  case Instruction::BitCast: {
1686  Type *SrcTy = VL0->getOperand(0)->getType();
1687  for (unsigned i = 0; i < VL.size(); ++i) {
1688  Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1689  if (Ty != SrcTy || !isValidElementType(Ty)) {
1690  BS.cancelScheduling(VL, VL0);
1691  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1692  LLVM_DEBUG(dbgs()
1693  << "SLP: Gathering casts with different src types.\n");
1694  return;
1695  }
1696  }
1697  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1698  LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1699 
1700  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1701  ValueList Operands;
1702  // Prepare the operand vector.
1703  for (Value *j : VL)
1704  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1705 
1706  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1707  }
1708  return;
1709  }
1710  case Instruction::ICmp:
1711  case Instruction::FCmp: {
1712  // Check that all of the compares have the same predicate.
1713  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1714  Type *ComparedTy = VL0->getOperand(0)->getType();
1715  for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1716  CmpInst *Cmp = cast<CmpInst>(VL[i]);
1717  if (Cmp->getPredicate() != P0 ||
1718  Cmp->getOperand(0)->getType() != ComparedTy) {
1719  BS.cancelScheduling(VL, VL0);
1720  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1721  LLVM_DEBUG(dbgs()
1722  << "SLP: Gathering cmp with different predicate.\n");
1723  return;
1724  }
1725  }
1726 
1727  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1728  LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1729 
1730  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1731  ValueList Operands;
1732  // Prepare the operand vector.
1733  for (Value *j : VL)
1734  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1735 
1736  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1737  }
1738  return;
1739  }
1740  case Instruction::Select:
1741  case Instruction::Add:
1742  case Instruction::FAdd:
1743  case Instruction::Sub:
1744  case Instruction::FSub:
1745  case Instruction::Mul:
1746  case Instruction::FMul:
1747  case Instruction::UDiv:
1748  case Instruction::SDiv:
1749  case Instruction::FDiv:
1750  case Instruction::URem:
1751  case Instruction::SRem:
1752  case Instruction::FRem:
1753  case Instruction::Shl:
1754  case Instruction::LShr:
1755  case Instruction::AShr:
1756  case Instruction::And:
1757  case Instruction::Or:
1758  case Instruction::Xor:
1759  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1760  LLVM_DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1761 
1762  // Sort operands of the instructions so that each side is more likely to
1763  // have the same opcode.
1764  if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1765  ValueList Left, Right;
1766  reorderInputsAccordingToOpcode(S.getOpcode(), VL, Left, Right);
1767  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1768  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1769  return;
1770  }
1771 
1772  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1773  ValueList Operands;
1774  // Prepare the operand vector.
1775  for (Value *j : VL)
1776  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1777 
1778  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1779  }
1780  return;
1781 
1782  case Instruction::GetElementPtr: {
1783  // We don't combine GEPs with complicated (nested) indexing.
1784  for (unsigned j = 0; j < VL.size(); ++j) {
1785  if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1786  LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1787  BS.cancelScheduling(VL, VL0);
1788  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1789  return;
1790  }
1791  }
1792 
1793  // We can't combine several GEPs into one vector if they operate on
1794  // different types.
1795  Type *Ty0 = VL0->getOperand(0)->getType();
1796  for (unsigned j = 0; j < VL.size(); ++j) {
1797  Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1798  if (Ty0 != CurTy) {
1799  LLVM_DEBUG(dbgs()
1800  << "SLP: not-vectorizable GEP (different types).\n");
1801  BS.cancelScheduling(VL, VL0);
1802  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1803  return;
1804  }
1805  }
1806 
1807  // We don't combine GEPs with non-constant indexes.
1808  for (unsigned j = 0; j < VL.size(); ++j) {
1809  auto Op = cast<Instruction>(VL[j])->getOperand(1);
1810  if (!isa<ConstantInt>(Op)) {
1811  LLVM_DEBUG(dbgs()
1812  << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1813  BS.cancelScheduling(VL, VL0);
1814  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1815  return;
1816  }
1817  }
1818 
1819  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1820  LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1821  for (unsigned i = 0, e = 2; i < e; ++i) {
1822  ValueList Operands;
1823  // Prepare the operand vector.
1824  for (Value *j : VL)
1825  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1826 
1827  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1828  }
1829  return;
1830  }
1831  case Instruction::Store: {
1832  // Check if the stores are consecutive or of we need to swizzle them.
1833  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1834  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1835  BS.cancelScheduling(VL, VL0);
1836  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1837  LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1838  return;
1839  }
1840 
1841  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1842  LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1843 
1844  ValueList Operands;
1845  for (Value *j : VL)
1846  Operands.push_back(cast<Instruction>(j)->getOperand(0));
1847 
1848  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1849  return;
1850  }
1851  case Instruction::Call: {
1852  // Check if the calls are all to the same vectorizable intrinsic.
1853  CallInst *CI = cast<CallInst>(VL0);
1854  // Check if this is an Intrinsic call or something that can be
1855  // represented by an intrinsic call
1857  if (!isTriviallyVectorizable(ID)) {
1858  BS.cancelScheduling(VL, VL0);
1859  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1860  LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1861  return;
1862  }
1863  Function *Int = CI->getCalledFunction();
1864  unsigned NumArgs = CI->getNumArgOperands();
1865  SmallVector<Value*, 4> ScalarArgs(NumArgs, nullptr);
1866  for (unsigned j = 0; j != NumArgs; ++j)
1867  if (hasVectorInstrinsicScalarOpd(ID, j))
1868  ScalarArgs[j] = CI->getArgOperand(j);
1869  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1870  CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1871  if (!CI2 || CI2->getCalledFunction() != Int ||
1872  getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1873  !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1874  BS.cancelScheduling(VL, VL0);
1875  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1876  LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1877  << "\n");
1878  return;
1879  }
1880  // Some intrinsics have scalar arguments and should be same in order for
1881  // them to be vectorized.
1882  for (unsigned j = 0; j != NumArgs; ++j) {
1883  if (hasVectorInstrinsicScalarOpd(ID, j)) {
1884  Value *A1J = CI2->getArgOperand(j);
1885  if (ScalarArgs[j] != A1J) {
1886  BS.cancelScheduling(VL, VL0);
1887  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1888  LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1889  << " argument " << ScalarArgs[j] << "!=" << A1J
1890  << "\n");
1891  return;
1892  }
1893  }
1894  }
1895  // Verify that the bundle operands are identical between the two calls.
1896  if (CI->hasOperandBundles() &&
1898  CI->op_begin() + CI->getBundleOperandsEndIndex(),
1899  CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1900  BS.cancelScheduling(VL, VL0);
1901  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1902  LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"
1903  << *CI << "!=" << *VL[i] << '\n');
1904  return;
1905  }
1906  }
1907 
1908  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1909  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1910  ValueList Operands;
1911  // Prepare the operand vector.
1912  for (Value *j : VL) {
1913  CallInst *CI2 = dyn_cast<CallInst>(j);
1914  Operands.push_back(CI2->getArgOperand(i));
1915  }
1916  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1917  }
1918  return;
1919  }
1920  case Instruction::ShuffleVector:
1921  // If this is not an alternate sequence of opcode like add-sub
1922  // then do not vectorize this instruction.
1923  if (!S.isAltShuffle()) {
1924  BS.cancelScheduling(VL, VL0);
1925  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1926  LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1927  return;
1928  }
1929  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1930  LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1931 
1932  // Reorder operands if reordering would enable vectorization.
1933  if (isa<BinaryOperator>(VL0)) {
1934  ValueList Left, Right;
1935  reorderAltShuffleOperands(S, VL, Left, Right);
1936  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1937  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1938  return;
1939  }
1940 
1941  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1942  ValueList Operands;
1943  // Prepare the operand vector.
1944  for (Value *j : VL)
1945  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1946 
1947  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1948  }
1949  return;
1950 
1951  default:
1952  BS.cancelScheduling(VL, VL0);
1953  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1954  LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1955  return;
1956  }
1957 }
1958 
1959 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1960  unsigned N;
1961  Type *EltTy;
1962  auto *ST = dyn_cast<StructType>(T);
1963  if (ST) {
1964  N = ST->getNumElements();
1965  EltTy = *ST->element_begin();
1966  } else {
1967  N = cast<ArrayType>(T)->getNumElements();
1968  EltTy = cast<ArrayType>(T)->getElementType();
1969  }
1970  if (!isValidElementType(EltTy))
1971  return 0;
1972  uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1973  if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1974  return 0;
1975  if (ST) {
1976  // Check that struct is homogeneous.
1977  for (const auto *Ty : ST->elements())
1978  if (Ty != EltTy)
1979  return 0;
1980  }
1981  return N;
1982 }
1983 
1984 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1985  SmallVectorImpl<unsigned> &CurrentOrder) const {
1986  Instruction *E0 = cast<Instruction>(OpValue);
1987  assert(E0->getOpcode() == Instruction::ExtractElement ||
1988  E0->getOpcode() == Instruction::ExtractValue);
1989  assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode");
1990  // Check if all of the extracts come from the same vector and from the
1991  // correct offset.
1992  Value *Vec = E0->getOperand(0);
1993 
1994  CurrentOrder.clear();
1995 
1996  // We have to extract from a vector/aggregate with the same number of elements.
1997  unsigned NElts;
1998  if (E0->getOpcode() == Instruction::ExtractValue) {
1999  const DataLayout &DL = E0->getModule()->getDataLayout();
2000  NElts = canMapToVector(Vec->getType(), DL);
2001  if (!NElts)
2002  return false;
2003  // Check if load can be rewritten as load of vector.
2004  LoadInst *LI = dyn_cast<LoadInst>(Vec);
2005  if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2006  return false;
2007  } else {
2008  NElts = Vec->getType()->getVectorNumElements();
2009  }
2010 
2011  if (NElts != VL.size())
2012  return false;
2013 
2014  // Check that all of the indices extract from the correct offset.
2015  bool ShouldKeepOrder = true;
2016  unsigned E = VL.size();
2017  // Assign to all items the initial value E + 1 so we can check if the extract
2018  // instruction index was used already.
2019  // Also, later we can check that all the indices are used and we have a
2020  // consecutive access in the extract instructions, by checking that no
2021  // element of CurrentOrder still has value E + 1.
2022  CurrentOrder.assign(E, E + 1);
2023  unsigned I = 0;
2024  for (; I < E; ++I) {
2025  auto *Inst = cast<Instruction>(VL[I]);
2026  if (Inst->getOperand(0) != Vec)
2027  break;
2028  Optional<unsigned> Idx = getExtractIndex(Inst);
2029  if (!Idx)
2030  break;
2031  const unsigned ExtIdx = *Idx;
2032  if (ExtIdx != I) {
2033  if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
2034  break;
2035  ShouldKeepOrder = false;
2036  CurrentOrder[ExtIdx] = I;
2037  } else {
2038  if (CurrentOrder[I] != E + 1)
2039  break;
2040  CurrentOrder[I] = I;
2041  }
2042  }
2043  if (I < E) {
2044  CurrentOrder.clear();
2045  return false;
2046  }
2047 
2048  return ShouldKeepOrder;
2049 }
2050 
2051 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2052  return I->hasOneUse() ||
2053  std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2054  return ScalarToTreeEntry.count(U) > 0;
2055  });
2056 }
2057 
2058 int BoUpSLP::getEntryCost(TreeEntry *E) {
2059  ArrayRef<Value*> VL = E->Scalars;
2060 
2061  Type *ScalarTy = VL[0]->getType();
2062  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2063  ScalarTy = SI->getValueOperand()->getType();
2064  else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2065  ScalarTy = CI->getOperand(0)->getType();
2066  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2067 
2068  // If we have computed a smaller type for the expression, update VecTy so
2069  // that the costs will be accurate.
2070  if (MinBWs.count(VL[0]))
2071  VecTy = VectorType::get(
2072  IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2073 
2074  unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2075  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2076  int ReuseShuffleCost = 0;
2077  if (NeedToShuffleReuses) {
2078  ReuseShuffleCost =
2079  TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2080  }
2081  if (E->NeedToGather) {
2082  if (allConstant(VL))
2083  return 0;
2084  if (isSplat(VL)) {
2085  return ReuseShuffleCost +
2086  TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2087  }
2088  if (getSameOpcode(VL).getOpcode() == Instruction::ExtractElement &&
2089  allSameType(VL) && allSameBlock(VL)) {
2091  if (ShuffleKind.hasValue()) {
2092  int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2093  for (auto *V : VL) {
2094  // If all users of instruction are going to be vectorized and this
2095  // instruction itself is not going to be vectorized, consider this
2096  // instruction as dead and remove its cost from the final cost of the
2097  // vectorized tree.
2098  if (areAllUsersVectorized(cast<Instruction>(V)) &&
2099  !ScalarToTreeEntry.count(V)) {
2100  auto *IO = cast<ConstantInt>(
2101  cast<ExtractElementInst>(V)->getIndexOperand());
2102  Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2103  IO->getZExtValue());
2104  }
2105  }
2106  return ReuseShuffleCost + Cost;
2107  }
2108  }
2109  return ReuseShuffleCost + getGatherCost(VL);
2110  }
2111  InstructionsState S = getSameOpcode(VL);
2112  assert(S.getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2113  Instruction *VL0 = cast<Instruction>(S.OpValue);
2114  unsigned ShuffleOrOp = S.isAltShuffle() ?
2115  (unsigned) Instruction::ShuffleVector : S.getOpcode();
2116  switch (ShuffleOrOp) {
2117  case Instruction::PHI:
2118  return 0;
2119 
2120  case Instruction::ExtractValue:
2121  case Instruction::ExtractElement:
2122  if (NeedToShuffleReuses) {
2123  unsigned Idx = 0;
2124  for (unsigned I : E->ReuseShuffleIndices) {
2125  if (ShuffleOrOp == Instruction::ExtractElement) {
2126  auto *IO = cast<ConstantInt>(
2127  cast<ExtractElementInst>(VL[I])->getIndexOperand());
2128  Idx = IO->getZExtValue();
2129  ReuseShuffleCost -= TTI->getVectorInstrCost(
2130  Instruction::ExtractElement, VecTy, Idx);
2131  } else {
2132  ReuseShuffleCost -= TTI->getVectorInstrCost(
2133  Instruction::ExtractElement, VecTy, Idx);
2134  ++Idx;
2135  }
2136  }
2137  Idx = ReuseShuffleNumbers;
2138  for (Value *V : VL) {
2139  if (ShuffleOrOp == Instruction::ExtractElement) {
2140  auto *IO = cast<ConstantInt>(
2141  cast<ExtractElementInst>(V)->getIndexOperand());
2142  Idx = IO->getZExtValue();
2143  } else {
2144  --Idx;
2145  }
2146  ReuseShuffleCost +=
2147  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2148  }
2149  }
2150  if (!E->NeedToGather) {
2151  int DeadCost = ReuseShuffleCost;
2152  if (!E->ReorderIndices.empty()) {
2153  // TODO: Merge this shuffle with the ReuseShuffleCost.
2154  DeadCost += TTI->getShuffleCost(
2156  }
2157  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2158  Instruction *E = cast<Instruction>(VL[i]);
2159  // If all users are going to be vectorized, instruction can be
2160  // considered as dead.
2161  // The same, if have only one user, it will be vectorized for sure.
2162  if (areAllUsersVectorized(E)) {
2163  // Take credit for instruction that will become dead.
2164  if (E->hasOneUse()) {
2165  Instruction *Ext = E->user_back();
2166  if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2167  all_of(Ext->users(),
2168  [](User *U) { return isa<GetElementPtrInst>(U); })) {
2169  // Use getExtractWithExtendCost() to calculate the cost of
2170  // extractelement/ext pair.
2171  DeadCost -= TTI->getExtractWithExtendCost(
2172  Ext->getOpcode(), Ext->getType(), VecTy, i);
2173  // Add back the cost of s|zext which is subtracted separately.
2174  DeadCost += TTI->getCastInstrCost(
2175  Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
2176  continue;
2177  }
2178  }
2179  DeadCost -=
2180  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2181  }
2182  }
2183  return DeadCost;
2184  }
2185  return ReuseShuffleCost + getGatherCost(VL);
2186 
2187  case Instruction::ZExt:
2188  case Instruction::SExt:
2189  case Instruction::FPToUI:
2190  case Instruction::FPToSI:
2191  case Instruction::FPExt:
2192  case Instruction::PtrToInt:
2193  case Instruction::IntToPtr:
2194  case Instruction::SIToFP:
2195  case Instruction::UIToFP:
2196  case Instruction::Trunc:
2197  case Instruction::FPTrunc:
2198  case Instruction::BitCast: {
2199  Type *SrcTy = VL0->getOperand(0)->getType();
2200  int ScalarEltCost =
2201  TTI->getCastInstrCost(S.getOpcode(), ScalarTy, SrcTy, VL0);
2202  if (NeedToShuffleReuses) {
2203  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2204  }
2205 
2206  // Calculate the cost of this instruction.
2207  int ScalarCost = VL.size() * ScalarEltCost;
2208 
2209  VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2210  int VecCost = 0;
2211  // Check if the values are candidates to demote.
2212  if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2213  VecCost = ReuseShuffleCost +
2214  TTI->getCastInstrCost(S.getOpcode(), VecTy, SrcVecTy, VL0);
2215  }
2216  return VecCost - ScalarCost;
2217  }
2218  case Instruction::FCmp:
2219  case Instruction::ICmp:
2220  case Instruction::Select: {
2221  // Calculate the cost of this instruction.
2222  int ScalarEltCost = TTI->getCmpSelInstrCost(S.getOpcode(), ScalarTy,
2223  Builder.getInt1Ty(), VL0);
2224  if (NeedToShuffleReuses) {
2225  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2226  }
2227  VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2228  int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2229  int VecCost = TTI->getCmpSelInstrCost(S.getOpcode(), VecTy, MaskTy, VL0);
2230  return ReuseShuffleCost + VecCost - ScalarCost;
2231  }
2232  case Instruction::Add:
2233  case Instruction::FAdd:
2234  case Instruction::Sub:
2235  case Instruction::FSub:
2236  case Instruction::Mul:
2237  case Instruction::FMul:
2238  case Instruction::UDiv:
2239  case Instruction::SDiv:
2240  case Instruction::FDiv:
2241  case Instruction::URem:
2242  case Instruction::SRem:
2243  case Instruction::FRem:
2244  case Instruction::Shl:
2245  case Instruction::LShr:
2246  case Instruction::AShr:
2247  case Instruction::And:
2248  case Instruction::Or:
2249  case Instruction::Xor: {
2250  // Certain instructions can be cheaper to vectorize if they have a
2251  // constant second vector operand.
2260 
2261  // If all operands are exactly the same ConstantInt then set the
2262  // operand kind to OK_UniformConstantValue.
2263  // If instead not all operands are constants, then set the operand kind
2264  // to OK_AnyValue. If all operands are constants but not the same,
2265  // then set the operand kind to OK_NonUniformConstantValue.
2266  ConstantInt *CInt0 = nullptr;
2267  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2268  const Instruction *I = cast<Instruction>(VL[i]);
2269  ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(1));
2270  if (!CInt) {
2273  break;
2274  }
2275  if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
2276  !CInt->getValue().isPowerOf2())
2278  if (i == 0) {
2279  CInt0 = CInt;
2280  continue;
2281  }
2282  if (CInt0 != CInt)
2284  }
2285 
2286  SmallVector<const Value *, 4> Operands(VL0->operand_values());
2287  int ScalarEltCost = TTI->getArithmeticInstrCost(
2288  S.getOpcode(), ScalarTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands);
2289  if (NeedToShuffleReuses) {
2290  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2291  }
2292  int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2293  int VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy, Op1VK,
2294  Op2VK, Op1VP, Op2VP, Operands);
2295  return ReuseShuffleCost + VecCost - ScalarCost;
2296  }
2297  case Instruction::GetElementPtr: {
2302 
2303  int ScalarEltCost =
2304  TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2305  if (NeedToShuffleReuses) {
2306  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2307  }
2308  int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2309  int VecCost =
2310  TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2311  return ReuseShuffleCost + VecCost - ScalarCost;
2312  }
2313  case Instruction::Load: {
2314  // Cost of wide load - cost of scalar loads.
2315  unsigned alignment = cast<LoadInst>(VL0)->getAlignment();
2316  int ScalarEltCost =
2317  TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2318  if (NeedToShuffleReuses) {
2319  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2320  }
2321  int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
2322  int VecLdCost =
2323  TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, VL0);
2324  if (!E->ReorderIndices.empty()) {
2325  // TODO: Merge this shuffle with the ReuseShuffleCost.
2326  VecLdCost += TTI->getShuffleCost(
2328  }
2329  return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2330  }
2331  case Instruction::Store: {
2332  // We know that we can merge the stores. Calculate the cost.
2333  unsigned alignment = cast<StoreInst>(VL0)->getAlignment();
2334  int ScalarEltCost =
2335  TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2336  if (NeedToShuffleReuses) {
2337  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2338  }
2339  int ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
2340  int VecStCost =
2341  TTI->getMemoryOpCost(Instruction::Store, VecTy, alignment, 0, VL0);
2342  return ReuseShuffleCost + VecStCost - ScalarStCost;
2343  }
2344  case Instruction::Call: {
2345  CallInst *CI = cast<CallInst>(VL0);
2347 
2348  // Calculate the cost of the scalar and vector calls.
2349  SmallVector<Type *, 4> ScalarTys;
2350  for (unsigned op = 0, opc = CI->getNumArgOperands(); op != opc; ++op)
2351  ScalarTys.push_back(CI->getArgOperand(op)->getType());
2352 
2353  FastMathFlags FMF;
2354  if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2355  FMF = FPMO->getFastMathFlags();
2356 
2357  int ScalarEltCost =
2358  TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2359  if (NeedToShuffleReuses) {
2360  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2361  }
2362  int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
2363 
2365  int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2366  VecTy->getNumElements());
2367 
2368  LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCost
2369  << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2370  << " for " << *CI << "\n");
2371 
2372  return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2373  }
2374  case Instruction::ShuffleVector: {
2375  assert(S.isAltShuffle() &&
2376  ((Instruction::isBinaryOp(S.getOpcode()) &&
2377  Instruction::isBinaryOp(S.getAltOpcode())) ||
2378  (Instruction::isCast(S.getOpcode()) &&
2379  Instruction::isCast(S.getAltOpcode()))) &&
2380  "Invalid Shuffle Vector Operand");
2381  int ScalarCost = 0;
2382  if (NeedToShuffleReuses) {
2383  for (unsigned Idx : E->ReuseShuffleIndices) {
2384  Instruction *I = cast<Instruction>(VL[Idx]);
2385  ReuseShuffleCost -= TTI->getInstructionCost(
2387  }
2388  for (Value *V : VL) {
2389  Instruction *I = cast<Instruction>(V);
2390  ReuseShuffleCost += TTI->getInstructionCost(
2392  }
2393  }
2394  for (Value *i : VL) {
2395  Instruction *I = cast<Instruction>(i);
2396  assert(S.isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
2397  ScalarCost += TTI->getInstructionCost(
2399  }
2400  // VecCost is equal to sum of the cost of creating 2 vectors
2401  // and the cost of creating shuffle.
2402  int VecCost = 0;
2403  if (Instruction::isBinaryOp(S.getOpcode())) {
2404  VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy);
2405  VecCost += TTI->getArithmeticInstrCost(S.getAltOpcode(), VecTy);
2406  } else {
2407  Type *Src0SclTy = S.MainOp->getOperand(0)->getType();
2408  Type *Src1SclTy = S.AltOp->getOperand(0)->getType();
2409  VectorType *Src0Ty = VectorType::get(Src0SclTy, VL.size());
2410  VectorType *Src1Ty = VectorType::get(Src1SclTy, VL.size());
2411  VecCost = TTI->getCastInstrCost(S.getOpcode(), VecTy, Src0Ty);
2412  VecCost += TTI->getCastInstrCost(S.getAltOpcode(), VecTy, Src1Ty);
2413  }
2414  VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
2415  return ReuseShuffleCost + VecCost - ScalarCost;
2416  }
2417  default:
2418  llvm_unreachable("Unknown instruction");
2419  }
2420 }
2421 
2422 bool BoUpSLP::isFullyVectorizableTinyTree() {
2423  LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "
2424  << VectorizableTree.size() << " is fully vectorizable .\n");
2425 
2426  // We only handle trees of heights 1 and 2.
2427  if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2428  return true;
2429 
2430  if (VectorizableTree.size() != 2)
2431  return false;
2432 
2433  // Handle splat and all-constants stores.
2434  if (!VectorizableTree[0].NeedToGather &&
2435  (allConstant(VectorizableTree[1].Scalars) ||
2436  isSplat(VectorizableTree[1].Scalars)))
2437  return true;
2438 
2439  // Gathering cost would be too much for tiny trees.
2440  if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2441  return false;
2442 
2443  return true;
2444 }
2445 
2446 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2447  // We can vectorize the tree if its size is greater than or equal to the
2448  // minimum size specified by the MinTreeSize command line option.
2449  if (VectorizableTree.size() >= MinTreeSize)
2450  return false;
2451 
2452  // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2453  // can vectorize it if we can prove it fully vectorizable.
2454  if (isFullyVectorizableTinyTree())
2455  return false;
2456 
2457  assert(VectorizableTree.empty()
2458  ? ExternalUses.empty()
2459  : true && "We shouldn't have any external users");
2460 
2461  // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2462  // vectorizable.
2463  return true;
2464 }
2465 
2466 int BoUpSLP::getSpillCost() {
2467  // Walk from the bottom of the tree to the top, tracking which values are
2468  // live. When we see a call instruction that is not part of our tree,
2469  // query TTI to see if there is a cost to keeping values live over it
2470  // (for example, if spills and fills are required).
2471  unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2472  int Cost = 0;
2473 
2474  SmallPtrSet<Instruction*, 4> LiveValues;
2475  Instruction *PrevInst = nullptr;
2476 
2477  for (const auto &N : VectorizableTree) {
2478  Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2479  if (!Inst)
2480  continue;
2481 
2482  if (!PrevInst) {
2483  PrevInst = Inst;
2484  continue;
2485  }
2486 
2487  // Update LiveValues.
2488  LiveValues.erase(PrevInst);
2489  for (auto &J : PrevInst->operands()) {
2490  if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2491  LiveValues.insert(cast<Instruction>(&*J));
2492  }
2493 
2494  LLVM_DEBUG({
2495  dbgs() << "SLP: #LV: " << LiveValues.size();
2496  for (auto *X : LiveValues)
2497  dbgs() << " " << X->getName();
2498  dbgs() << ", Looking at ";
2499  Inst->dump();
2500  });
2501 
2502  // Now find the sequence of instructions between PrevInst and Inst.
2504  PrevInstIt =
2505  PrevInst->getIterator().getReverse();
2506  while (InstIt != PrevInstIt) {
2507  if (PrevInstIt == PrevInst->getParent()->rend()) {
2508  PrevInstIt = Inst->getParent()->rbegin();
2509  continue;
2510  }
2511 
2512  // Debug informations don't impact spill cost.
2513  if ((isa<CallInst>(&*PrevInstIt) &&
2514  !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
2515  &*PrevInstIt != PrevInst) {
2517  for (auto *II : LiveValues)
2518  V.push_back(VectorType::get(II->getType(), BundleWidth));
2519  Cost += TTI->getCostOfKeepingLiveOverCall(V);
2520  }
2521 
2522  ++PrevInstIt;
2523  }
2524 
2525  PrevInst = Inst;
2526  }
2527 
2528  return Cost;
2529 }
2530 
2531 int BoUpSLP::getTreeCost() {
2532  int Cost = 0;
2533  LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
2534  << VectorizableTree.size() << ".\n");
2535 
2536  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2537 
2538  for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
2539  TreeEntry &TE = VectorizableTree[I];
2540 
2541  // We create duplicate tree entries for gather sequences that have multiple
2542  // uses. However, we should not compute the cost of duplicate sequences.
2543  // For example, if we have a build vector (i.e., insertelement sequence)
2544  // that is used by more than one vector instruction, we only need to
2545  // compute the cost of the insertelement instructions once. The redundant
2546  // instructions will be eliminated by CSE.
2547  //
2548  // We should consider not creating duplicate tree entries for gather
2549  // sequences, and instead add additional edges to the tree representing
2550  // their uses. Since such an approach results in fewer total entries,
2551  // existing heuristics based on tree size may yield different results.
2552  //
2553  if (TE.NeedToGather &&
2554  std::any_of(std::next(VectorizableTree.begin(), I + 1),
2555  VectorizableTree.end(), [TE](TreeEntry &Entry) {
2556  return Entry.NeedToGather && Entry.isSame(TE.Scalars);
2557  }))
2558  continue;
2559 
2560  int C = getEntryCost(&TE);
2561  LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
2562  << " for bundle that starts with " << *TE.Scalars[0]
2563  << ".\n");
2564  Cost += C;
2565  }
2566 
2567  SmallPtrSet<Value *, 16> ExtractCostCalculated;
2568  int ExtractCost = 0;
2569  for (ExternalUser &EU : ExternalUses) {
2570  // We only add extract cost once for the same scalar.
2571  if (!ExtractCostCalculated.insert(EU.Scalar).second)
2572  continue;
2573 
2574  // Uses by ephemeral values are free (because the ephemeral value will be
2575  // removed prior to code generation, and so the extraction will be
2576  // removed as well).
2577  if (EphValues.count(EU.User))
2578  continue;
2579 
2580  // If we plan to rewrite the tree in a smaller type, we will need to sign
2581  // extend the extracted value back to the original type. Here, we account
2582  // for the extract and the added cost of the sign extend if needed.
2583  auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2584  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2585  if (MinBWs.count(ScalarRoot)) {
2586  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2587  auto Extend =
2588  MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2589  VecTy = VectorType::get(MinTy, BundleWidth);
2590  ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2591  VecTy, EU.Lane);
2592  } else {
2593  ExtractCost +=
2594  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2595  }
2596  }
2597 
2598  int SpillCost = getSpillCost();
2599  Cost += SpillCost + ExtractCost;
2600 
2601  std::string Str;
2602  {
2603  raw_string_ostream OS(Str);
2604  OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2605  << "SLP: Extract Cost = " << ExtractCost << ".\n"
2606  << "SLP: Total Cost = " << Cost << ".\n";
2607  }
2608  LLVM_DEBUG(dbgs() << Str);
2609 
2610  if (ViewSLPTree)
2611  ViewGraph(this, "SLP" + F->getName(), false, Str);
2612 
2613  return Cost;
2614 }
2615 
2616 int BoUpSLP::getGatherCost(Type *Ty,
2617  const DenseSet<unsigned> &ShuffledIndices) {
2618  int Cost = 0;
2619  for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2620  if (!ShuffledIndices.count(i))
2621  Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2622  if (!ShuffledIndices.empty())
2623  Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2624  return Cost;
2625 }
2626 
2627 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2628  // Find the type of the operands in VL.
2629  Type *ScalarTy = VL[0]->getType();
2630  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2631  ScalarTy = SI->getValueOperand()->getType();
2632  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2633  // Find the cost of inserting/extracting values from the vector.
2634  // Check if the same elements are inserted several times and count them as
2635  // shuffle candidates.
2636  DenseSet<unsigned> ShuffledElements;
2637  DenseSet<Value *> UniqueElements;
2638  // Iterate in reverse order to consider insert elements with the high cost.
2639  for (unsigned I = VL.size(); I > 0; --I) {
2640  unsigned Idx = I - 1;
2641  if (!UniqueElements.insert(VL[Idx]).second)
2642  ShuffledElements.insert(Idx);
2643  }
2644  return getGatherCost(VecTy, ShuffledElements);
2645 }
2646 
2647 // Reorder commutative operations in alternate shuffle if the resulting vectors
2648 // are consecutive loads. This would allow us to vectorize the tree.
2649 // If we have something like-
2650 // load a[0] - load b[0]
2651 // load b[1] + load a[1]
2652 // load a[2] - load b[2]
2653 // load a[3] + load b[3]
2654 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2655 // code.
2656 void BoUpSLP::reorderAltShuffleOperands(const InstructionsState &S,
2657  ArrayRef<Value *> VL,
2660  // Push left and right operands of binary operation into Left and Right
2661  for (Value *V : VL) {
2662  auto *I = cast<Instruction>(V);
2663  assert(S.isOpcodeOrAlt(I) && "Incorrect instruction in vector");
2664  Left.push_back(I->getOperand(0));
2665  Right.push_back(I->getOperand(1));
2666  }
2667 
2668  // Reorder if we have a commutative operation and consecutive access
2669  // are on either side of the alternate instructions.
2670  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2671  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2672  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2673  Instruction *VL1 = cast<Instruction>(VL[j]);
2674  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2675  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2676  std::swap(Left[j], Right[j]);
2677  continue;
2678  } else if (VL2->isCommutative() &&
2679  isConsecutiveAccess(L, L1, *DL, *SE)) {
2680  std::swap(Left[j + 1], Right[j + 1]);
2681  continue;
2682  }
2683  // else unchanged
2684  }
2685  }
2686  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2687  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2688  Instruction *VL1 = cast<Instruction>(VL[j]);
2689  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2690  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2691  std::swap(Left[j], Right[j]);
2692  continue;
2693  } else if (VL2->isCommutative() &&
2694  isConsecutiveAccess(L, L1, *DL, *SE)) {
2695  std::swap(Left[j + 1], Right[j + 1]);
2696  continue;
2697  }
2698  // else unchanged
2699  }
2700  }
2701  }
2702 }
2703 
2704 // Return true if I should be commuted before adding it's left and right
2705 // operands to the arrays Left and Right.
2706 //
2707 // The vectorizer is trying to either have all elements one side being
2708 // instruction with the same opcode to enable further vectorization, or having
2709 // a splat to lower the vectorizing cost.
2711  int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2712  ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2713  bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2714  VLeft = I.getOperand(0);
2715  VRight = I.getOperand(1);
2716  // If we have "SplatRight", try to see if commuting is needed to preserve it.
2717  if (SplatRight) {
2718  if (VRight == Right[i - 1])
2719  // Preserve SplatRight
2720  return false;
2721  if (VLeft == Right[i - 1]) {
2722  // Commuting would preserve SplatRight, but we don't want to break
2723  // SplatLeft either, i.e. preserve the original order if possible.
2724  // (FIXME: why do we care?)
2725  if (SplatLeft && VLeft == Left[i - 1])
2726  return false;
2727  return true;
2728  }
2729  }
2730  // Symmetrically handle Right side.
2731  if (SplatLeft) {
2732  if (VLeft == Left[i - 1])
2733  // Preserve SplatLeft
2734  return false;
2735  if (VRight == Left[i - 1])
2736  return true;
2737  }
2738 
2739  Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2740  Instruction *IRight = dyn_cast<Instruction>(VRight);
2741 
2742  // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2743  // it and not the right, in this case we want to commute.
2744  if (AllSameOpcodeRight) {
2745  unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2746  if (IRight && RightPrevOpcode == IRight->getOpcode())
2747  // Do not commute, a match on the right preserves AllSameOpcodeRight
2748  return false;
2749  if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2750  // We have a match and may want to commute, but first check if there is
2751  // not also a match on the existing operands on the Left to preserve
2752  // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2753  // (FIXME: why do we care?)
2754  if (AllSameOpcodeLeft && ILeft &&
2755  cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2756  return false;
2757  return true;
2758  }
2759  }
2760  // Symmetrically handle Left side.
2761  if (AllSameOpcodeLeft) {
2762  unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2763  if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2764  return false;
2765  if (IRight && LeftPrevOpcode == IRight->getOpcode())
2766  return true;
2767  }
2768  return false;
2769 }
2770 
2771 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2772  ArrayRef<Value *> VL,
2774  SmallVectorImpl<Value *> &Right) {
2775  if (!VL.empty()) {
2776  // Peel the first iteration out of the loop since there's nothing
2777  // interesting to do anyway and it simplifies the checks in the loop.
2778  auto *I = cast<Instruction>(VL[0]);
2779  Value *VLeft = I->getOperand(0);
2780  Value *VRight = I->getOperand(1);
2781  if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2782  // Favor having instruction to the right. FIXME: why?
2783  std::swap(VLeft, VRight);
2784  Left.push_back(VLeft);
2785  Right.push_back(VRight);
2786  }
2787 
2788  // Keep track if we have instructions with all the same opcode on one side.
2789  bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2790  bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2791  // Keep track if we have one side with all the same value (broadcast).
2792  bool SplatLeft = true;
2793  bool SplatRight = true;
2794 
2795  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2796  Instruction *I = cast<Instruction>(VL[i]);
2797  assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2798  (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2799  "Can only process commutative instruction");
2800  // Commute to favor either a splat or maximizing having the same opcodes on
2801  // one side.
2802  Value *VLeft;
2803  Value *VRight;
2804  if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2805  AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2806  VRight)) {
2807  Left.push_back(VRight);
2808  Right.push_back(VLeft);
2809  } else {
2810  Left.push_back(VLeft);
2811  Right.push_back(VRight);
2812  }
2813  // Update Splat* and AllSameOpcode* after the insertion.
2814  SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2815  SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2816  AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2817  (cast<Instruction>(Left[i - 1])->getOpcode() ==
2818  cast<Instruction>(Left[i])->getOpcode());
2819  AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2820  (cast<Instruction>(Right[i - 1])->getOpcode() ==
2821  cast<Instruction>(Right[i])->getOpcode());
2822  }
2823 
2824  // If one operand end up being broadcast, return this operand order.
2825  if (SplatRight || SplatLeft)
2826  return;
2827 
2828  // Finally check if we can get longer vectorizable chain by reordering
2829  // without breaking the good operand order detected above.
2830  // E.g. If we have something like-
2831  // load a[0] load b[0]
2832  // load b[1] load a[1]
2833  // load a[2] load b[2]
2834  // load a[3] load b[3]
2835  // Reordering the second load b[1] load a[1] would allow us to vectorize
2836  // this code and we still retain AllSameOpcode property.
2837  // FIXME: This load reordering might break AllSameOpcode in some rare cases
2838  // such as-
2839  // add a[0],c[0] load b[0]
2840  // add a[1],c[2] load b[1]
2841  // b[2] load b[2]
2842  // add a[3],c[3] load b[3]
2843  for (unsigned j = 0, e = VL.size() - 1; j < e; ++j) {
2844  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2845  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2846  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2847  std::swap(Left[j + 1], Right[j + 1]);
2848  continue;
2849  }
2850  }
2851  }
2852  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2853  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2854  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2855  std::swap(Left[j + 1], Right[j + 1]);
2856  continue;
2857  }
2858  }
2859  }
2860  // else unchanged
2861  }
2862 }
2863 
2864 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL,
2865  const InstructionsState &S) {
2866  // Get the basic block this bundle is in. All instructions in the bundle
2867  // should be in this block.
2868  auto *Front = cast<Instruction>(S.OpValue);
2869  auto *BB = Front->getParent();
2870  assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2871  auto *I = cast<Instruction>(V);
2872  return !S.isOpcodeOrAlt(I) || I->getParent() == BB;
2873  }));
2874 
2875  // The last instruction in the bundle in program order.
2876  Instruction *LastInst = nullptr;
2877 
2878  // Find the last instruction. The common case should be that BB has been
2879  // scheduled, and the last instruction is VL.back(). So we start with
2880  // VL.back() and iterate over schedule data until we reach the end of the
2881  // bundle. The end of the bundle is marked by null ScheduleData.
2882  if (BlocksSchedules.count(BB)) {
2883  auto *Bundle =
2884  BlocksSchedules[BB]->getScheduleData(isOneOf(S, VL.back()));
2885  if (Bundle && Bundle->isPartOfBundle())
2886  for (; Bundle; Bundle = Bundle->NextInBundle)
2887  if (Bundle->OpValue == Bundle->Inst)
2888  LastInst = Bundle->Inst;
2889  }
2890 
2891  // LastInst can still be null at this point if there's either not an entry
2892  // for BB in BlocksSchedules or there's no ScheduleData available for
2893  // VL.back(). This can be the case if buildTree_rec aborts for various
2894  // reasons (e.g., the maximum recursion depth is reached, the maximum region
2895  // size is reached, etc.). ScheduleData is initialized in the scheduling
2896  // "dry-run".
2897  //
2898  // If this happens, we can still find the last instruction by brute force. We
2899  // iterate forwards from Front (inclusive) until we either see all
2900  // instructions in the bundle or reach the end of the block. If Front is the
2901  // last instruction in program order, LastInst will be set to Front, and we
2902  // will visit all the remaining instructions in the block.
2903  //
2904  // One of the reasons we exit early from buildTree_rec is to place an upper
2905  // bound on compile-time. Thus, taking an additional compile-time hit here is
2906  // not ideal. However, this should be exceedingly rare since it requires that
2907  // we both exit early from buildTree_rec and that the bundle be out-of-order
2908  // (causing us to iterate all the way to the end of the block).
2909  if (!LastInst) {
2910  SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2911  for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2912  if (Bundle.erase(&I) && S.isOpcodeOrAlt(&I))
2913  LastInst = &I;
2914  if (Bundle.empty())
2915  break;
2916  }
2917  }
2918 
2919  // Set the insertion point after the last instruction in the bundle. Set the
2920  // debug location to Front.
2921  Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2922  Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2923 }
2924 
2925 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2926  Value *Vec = UndefValue::get(Ty);
2927  // Generate the 'InsertElement' instruction.
2928  for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2929  Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2930  if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2931  GatherSeq.insert(Insrt);
2932  CSEBlocks.insert(Insrt->getParent());
2933 
2934  // Add to our 'need-to-extract' list.
2935  if (TreeEntry *E = getTreeEntry(VL[i])) {
2936  // Find which lane we need to extract.
2937  int FoundLane = -1;
2938  for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2939  // Is this the lane of the scalar that we are looking for ?
2940  if (E->Scalars[Lane] == VL[i]) {
2941  FoundLane = Lane;
2942  break;
2943  }
2944  }
2945  assert(FoundLane >= 0 && "Could not find the correct lane");
2946  if (!E->ReuseShuffleIndices.empty()) {
2947  FoundLane =
2948  std::distance(E->ReuseShuffleIndices.begin(),
2949  llvm::find(E->ReuseShuffleIndices, FoundLane));
2950  }
2951  ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2952  }
2953  }
2954  }
2955 
2956  return Vec;
2957 }
2958 
2959 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2960  InstructionsState S = getSameOpcode(VL);
2961  if (S.getOpcode()) {
2962  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2963  if (E->isSame(VL)) {
2964  Value *V = vectorizeTree(E);
2965  if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
2966  // We need to get the vectorized value but without shuffle.
2967  if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
2968  V = SV->getOperand(0);
2969  } else {
2970  // Reshuffle to get only unique values.
2971  SmallVector<unsigned, 4> UniqueIdxs;
2972  SmallSet<unsigned, 4> UsedIdxs;
2973  for(unsigned Idx : E->ReuseShuffleIndices)
2974  if (UsedIdxs.insert(Idx).second)
2975  UniqueIdxs.emplace_back(Idx);
2976  V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
2977  UniqueIdxs);
2978  }
2979  }
2980  return V;
2981  }
2982  }
2983  }
2984 
2985  Type *ScalarTy = S.OpValue->getType();
2986  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2987  ScalarTy = SI->getValueOperand()->getType();
2988 
2989  // Check that every instruction appears once in this bundle.
2990  SmallVector<unsigned, 4> ReuseShuffleIndicies;
2991  SmallVector<Value *, 4> UniqueValues;
2992  if (VL.size() > 2) {
2993  DenseMap<Value *, unsigned> UniquePositions;
2994  for (Value *V : VL) {
2995  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2996  ReuseShuffleIndicies.emplace_back(Res.first->second);
2997  if (Res.second || isa<Constant>(V))
2998  UniqueValues.emplace_back(V);
2999  }
3000  // Do not shuffle single element or if number of unique values is not power
3001  // of 2.
3002  if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
3003  !llvm::isPowerOf2_32(UniqueValues.size()))
3004  ReuseShuffleIndicies.clear();
3005  else
3006  VL = UniqueValues;
3007  }
3008  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3009 
3010  Value *V = Gather(VL, VecTy);
3011  if (!ReuseShuffleIndicies.empty()) {
3012  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3013  ReuseShuffleIndicies, "shuffle");
3014  if (auto *I = dyn_cast<Instruction>(V)) {
3015  GatherSeq.insert(I);
3016  CSEBlocks.insert(I->getParent());
3017  }
3018  }
3019  return V;
3020 }
3021 
3024  Mask.clear();
3025  const unsigned E = Indices.size();
3026  Mask.resize(E);
3027  for (unsigned I = 0; I < E; ++I)
3028  Mask[Indices[I]] = I;
3029 }
3030 
3031 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
3032  IRBuilder<>::InsertPointGuard Guard(Builder);
3033 
3034  if (E->VectorizedValue) {
3035  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
3036  return E->VectorizedValue;
3037  }
3038 
3039  InstructionsState S = getSameOpcode(E->Scalars);
3040  Instruction *VL0 = cast<Instruction>(S.OpValue);
3041  Type *ScalarTy = VL0->getType();
3042  if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3043  ScalarTy = SI->getValueOperand()->getType();
3044  VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
3045 
3046  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3047 
3048  if (E->NeedToGather) {
3049  setInsertPointAfterBundle(E->Scalars, S);
3050  auto *V = Gather(E->Scalars, VecTy);
3051  if (NeedToShuffleReuses) {
3052  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3053  E->ReuseShuffleIndices, "shuffle");
3054  if (auto *I = dyn_cast<Instruction>(V)) {
3055  GatherSeq.insert(I);
3056  CSEBlocks.insert(I->getParent());
3057  }
3058  }
3059  E->VectorizedValue = V;
3060  return V;
3061  }
3062 
3063  unsigned ShuffleOrOp = S.isAltShuffle() ?
3064  (unsigned) Instruction::ShuffleVector : S.getOpcode();
3065  switch (ShuffleOrOp) {
3066  case Instruction::PHI: {
3067  PHINode *PH = dyn_cast<PHINode>(VL0);
3068  Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
3069  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3070  PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
3071  Value *V = NewPhi;
3072  if (NeedToShuffleReuses) {
3073  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3074  E->ReuseShuffleIndices, "shuffle");
3075  }
3076  E->VectorizedValue = V;
3077 
3078  // PHINodes may have multiple entries from the same block. We want to
3079  // visit every block once.
3080  SmallPtrSet<BasicBlock*, 4> VisitedBBs;
3081 
3082  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
3083  ValueList Operands;
3084  BasicBlock *IBB = PH->getIncomingBlock(i);
3085 
3086  if (!VisitedBBs.insert(IBB).second) {
3087  NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
3088  continue;
3089  }
3090 
3091  // Prepare the operand vector.
3092  for (Value *V : E->Scalars)
3093  Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
3094 
3095  Builder.SetInsertPoint(IBB->getTerminator());
3096  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3097  Value *Vec = vectorizeTree(Operands);
3098  NewPhi->addIncoming(Vec, IBB);
3099  }
3100 
3101  assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
3102  "Invalid number of incoming values");
3103  return V;
3104  }
3105 
3106  case Instruction::ExtractElement: {
3107  if (!E->NeedToGather) {
3108  Value *V = VL0->getOperand(0);
3109  if (!E->ReorderIndices.empty()) {
3110  OrdersType Mask;
3111  inversePermutation(E->ReorderIndices, Mask);
3112  Builder.SetInsertPoint(VL0);
3113  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
3114  "reorder_shuffle");
3115  }
3116  if (NeedToShuffleReuses) {
3117  // TODO: Merge this shuffle with the ReorderShuffleMask.
3118  if (E->ReorderIndices.empty())
3119  Builder.SetInsertPoint(VL0);
3120  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3121  E->ReuseShuffleIndices, "shuffle");
3122  }
3123  E->VectorizedValue = V;
3124  return V;
3125  }
3126  setInsertPointAfterBundle(E->Scalars, S);
3127  auto *V = Gather(E->Scalars, VecTy);
3128  if (NeedToShuffleReuses) {
3129  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3130  E->ReuseShuffleIndices, "shuffle");
3131  if (auto *I = dyn_cast<Instruction>(V)) {
3132  GatherSeq.insert(I);
3133  CSEBlocks.insert(I->getParent());
3134  }
3135  }
3136  E->VectorizedValue = V;
3137  return V;
3138  }
3139  case Instruction::ExtractValue: {
3140  if (!E->NeedToGather) {
3141  LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3142  Builder.SetInsertPoint(LI);
3143  PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3144  Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3145  LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlignment());
3146  Value *NewV = propagateMetadata(V, E->Scalars);
3147  if (!E->ReorderIndices.empty()) {
3148  OrdersType Mask;
3149  inversePermutation(E->ReorderIndices, Mask);
3150  NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
3151  "reorder_shuffle");
3152  }
3153  if (NeedToShuffleReuses) {
3154  // TODO: Merge this shuffle with the ReorderShuffleMask.
3155  NewV = Builder.CreateShuffleVector(
3156  NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3157  }
3158  E->VectorizedValue = NewV;
3159  return NewV;
3160  }
3161  setInsertPointAfterBundle(E->Scalars, S);
3162  auto *V = Gather(E->Scalars, VecTy);
3163  if (NeedToShuffleReuses) {
3164  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3165  E->ReuseShuffleIndices, "shuffle");
3166  if (auto *I = dyn_cast<Instruction>(V)) {
3167  GatherSeq.insert(I);
3168  CSEBlocks.insert(I->getParent());
3169  }
3170  }
3171  E->VectorizedValue = V;
3172  return V;
3173  }
3174  case Instruction::ZExt:
3175  case Instruction::SExt:
3176  case Instruction::FPToUI:
3177  case Instruction::FPToSI:
3178  case Instruction::FPExt:
3179  case Instruction::PtrToInt:
3180  case Instruction::IntToPtr:
3181  case Instruction::SIToFP:
3182  case Instruction::UIToFP:
3183  case Instruction::Trunc:
3184  case Instruction::FPTrunc:
3185  case Instruction::BitCast: {
3186  ValueList INVL;
3187  for (Value *V : E->Scalars)
3188  INVL.push_back(cast<Instruction>(V)->getOperand(0));
3189 
3190  setInsertPointAfterBundle(E->Scalars, S);
3191 
3192  Value *InVec = vectorizeTree(INVL);
3193 
3194  if (E->VectorizedValue) {
3195  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3196  return E->VectorizedValue;
3197  }
3198 
3199  CastInst *CI = dyn_cast<CastInst>(VL0);
3200  Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3201  if (NeedToShuffleReuses) {
3202  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3203  E->ReuseShuffleIndices, "shuffle");
3204  }
3205  E->VectorizedValue = V;
3206  ++NumVectorInstructions;
3207  return V;
3208  }
3209  case Instruction::FCmp:
3210  case Instruction::ICmp: {
3211  ValueList LHSV, RHSV;
3212  for (Value *V : E->Scalars) {
3213  LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3214  RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3215  }
3216 
3217  setInsertPointAfterBundle(E->Scalars, S);
3218 
3219  Value *L = vectorizeTree(LHSV);
3220  Value *R = vectorizeTree(RHSV);
3221 
3222  if (E->VectorizedValue) {
3223  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3224  return E->VectorizedValue;
3225  }
3226 
3227  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3228  Value *V;
3229  if (S.getOpcode() == Instruction::FCmp)
3230  V = Builder.CreateFCmp(P0, L, R);
3231  else
3232  V = Builder.CreateICmp(P0, L, R);
3233 
3234  propagateIRFlags(V, E->Scalars, VL0);
3235  if (NeedToShuffleReuses) {
3236  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3237  E->ReuseShuffleIndices, "shuffle");
3238  }
3239  E->VectorizedValue = V;
3240  ++NumVectorInstructions;
3241  return V;
3242  }
3243  case Instruction::Select: {
3244  ValueList TrueVec, FalseVec, CondVec;
3245  for (Value *V : E->Scalars) {
3246  CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3247  TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3248  FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3249  }
3250 
3251  setInsertPointAfterBundle(E->Scalars, S);
3252 
3253  Value *Cond = vectorizeTree(CondVec);
3254  Value *True = vectorizeTree(TrueVec);
3255  Value *False = vectorizeTree(FalseVec);
3256 
3257  if (E->VectorizedValue) {
3258  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3259  return E->VectorizedValue;
3260  }
3261 
3262  Value *V = Builder.CreateSelect(Cond, True, False);
3263  if (NeedToShuffleReuses) {
3264  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3265  E->ReuseShuffleIndices, "shuffle");
3266  }
3267  E->VectorizedValue = V;
3268  ++NumVectorInstructions;
3269  return V;
3270  }
3271  case Instruction::Add:
3272  case Instruction::FAdd:
3273  case Instruction::Sub:
3274  case Instruction::FSub:
3275  case Instruction::Mul:
3276  case Instruction::FMul:
3277  case Instruction::UDiv:
3278  case Instruction::SDiv:
3279  case Instruction::FDiv:
3280  case Instruction::URem:
3281  case Instruction::SRem:
3282  case Instruction::FRem:
3283  case Instruction::Shl:
3284  case Instruction::LShr:
3285  case Instruction::AShr:
3286  case Instruction::And:
3287  case Instruction::Or:
3288  case Instruction::Xor: {
3289  ValueList LHSVL, RHSVL;
3290  if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3291  reorderInputsAccordingToOpcode(S.getOpcode(), E->Scalars, LHSVL,
3292  RHSVL);
3293  else
3294  for (Value *V : E->Scalars) {
3295  auto *I = cast<Instruction>(V);
3296  LHSVL.push_back(I->getOperand(0));
3297  RHSVL.push_back(I->getOperand(1));
3298  }
3299 
3300  setInsertPointAfterBundle(E->Scalars, S);
3301 
3302  Value *LHS = vectorizeTree(LHSVL);
3303  Value *RHS = vectorizeTree(RHSVL);
3304 
3305  if (E->VectorizedValue) {
3306  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3307  return E->VectorizedValue;
3308  }
3309 
3310  Value *V = Builder.CreateBinOp(
3311  static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3312  propagateIRFlags(V, E->Scalars, VL0);
3313  if (auto *I = dyn_cast<Instruction>(V))
3314  V = propagateMetadata(I, E->Scalars);
3315 
3316  if (NeedToShuffleReuses) {
3317  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3318  E->ReuseShuffleIndices, "shuffle");
3319  }
3320  E->VectorizedValue = V;
3321  ++NumVectorInstructions;
3322 
3323  return V;
3324  }
3325  case Instruction::Load: {
3326  // Loads are inserted at the head of the tree because we don't want to
3327  // sink them all the way down past store instructions.
3328  bool IsReorder = !E->ReorderIndices.empty();
3329  if (IsReorder) {
3330  S = getSameOpcode(E->Scalars, E->ReorderIndices.front());
3331  VL0 = cast<Instruction>(S.OpValue);
3332  }
3333  setInsertPointAfterBundle(E->Scalars, S);
3334 
3335  LoadInst *LI = cast<LoadInst>(VL0);
3336  Type *ScalarLoadTy = LI->getType();
3337  unsigned AS = LI->getPointerAddressSpace();
3338 
3339  Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3340  VecTy->getPointerTo(AS));
3341 
3342  // The pointer operand uses an in-tree scalar so we add the new BitCast to
3343  // ExternalUses list to make sure that an extract will be generated in the
3344  // future.
3345  Value *PO = LI->getPointerOperand();
3346  if (getTreeEntry(PO))
3347  ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3348 
3349  unsigned Alignment = LI->getAlignment();
3350  LI = Builder.CreateLoad(VecTy, VecPtr);
3351  if (!Alignment) {
3352  Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3353  }
3354  LI->setAlignment(Alignment);
3355  Value *V = propagateMetadata(LI, E->Scalars);
3356  if (IsReorder) {
3357  OrdersType Mask;
3358  inversePermutation(E->ReorderIndices, Mask);
3359  V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3360  Mask, "reorder_shuffle");
3361  }
3362  if (NeedToShuffleReuses) {
3363  // TODO: Merge this shuffle with the ReorderShuffleMask.
3364  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3365  E->ReuseShuffleIndices, "shuffle");
3366  }
3367  E->VectorizedValue = V;
3368  ++NumVectorInstructions;
3369  return V;
3370  }
3371  case Instruction::Store: {
3372  StoreInst *SI = cast<StoreInst>(VL0);
3373  unsigned Alignment = SI->getAlignment();
3374  unsigned AS = SI->getPointerAddressSpace();
3375 
3376  ValueList ScalarStoreValues;
3377  for (Value *V : E->Scalars)
3378  ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3379 
3380  setInsertPointAfterBundle(E->Scalars, S);
3381 
3382  Value *VecValue = vectorizeTree(ScalarStoreValues);
3383  Value *ScalarPtr = SI->getPointerOperand();
3384  Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3385  StoreInst *ST = Builder.CreateStore(VecValue, VecPtr);
3386 
3387  // The pointer operand uses an in-tree scalar, so add the new BitCast to
3388  // ExternalUses to make sure that an extract will be generated in the
3389  // future.
3390  if (getTreeEntry(ScalarPtr))
3391  ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3392 
3393  if (!Alignment)
3394  Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3395 
3396  ST->setAlignment(Alignment);
3397  Value *V = propagateMetadata(ST, E->Scalars);
3398  if (NeedToShuffleReuses) {
3399  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3400  E->ReuseShuffleIndices, "shuffle");
3401  }
3402  E->VectorizedValue = V;
3403  ++NumVectorInstructions;
3404  return V;
3405  }
3406  case Instruction::GetElementPtr: {
3407  setInsertPointAfterBundle(E->Scalars, S);
3408 
3409  ValueList Op0VL;
3410  for (Value *V : E->Scalars)
3411  Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3412 
3413  Value *Op0 = vectorizeTree(Op0VL);
3414 
3415  std::vector<Value *> OpVecs;
3416  for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3417  ++j) {
3418  ValueList OpVL;
3419  for (Value *V : E->Scalars)
3420  OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3421 
3422  Value *OpVec = vectorizeTree(OpVL);
3423  OpVecs.push_back(OpVec);
3424  }
3425 
3426  Value *V = Builder.CreateGEP(
3427  cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3428  if (Instruction *I = dyn_cast<Instruction>(V))
3429  V = propagateMetadata(I, E->Scalars);
3430 
3431  if (NeedToShuffleReuses) {
3432  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3433  E->ReuseShuffleIndices, "shuffle");
3434  }
3435  E->VectorizedValue = V;
3436  ++NumVectorInstructions;
3437 
3438  return V;
3439  }
3440  case Instruction::Call: {
3441  CallInst *CI = cast<CallInst>(VL0);
3442  setInsertPointAfterBundle(E->Scalars, S);
3443  Function *FI;
3445  Value *ScalarArg = nullptr;
3446  if (CI && (FI = CI->getCalledFunction())) {
3447  IID = FI->getIntrinsicID();
3448  }
3449  std::vector<Value *> OpVecs;
3450  for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3451  ValueList OpVL;
3452  // Some intrinsics have scalar arguments. This argument should not be
3453  // vectorized.
3454  if (hasVectorInstrinsicScalarOpd(IID, j)) {
3455  CallInst *CEI = cast<CallInst>(VL0);
3456  ScalarArg = CEI->getArgOperand(j);
3457  OpVecs.push_back(CEI->getArgOperand(j));
3458  continue;
3459  }
3460  for (Value *V : E->Scalars) {
3461  CallInst *CEI = cast<CallInst>(V);
3462  OpVL.push_back(CEI->getArgOperand(j));
3463  }
3464 
3465  Value *OpVec = vectorizeTree(OpVL);
3466  LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3467  OpVecs.push_back(OpVec);
3468  }
3469 
3470  Module *M = F->getParent();
3472  Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3473  Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3475  CI->getOperandBundlesAsDefs(OpBundles);
3476  Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3477 
3478  // The scalar argument uses an in-tree scalar so we add the new vectorized
3479  // call to ExternalUses list to make sure that an extract will be
3480  // generated in the future.
3481  if (ScalarArg && getTreeEntry(ScalarArg))
3482  ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3483 
3484  propagateIRFlags(V, E->Scalars, VL0);
3485  if (NeedToShuffleReuses) {
3486  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3487  E->ReuseShuffleIndices, "shuffle");
3488  }
3489  E->VectorizedValue = V;
3490  ++NumVectorInstructions;
3491  return V;
3492  }
3493  case Instruction::ShuffleVector: {
3494  ValueList LHSVL, RHSVL;
3495  assert(S.isAltShuffle() &&
3496  ((Instruction::isBinaryOp(S.getOpcode()) &&
3497  Instruction::isBinaryOp(S.getAltOpcode())) ||
3498  (Instruction::isCast(S.getOpcode()) &&
3499  Instruction::isCast(S.getAltOpcode()))) &&
3500  "Invalid Shuffle Vector Operand");
3501 
3502  Value *LHS, *RHS;
3503  if (Instruction::isBinaryOp(S.getOpcode())) {
3504  reorderAltShuffleOperands(S, E->Scalars, LHSVL, RHSVL);
3505  setInsertPointAfterBundle(E->Scalars, S);
3506  LHS = vectorizeTree(LHSVL);
3507  RHS = vectorizeTree(RHSVL);
3508  } else {
3509  ValueList INVL;
3510  for (Value *V : E->Scalars)
3511  INVL.push_back(cast<Instruction>(V)->getOperand(0));
3512  setInsertPointAfterBundle(E->Scalars, S);
3513  LHS = vectorizeTree(INVL);
3514  }
3515 
3516  if (E->VectorizedValue) {
3517  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3518  return E->VectorizedValue;
3519  }
3520 
3521  Value *V0, *V1;
3522  if (Instruction::isBinaryOp(S.getOpcode())) {
3523  V0 = Builder.CreateBinOp(
3524  static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3525  V1 = Builder.CreateBinOp(
3526  static_cast<Instruction::BinaryOps>(S.getAltOpcode()), LHS, RHS);
3527  } else {
3528  V0 = Builder.CreateCast(
3529  static_cast<Instruction::CastOps>(S.getOpcode()), LHS, VecTy);
3530  V1 = Builder.CreateCast(
3531  static_cast<Instruction::CastOps>(S.getAltOpcode()), LHS, VecTy);
3532  }
3533 
3534  // Create shuffle to take alternate operations from the vector.
3535  // Also, gather up main and alt scalar ops to propagate IR flags to
3536  // each vector operation.
3537  ValueList OpScalars, AltScalars;
3538  unsigned e = E->Scalars.size();
3540  for (unsigned i = 0; i < e; ++i) {
3541  auto *OpInst = cast<Instruction>(E->Scalars[i]);
3542  assert(S.isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode");
3543  if (OpInst->getOpcode() == S.getAltOpcode()) {
3544  Mask[i] = Builder.getInt32(e + i);
3545  AltScalars.push_back(E->Scalars[i]);
3546  } else {
3547  Mask[i] = Builder.getInt32(i);
3548  OpScalars.push_back(E->Scalars[i]);
3549  }
3550  }
3551 
3552  Value *ShuffleMask = ConstantVector::get(Mask);
3553  propagateIRFlags(V0, OpScalars);
3554  propagateIRFlags(V1, AltScalars);
3555 
3556  Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3557  if (Instruction *I = dyn_cast<Instruction>(V))
3558  V = propagateMetadata(I, E->Scalars);
3559  if (NeedToShuffleReuses) {
3560  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3561  E->ReuseShuffleIndices, "shuffle");
3562  }
3563  E->VectorizedValue = V;
3564  ++NumVectorInstructions;
3565 
3566  return V;
3567  }
3568  default:
3569  llvm_unreachable("unknown inst");
3570  }
3571  return nullptr;
3572 }
3573 
3574 Value *BoUpSLP::vectorizeTree() {
3575  ExtraValueToDebugLocsMap ExternallyUsedValues;
3576  return vectorizeTree(ExternallyUsedValues);
3577 }
3578 
3579 Value *
3580 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3581  // All blocks must be scheduled before any instructions are inserted.
3582  for (auto &BSIter : BlocksSchedules) {
3583  scheduleBlock(BSIter.second.get());
3584  }
3585 
3586  Builder.SetInsertPoint(&F->getEntryBlock().front());
3587  auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3588 
3589  // If the vectorized tree can be rewritten in a smaller type, we truncate the
3590  // vectorized root. InstCombine will then rewrite the entire expression. We
3591  // sign extend the extracted values below.
3592  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3593  if (MinBWs.count(ScalarRoot)) {
3594  if (auto *I = dyn_cast<Instruction>(VectorRoot))
3595  Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3596  auto BundleWidth = VectorizableTree[0].Scalars.size();
3597  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3598  auto *VecTy = VectorType::get(MinTy, BundleWidth);
3599  auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3600  VectorizableTree[0].VectorizedValue = Trunc;
3601  }
3602 
3603  LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()
3604  << " values .\n");
3605 
3606  // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3607  // specified by ScalarType.
3608  auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3609  if (!MinBWs.count(ScalarRoot))
3610  return Ex;
3611  if (MinBWs[ScalarRoot].second)
3612  return Builder.CreateSExt(Ex, ScalarType);
3613  return Builder.CreateZExt(Ex, ScalarType);
3614  };
3615 
3616  // Extract all of the elements with the external uses.
3617  for (const auto &ExternalUse : ExternalUses) {
3618  Value *Scalar = ExternalUse.Scalar;
3619  llvm::User *User = ExternalUse.User;
3620 
3621  // Skip users that we already RAUW. This happens when one instruction
3622  // has multiple uses of the same value.
3623  if (User && !is_contained(Scalar->users(), User))
3624  continue;
3625  TreeEntry *E = getTreeEntry(Scalar);
3626  assert(E && "Invalid scalar");
3627  assert(!E->NeedToGather && "Extracting from a gather list");
3628 
3629  Value *Vec = E->VectorizedValue;
3630  assert(Vec && "Can't find vectorizable value");
3631 
3632  Value *Lane = Builder.getInt32(ExternalUse.Lane);
3633  // If User == nullptr, the Scalar is used as extra arg. Generate
3634  // ExtractElement instruction and update the record for this scalar in
3635  // ExternallyUsedValues.
3636  if (!User) {
3637  assert(ExternallyUsedValues.count(Scalar) &&
3638  "Scalar with nullptr as an external user must be registered in "
3639  "ExternallyUsedValues map");
3640  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3641  Builder.SetInsertPoint(VecI->getParent(),
3642  std::next(VecI->getIterator()));
3643  } else {
3644  Builder.SetInsertPoint(&F->getEntryBlock().front());
3645  }
3646  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3647  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3648  CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3649  auto &Locs = ExternallyUsedValues[Scalar];
3650  ExternallyUsedValues.insert({Ex, Locs});
3651  ExternallyUsedValues.erase(Scalar);
3652  // Required to update internally referenced instructions.
3653  Scalar->replaceAllUsesWith(Ex);
3654  continue;
3655  }
3656 
3657  // Generate extracts for out-of-tree users.
3658  // Find the insertion point for the extractelement lane.
3659  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3660  if (PHINode *PH = dyn_cast<PHINode>(User)) {
3661  for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3662  if (PH->getIncomingValue(i) == Scalar) {
3663  Instruction *IncomingTerminator =
3664  PH->getIncomingBlock(i)->getTerminator();
3665  if (isa<CatchSwitchInst>(IncomingTerminator)) {
3666  Builder.SetInsertPoint(VecI->getParent(),
3667  std::next(VecI->getIterator()));
3668  } else {
3669  Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3670  }
3671  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3672  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3673  CSEBlocks.insert(PH->getIncomingBlock(i));
3674  PH->setOperand(i, Ex);
3675  }
3676  }
3677  } else {
3678  Builder.SetInsertPoint(cast<Instruction>(User));
3679  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3680  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3681  CSEBlocks.insert(cast<Instruction>(User)->getParent());
3682  User->replaceUsesOfWith(Scalar, Ex);
3683  }
3684  } else {
3685  Builder.SetInsertPoint(&F->getEntryBlock().front());
3686  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3687  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3688  CSEBlocks.insert(&F->getEntryBlock());
3689  User->replaceUsesOfWith(Scalar, Ex);
3690  }
3691 
3692  LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3693  }
3694 
3695  // For each vectorized value:
3696  for (TreeEntry &EIdx : VectorizableTree) {
3697  TreeEntry *Entry = &EIdx;
3698 
3699  // No need to handle users of gathered values.
3700  if (Entry->NeedToGather)
3701  continue;
3702 
3703  assert(Entry->VectorizedValue && "Can't find vectorizable value");
3704 
3705  // For each lane:
3706  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3707  Value *Scalar = Entry->Scalars[Lane];
3708 
3709  Type *Ty = Scalar->getType();
3710  if (!Ty->isVoidTy()) {
3711 #ifndef NDEBUG
3712  for (User *U : Scalar->users()) {
3713  LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3714 
3715  // It is legal to replace users in the ignorelist by undef.
3716  assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3717  "Replacing out-of-tree value with undef");
3718  }
3719 #endif
3720  Value *Undef = UndefValue::get(Ty);
3721  Scalar->replaceAllUsesWith(Undef);
3722  }
3723  LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3724  eraseInstruction(cast<Instruction>(Scalar));
3725  }
3726  }
3727 
3728  Builder.ClearInsertionPoint();
3729 
3730  return VectorizableTree[0].VectorizedValue;
3731 }
3732 
3733 void BoUpSLP::optimizeGatherSequence() {
3734  LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3735  << " gather sequences instructions.\n");
3736  // LICM InsertElementInst sequences.
3737  for (Instruction *I : GatherSeq) {
3738  if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3739  continue;
3740 
3741  // Check if this block is inside a loop.
3742  Loop *L = LI->getLoopFor(I->getParent());
3743  if (!L)
3744  continue;
3745 
3746  // Check if it has a preheader.
3747  BasicBlock *PreHeader = L->getLoopPreheader();
3748  if (!PreHeader)
3749  continue;
3750 
3751  // If the vector or the element that we insert into it are
3752  // instructions that are defined in this basic block then we can't
3753  // hoist this instruction.
3754  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3755  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3756  if (Op0 && L->contains(Op0))
3757  continue;
3758  if (Op1 && L->contains(Op1))
3759  continue;
3760 
3761  // We can hoist this instruction. Move it to the pre-header.
3762  I->moveBefore(PreHeader->getTerminator());
3763  }
3764 
3765  // Make a list of all reachable blocks in our CSE queue.
3767  CSEWorkList.reserve(CSEBlocks.size());
3768  for (BasicBlock *BB : CSEBlocks)
3769  if (DomTreeNode *N = DT->getNode(BB)) {
3770  assert(DT->isReachableFromEntry(N));
3771  CSEWorkList.push_back(N);
3772  }
3773 
3774  // Sort blocks by domination. This ensures we visit a block after all blocks
3775  // dominating it are visited.
3776  std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3777  [this](const DomTreeNode *A, const DomTreeNode *B) {
3778  return DT->properlyDominates(A, B);
3779  });
3780 
3781  // Perform O(N^2) search over the gather sequences and merge identical
3782  // instructions. TODO: We can further optimize this scan if we split the
3783  // instructions into different buckets based on the insert lane.
3785  for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3786  assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
3787  "Worklist not sorted properly!");
3788  BasicBlock *BB = (*I)->getBlock();
3789  // For all instructions in blocks containing gather sequences:
3790  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3791  Instruction *In = &*it++;
3792  if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3793  continue;
3794 
3795  // Check if we can replace this instruction with any of the
3796  // visited instructions.
3797  for (Instruction *v : Visited) {
3798  if (In->isIdenticalTo(v) &&
3799  DT->dominates(v->getParent(), In->getParent())) {
3800  In->replaceAllUsesWith(v);
3801  eraseInstruction(In);
3802  In = nullptr;
3803  break;
3804  }
3805  }
3806  if (In) {
3807  assert(!is_contained(Visited, In));
3808  Visited.push_back(In);
3809  }
3810  }
3811  }
3812  CSEBlocks.clear();
3813  GatherSeq.clear();
3814 }
3815 
3816 // Groups the instructions to a bundle (which is then a single scheduling entity)
3817 // and schedules instructions until the bundle gets ready.
3818 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3819  BoUpSLP *SLP,
3820  const InstructionsState &S) {
3821  if (isa<PHINode>(S.OpValue))
3822  return true;
3823 
3824  // Initialize the instruction bundle.
3825  Instruction *OldScheduleEnd = ScheduleEnd;
3826  ScheduleData *PrevInBundle = nullptr;
3827  ScheduleData *Bundle = nullptr;
3828  bool ReSchedule = false;
3829  LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n");
3830 
3831  // Make sure that the scheduling region contains all
3832  // instructions of the bundle.
3833  for (Value *V : VL) {
3834  if (!extendSchedulingRegion(V, S))
3835  return false;
3836  }
3837 
3838  for (Value *V : VL) {
3839  ScheduleData *BundleMember = getScheduleData(V);
3840  assert(BundleMember &&
3841  "no ScheduleData for bundle member (maybe not in same basic block)");
3842  if (BundleMember->IsScheduled) {
3843  // A bundle member was scheduled as single instruction before and now
3844  // needs to be scheduled as part of the bundle. We just get rid of the
3845  // existing schedule.
3846  LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3847  << " was already scheduled\n");
3848  ReSchedule = true;
3849  }
3850  assert(BundleMember->isSchedulingEntity() &&
3851  "bundle member already part of other bundle");
3852  if (PrevInBundle) {
3853  PrevInBundle->NextInBundle = BundleMember;
3854  } else {
3855  Bundle = BundleMember;
3856  }
3857  BundleMember->UnscheduledDepsInBundle = 0;
3858  Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3859 
3860  // Group the instructions to a bundle.
3861  BundleMember->FirstInBundle = Bundle;
3862  PrevInBundle = BundleMember;
3863  }
3864  if (ScheduleEnd != OldScheduleEnd) {
3865  // The scheduling region got new instructions at the lower end (or it is a
3866  // new region for the first bundle). This makes it necessary to
3867  // recalculate all dependencies.
3868  // It is seldom that this needs to be done a second time after adding the
3869  // initial bundle to the region.
3870  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3871  doForAllOpcodes(I, [](ScheduleData *SD) {
3872  SD->clearDependencies();
3873  });
3874  }
3875  ReSchedule = true;
3876  }
3877  if (ReSchedule) {
3878  resetSchedule();
3879  initialFillReadyList(ReadyInsts);
3880  }
3881 
3882  LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3883  << BB->getName() << "\n");
3884 
3885  calculateDependencies(Bundle, true, SLP);
3886 
3887  // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3888  // means that there are no cyclic dependencies and we can schedule it.
3889  // Note that's important that we don't "schedule" the bundle yet (see
3890  // cancelScheduling).
3891  while (!Bundle->isReady() && !ReadyInsts.empty()) {
3892 
3893  ScheduleData *pickedSD = ReadyInsts.back();
3894  ReadyInsts.pop_back();
3895 
3896  if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3897  schedule(pickedSD, ReadyInsts);
3898  }
3899  }
3900  if (!Bundle->isReady()) {
3901  cancelScheduling(VL, S.OpValue);
3902  return false;
3903  }
3904  return true;
3905 }
3906 
3907 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3908  Value *OpValue) {
3909  if (isa<PHINode>(OpValue))
3910  return;
3911 
3912  ScheduleData *Bundle = getScheduleData(OpValue);
3913  LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3914  assert(!Bundle->IsScheduled &&
3915  "Can't cancel bundle which is already scheduled");
3916  assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3917  "tried to unbundle something which is not a bundle");
3918 
3919  // Un-bundle: make single instructions out of the bundle.
3920  ScheduleData *BundleMember = Bundle;
3921  while (BundleMember) {
3922  assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3923  BundleMember->FirstInBundle = BundleMember;
3924  ScheduleData *Next = BundleMember->NextInBundle;
3925  BundleMember->NextInBundle = nullptr;
3926  BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3927  if (BundleMember->UnscheduledDepsInBundle == 0) {
3928  ReadyInsts.insert(BundleMember);
3929  }
3930  BundleMember = Next;
3931  }
3932 }
3933 
3934 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3935  // Allocate a new ScheduleData for the instruction.
3936  if (ChunkPos >= ChunkSize) {
3937  ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3938  ChunkPos = 0;
3939  }
3940  return &(ScheduleDataChunks.back()[ChunkPos++]);
3941 }
3942 
3943 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3944  const InstructionsState &S) {
3945  if (getScheduleData(V, isOneOf(S, V)))
3946  return true;
3947  Instruction *I = dyn_cast<Instruction>(V);
3948  assert(I && "bundle member must be an instruction");
3949  assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3950  auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool {
3951  ScheduleData *ISD = getScheduleData(I);
3952  if (!ISD)
3953  return false;
3954  assert(isInSchedulingRegion(ISD) &&
3955  "ScheduleData not in scheduling region");
3956  ScheduleData *SD = allocateScheduleDataChunks();
3957  SD->Inst = I;
3958  SD->init(SchedulingRegionID, S.OpValue);
3959  ExtraScheduleDataMap[I][S.OpValue] = SD;
3960  return true;
3961  };
3962  if (CheckSheduleForI(I))
3963  return true;
3964  if (!ScheduleStart) {
3965  // It's the first instruction in the new region.
3966  initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3967  ScheduleStart = I;
3968  ScheduleEnd = I->getNextNode();
3969  if (isOneOf(S, I) != I)
3970  CheckSheduleForI(I);
3971  assert(ScheduleEnd && "tried to vectorize a terminator?");
3972  LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3973  return true;
3974  }
3975  // Search up and down at the same time, because we don't know if the new
3976  // instruction is above or below the existing scheduling region.
3978  ++ScheduleStart->getIterator().getReverse();
3979  BasicBlock::reverse_iterator UpperEnd = BB->rend();
3980  BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3981  BasicBlock::iterator LowerEnd = BB->end();
3982  while (true) {
3983  if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3984  LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3985  return false;
3986  }
3987 
3988  if (UpIter != UpperEnd) {
3989  if (&*UpIter == I) {
3990  initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3991  ScheduleStart = I;
3992  if (isOneOf(S, I) != I)
3993  CheckSheduleForI(I);
3994  LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I
3995  << "\n");
3996  return true;
3997  }
3998  UpIter++;
3999  }
4000  if (DownIter != LowerEnd) {
4001  if (&*DownIter == I) {
4002  initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
4003  nullptr);
4004  ScheduleEnd = I->getNextNode();
4005  if (isOneOf(S, I) != I)
4006  CheckSheduleForI(I);
4007  assert(ScheduleEnd && "tried to vectorize a terminator?");
4008  LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I
4009  << "\n");
4010  return true;
4011  }
4012  DownIter++;
4013  }
4014  assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
4015  "instruction not found in block");
4016  }
4017  return true;
4018 }
4019 
4020 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
4021  Instruction *ToI,
4022  ScheduleData *PrevLoadStore,
4023  ScheduleData *NextLoadStore) {
4024  ScheduleData *CurrentLoadStore = PrevLoadStore;
4025  for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
4026  ScheduleData *SD = ScheduleDataMap[I];
4027  if (!SD) {
4028  SD = allocateScheduleDataChunks();
4029  ScheduleDataMap[I] = SD;
4030  SD->Inst = I;
4031  }
4032  assert(!isInSchedulingRegion(SD) &&
4033  "new ScheduleData already in scheduling region");
4034  SD->init(SchedulingRegionID, I);
4035 
4036  if (I->mayReadOrWriteMemory() &&
4037  (!isa<IntrinsicInst>(I) ||
4038  cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
4039  // Update the linked list of memory accessing instructions.
4040  if (CurrentLoadStore) {
4041  CurrentLoadStore->NextLoadStore = SD;
4042  } else {
4043  FirstLoadStoreInRegion = SD;
4044  }
4045  CurrentLoadStore = SD;
4046  }
4047  }
4048  if (NextLoadStore) {
4049  if (CurrentLoadStore)
4050  CurrentLoadStore->NextLoadStore = NextLoadStore;
4051  } else {
4052  LastLoadStoreInRegion = CurrentLoadStore;
4053  }
4054 }
4055 
4056 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
4057  bool InsertInReadyList,
4058  BoUpSLP *SLP) {
4059  assert(SD->isSchedulingEntity());
4060 
4062  WorkList.push_back(SD);
4063 
4064  while (!WorkList.empty()) {
4065  ScheduleData *SD = WorkList.back();
4066  WorkList.pop_back();
4067 
4068  ScheduleData *BundleMember = SD;
4069  while (BundleMember) {
4070  assert(isInSchedulingRegion(BundleMember));
4071  if (!BundleMember->hasValidDependencies()) {
4072 
4073  LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember
4074  << "\n");
4075  BundleMember->Dependencies = 0;
4076  BundleMember->resetUnscheduledDeps();
4077 
4078  // Handle def-use chain dependencies.
4079  if (BundleMember->OpValue != BundleMember->Inst) {
4080  ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
4081  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4082  BundleMember->Dependencies++;
4083  ScheduleData *DestBundle = UseSD->FirstInBundle;
4084  if (!DestBundle->IsScheduled)
4085  BundleMember->incrementUnscheduledDeps(1);
4086  if (!DestBundle->hasValidDependencies())
4087  WorkList.push_back(DestBundle);
4088  }
4089  } else {
4090  for (User *U : BundleMember->Inst->users()) {
4091  if (isa<Instruction>(U)) {
4092  ScheduleData *UseSD = getScheduleData(U);
4093  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4094  BundleMember->Dependencies++;
4095  ScheduleData *DestBundle = UseSD->FirstInBundle;
4096  if (!DestBundle->IsScheduled)
4097  BundleMember->incrementUnscheduledDeps(1);
4098  if (!DestBundle->hasValidDependencies())
4099  WorkList.push_back(DestBundle);
4100  }
4101  } else {
4102  // I'm not sure if this can ever happen. But we need to be safe.
4103  // This lets the instruction/bundle never be scheduled and
4104  // eventually disable vectorization.
4105  BundleMember->Dependencies++;
4106  BundleMember->incrementUnscheduledDeps(1);
4107  }
4108  }
4109  }
4110 
4111  // Handle the memory dependencies.
4112  ScheduleData *DepDest = BundleMember->NextLoadStore;
4113  if (DepDest) {
4114  Instruction *SrcInst = BundleMember->Inst;
4115  MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
4116  bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
4117  unsigned numAliased = 0;
4118  unsigned DistToSrc = 1;
4119 
4120  while (DepDest) {
4121  assert(isInSchedulingRegion(DepDest));
4122 
4123  // We have two limits to reduce the complexity:
4124  // 1) AliasedCheckLimit: It's a small limit to reduce calls to
4125  // SLP->isAliased (which is the expensive part in this loop).
4126  // 2) MaxMemDepDistance: It's for very large blocks and it aborts
4127  // the whole loop (even if the loop is fast, it's quadratic).
4128  // It's important for the loop break condition (see below) to
4129  // check this limit even between two read-only instructions.
4130  if (DistToSrc >= MaxMemDepDistance ||
4131  ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
4132  (numAliased >= AliasedCheckLimit ||
4133  SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
4134 
4135  // We increment the counter only if the locations are aliased
4136  // (instead of counting all alias checks). This gives a better
4137  // balance between reduced runtime and accurate dependencies.
4138  numAliased++;
4139 
4140  DepDest->MemoryDependencies.push_back(BundleMember);
4141  BundleMember->Dependencies++;
4142  ScheduleData *DestBundle = DepDest->FirstInBundle;
4143  if (!DestBundle->IsScheduled) {
4144  BundleMember->incrementUnscheduledDeps(1);
4145  }
4146  if (!DestBundle->hasValidDependencies()) {
4147  WorkList.push_back(DestBundle);
4148  }
4149  }
4150  DepDest = DepDest->NextLoadStore;
4151 
4152  // Example, explaining the loop break condition: Let's assume our
4153  // starting instruction is i0 and MaxMemDepDistance = 3.
4154  //
4155  // +--------v--v--v
4156  // i0,i1,i2,i3,i4,i5,i6,i7,i8
4157  // +--------^--^--^
4158  //
4159  // MaxMemDepDistance let us stop alias-checking at i3 and we add
4160  // dependencies from i0 to i3,i4,.. (even if they are not aliased).
4161  // Previously we already added dependencies from i3 to i6,i7,i8
4162  // (because of MaxMemDepDistance). As we added a dependency from
4163  // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4164  // and we can abort this loop at i6.
4165  if (DistToSrc >= 2 * MaxMemDepDistance)
4166  break;
4167  DistToSrc++;
4168  }
4169  }
4170  }
4171  BundleMember = BundleMember->NextInBundle;
4172  }
4173  if (InsertInReadyList && SD->isReady()) {
4174  ReadyInsts.push_back(SD);
4175  LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst
4176  << "\n");
4177  }
4178  }
4179 }
4180 
4181 void BoUpSLP::BlockScheduling::resetSchedule() {
4182  assert(ScheduleStart &&
4183  "tried to reset schedule on block which has not been scheduled");
4184  for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4185  doForAllOpcodes(I, [&](ScheduleData *SD) {
4186  assert(isInSchedulingRegion(SD) &&
4187  "ScheduleData not in scheduling region");
4188  SD->IsScheduled = false;
4189  SD->resetUnscheduledDeps();
4190  });
4191  }
4192  ReadyInsts.clear();
4193 }
4194 
4195 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4196  if (!BS->ScheduleStart)
4197  return;
4198 
4199  LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
4200 
4201  BS->resetSchedule();
4202 
4203  // For the real scheduling we use a more sophisticated ready-list: it is
4204  // sorted by the original instruction location. This lets the final schedule
4205  // be as close as possible to the original instruction order.
4206  struct ScheduleDataCompare {
4207  bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4208  return SD2->SchedulingPriority < SD1->SchedulingPriority;
4209  }
4210  };
4211  std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4212 
4213  // Ensure that all dependency data is updated and fill the ready-list with
4214  // initial instructions.
4215  int Idx = 0;
4216  int NumToSchedule = 0;
4217  for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4218  I = I->getNextNode()) {
4219  BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4220  assert(SD->isPartOfBundle() ==
4221  (getTreeEntry(SD->Inst) != nullptr) &&
4222  "scheduler and vectorizer bundle mismatch");
4223  SD->FirstInBundle->SchedulingPriority = Idx++;
4224  if (SD->isSchedulingEntity()) {
4225  BS->calculateDependencies(SD, false, this);
4226  NumToSchedule++;
4227  }
4228  });
4229  }
4230  BS->initialFillReadyList(ReadyInsts);
4231 
4232  Instruction *LastScheduledInst = BS->ScheduleEnd;
4233 
4234  // Do the "real" scheduling.
4235  while (!ReadyInsts.empty()) {
4236  ScheduleData *picked = *ReadyInsts.begin();
4237  ReadyInsts.erase(ReadyInsts.begin());
4238 
4239  // Move the scheduled instruction(s) to their dedicated places, if not
4240  // there yet.
4241  ScheduleData *BundleMember = picked;
4242  while (BundleMember) {
4243  Instruction *pickedInst = BundleMember->Inst;
4244  if (LastScheduledInst->getNextNode() != pickedInst) {
4245  BS->BB->getInstList().remove(pickedInst);
4246  BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4247  pickedInst);
4248  }
4249  LastScheduledInst = pickedInst;
4250  BundleMember = BundleMember->NextInBundle;
4251  }
4252 
4253  BS->schedule(picked, ReadyInsts);
4254  NumToSchedule--;
4255  }
4256  assert(NumToSchedule == 0 && "could not schedule all instructions");
4257 
4258  // Avoid duplicate scheduling of the block.
4259  BS->ScheduleStart = nullptr;
4260 }
4261 
4262 unsigned BoUpSLP::getVectorElementSize(Value *V) {
4263  // If V is a store, just return the width of the stored value without
4264  // traversing the expression tree. This is the common case.
4265  if (auto *Store = dyn_cast<StoreInst>(V))
4266  return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4267 
4268  // If V is not a store, we can traverse the expression tree to find loads
4269  // that feed it. The type of the loaded value may indicate a more suitable
4270  // width than V's type. We want to base the vector element size on the width
4271  // of memory operations where possible.
4274  if (auto *I = dyn_cast<Instruction>(V))
4275  Worklist.push_back(I);
4276 
4277  // Traverse the expression tree in bottom-up order looking for loads. If we
4278  // encounter an instruction we don't yet handle, we give up.
4279  auto MaxWidth = 0u;
4280  auto FoundUnknownInst = false;
4281  while (!Worklist.empty() && !FoundUnknownInst) {
4282  auto *I = Worklist.pop_back_val();
4283  Visited.insert(I);
4284 
4285  // We should only be looking at scalar instructions here. If the current
4286  // instruction has a vector type, give up.
4287  auto *Ty = I->getType();
4288  if (isa<VectorType>(Ty))
4289  FoundUnknownInst = true;
4290 
4291  // If the current instruction is a load, update MaxWidth to reflect the
4292  // width of the loaded value.
4293  else if (isa<LoadInst>(I))
4294  MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4295 
4296  // Otherwise, we need to visit the operands of the instruction. We only
4297  // handle the interesting cases from buildTree here. If an operand is an
4298  // instruction we haven't yet visited, we add it to the worklist.
4299  else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4300  isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4301  for (Use &U : I->operands())
4302  if (auto *J = dyn_cast<Instruction>(U.get()))
4303  if (!Visited.count(J))
4304  Worklist.push_back(J);
4305  }
4306 
4307  // If we don't yet handle the instruction, give up.
4308  else
4309  FoundUnknownInst = true;
4310  }
4311 
4312  // If we didn't encounter a memory access in the expression tree, or if we
4313  // gave up for some reason, just return the width of V.
4314  if (!MaxWidth || FoundUnknownInst)
4315  return DL->getTypeSizeInBits(V->getType());
4316 
4317  // Otherwise, return the maximum width we found.
4318  return MaxWidth;
4319 }
4320 
4321 // Determine if a value V in a vectorizable expression Expr can be demoted to a
4322 // smaller type with a truncation. We collect the values that will be demoted
4323 // in ToDemote and additional roots that require investigating in Roots.
4325  SmallVectorImpl<Value *> &ToDemote,
4326  SmallVectorImpl<Value *> &Roots) {
4327  // We can always demote constants.
4328  if (isa<Constant>(V)) {
4329  ToDemote.push_back(V);
4330  return true;
4331  }
4332 
4333  // If the value is not an instruction in the expression with only one use, it
4334  // cannot be demoted.
4335  auto *I = dyn_cast<Instruction>(V);
4336  if (!I || !I->hasOneUse() || !Expr.count(I))
4337  return false;
4338 
4339  switch (I->getOpcode()) {
4340 
4341  // We can always demote truncations and extensions. Since truncations can
4342  // seed additional demotion, we save the truncated value.
4343  case Instruction::Trunc:
4344  Roots.push_back(I->getOperand(0));
4345  break;
4346  case Instruction::ZExt:
4347  case Instruction::SExt:
4348  break;
4349 
4350  // We can demote certain binary operations if we can demote both of their
4351  // operands.
4352  case Instruction::Add:
4353  case Instruction::Sub:
4354  case Instruction::Mul:
4355  case Instruction::And:
4356  case Instruction::Or:
4357  case Instruction::Xor:
4358  if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4359  !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4360  return false;
4361  break;
4362 
4363  // We can demote selects if we can demote their true and false values.
4364  case Instruction::Select: {
4365  SelectInst *SI = cast<SelectInst>(I);
4366  if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4367  !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4368  return false;
4369  break;
4370  }
4371 
4372  // We can demote phis if we can demote all their incoming operands. Note that
4373  // we don't need to worry about cycles since we ensure single use above.
4374  case Instruction::PHI: {
4375  PHINode *PN = cast<PHINode>(I);
4376  for (Value *IncValue : PN->incoming_values())
4377  if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4378  return false;
4379  break;
4380  }
4381 
4382  // Otherwise, conservatively give up.
4383  default:
4384  return false;
4385  }
4386 
4387  // Record the value that we can demote.
4388  ToDemote.push_back(V);
4389  return true;
4390 }
4391 
4393  // If there are no external uses, the expression tree must be rooted by a
4394  // store. We can't demote in-memory values, so there is nothing to do here.
4395  if (ExternalUses.empty())
4396  return;
4397 
4398  // We only attempt to truncate integer expressions.
4399  auto &TreeRoot = VectorizableTree[0].Scalars;
4400  auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4401  if (!TreeRootIT)
4402  return;
4403 
4404  // If the expression is not rooted by a store, these roots should have
4405  // external uses. We will rely on InstCombine to rewrite the expression in
4406  // the narrower type. However, InstCombine only rewrites single-use values.
4407  // This means that if a tree entry other than a root is used externally, it
4408  // must have multiple uses and InstCombine will not rewrite it. The code
4409  // below ensures that only the roots are used externally.
4410  SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4411  for (auto &EU : ExternalUses)
4412  if (!Expr.erase(EU.Scalar))
4413  return;
4414  if (!Expr.empty())
4415  return;
4416 
4417  // Collect the scalar values of the vectorizable expression. We will use this
4418  // context to determine which values can be demoted. If we see a truncation,
4419  // we mark it as seeding another demotion.
4420  for (auto &Entry : VectorizableTree)
4421  Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4422 
4423  // Ensure the roots of the vectorizable tree don't form a cycle. They must
4424  // have a single external user that is not in the vectorizable tree.
4425  for (auto *Root : TreeRoot)
4426  if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4427  return;
4428 
4429  // Conservatively determine if we can actually truncate the roots of the
4430  // expression. Collect the values that can be demoted in ToDemote and
4431  // additional roots that require investigating in Roots.
4432  SmallVector<Value *, 32> ToDemote;
4434  for (auto *Root : TreeRoot)
4435  if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4436  return;
4437 
4438  // The maximum bit width required to represent all the values that can be
4439  // demoted without loss of precision. It would be safe to truncate the roots
4440  // of the expression to this width.
4441  auto MaxBitWidth = 8u;
4442 
4443  // We first check if all the bits of the roots are demanded. If they're not,
4444  // we can truncate the roots to this narrower type.
4445  for (auto *Root : TreeRoot) {
4446  auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4447  MaxBitWidth = std::max<unsigned>(
4448  Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4449  }
4450 
4451  // True if the roots can be zero-extended back to their original type, rather
4452  // than sign-extended. We know that if the leading bits are not demanded, we
4453  // can safely zero-extend. So we initialize IsKnownPositive to True.
4454  bool IsKnownPositive = true;
4455 
4456  // If all the bits of the roots are demanded, we can try a little harder to
4457  // compute a narrower type. This can happen, for example, if the roots are
4458  // getelementptr indices. InstCombine promotes these indices to the pointer
4459  // width. Thus, all their bits are technically demanded even though the
4460  // address computation might be vectorized in a smaller type.
4461  //
4462  // We start by looking at each entry that can be demoted. We compute the
4463  // maximum bit width required to store the scalar by using ValueTracking to
4464  // compute the number of high-order bits we can truncate.
4465  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
4466  llvm::all_of(TreeRoot, [](Value *R) {
4467  assert(R->hasOneUse() && "Root should have only one use!");
4468  return isa<GetElementPtrInst>(R->user_back());
4469  })) {
4470  MaxBitWidth = 8u;
4471 
4472  // Determine if the sign bit of all the roots is known to be zero. If not,
4473  // IsKnownPositive is set to False.
4474  IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4475  KnownBits Known = computeKnownBits(R, *DL);
4476  return Known.isNonNegative();
4477  });
4478 
4479  // Determine the maximum number of bits required to store the scalar
4480  // values.
4481  for (auto *Scalar : ToDemote) {
4482  auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4483  auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4484  MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4485  }
4486 
4487  // If we can't prove that the sign bit is zero, we must add one to the
4488  // maximum bit width to account for the unknown sign bit. This preserves
4489  // the existing sign bit so we can safely sign-extend the root back to the
4490  // original type. Otherwise, if we know the sign bit is zero, we will
4491  // zero-extend the root instead.
4492  //
4493  // FIXME: This is somewhat suboptimal, as there will be cases where adding
4494  // one to the maximum bit width will yield a larger-than-necessary
4495  // type. In general, we need to add an extra bit only if we can't
4496  // prove that the upper bit of the original type is equal to the
4497  // upper bit of the proposed smaller type. If these two bits are the
4498  // same (either zero or one) we know that sign-extending from the
4499  // smaller type will result in the same value. Here, since we can't
4500  // yet prove this, we are just making the proposed smaller type
4501  // larger to ensure correctness.
4502  if (!IsKnownPositive)
4503  ++MaxBitWidth;
4504  }
4505 
4506  // Round MaxBitWidth up to the next power-of-two.
4507  if (!isPowerOf2_64(MaxBitWidth))
4508  MaxBitWidth = NextPowerOf2(MaxBitWidth);
4509 
4510  // If the maximum bit width we compute is less than the with of the roots'
4511  // type, we can proceed with the narrowing. Otherwise, do nothing.
4512  if (MaxBitWidth >= TreeRootIT->getBitWidth())
4513  return;
4514 
4515  // If we can truncate the root, we must collect additional values that might
4516  // be demoted as a result. That is, those seeded by truncations we will
4517  // modify.
4518  while (!Roots.empty())
4519  collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4520 
4521  // Finally, map the values we can demote to the maximum bit with we computed.
4522  for (auto *Scalar : ToDemote)
4523  MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4524 }
4525 
4526 namespace {
4527 
4528 /// The SLPVectorizer Pass.
4529 struct SLPVectorizer : public FunctionPass {
4530  SLPVectorizerPass Impl;
4531 
4532  /// Pass identification, replacement for typeid
4533  static char ID;
4534 
4535  explicit SLPVectorizer() : FunctionPass(ID) {
4537  }
4538 
4539  bool doInitialization(Module &M) override {
4540  return false;
4541  }
4542 
4543  bool runOnFunction(Function &F) override {
4544  if (skipFunction(F))
4545  return false;
4546 
4547  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4548  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4549  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4550  auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4551  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4552  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4553  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4554  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4555  auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4556  auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4557 
4558  return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4559  }
4560 
4561  void getAnalysisUsage(AnalysisUsage &AU) const override {
4575  AU.setPreservesCFG();
4576  }
4577 };
4578 
4579 } // end anonymous namespace
4580 
4582  auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4583  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4584  auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4585  auto *AA = &AM.getResult<AAManager>(F);
4586  auto *LI = &AM.getResult<LoopAnalysis>(F);
4587  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4588  auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4589  auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4590  auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4591 
4592  bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4593  if (!Changed)
4594  return PreservedAnalyses::all();
4595 
4596  PreservedAnalyses PA;
4597  PA.preserveSet<CFGAnalyses>();
4598  PA.preserve<AAManager>();
4599  PA.preserve<GlobalsAA>();
4600  return PA;
4601 }
4602 
4604  TargetTransformInfo *TTI_,
4605  TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4606  LoopInfo *LI_, DominatorTree *DT_,
4607  AssumptionCache *AC_, DemandedBits *DB_,
4608  OptimizationRemarkEmitter *ORE_) {
4609  SE = SE_;
4610  TTI = TTI_;
4611  TLI = TLI_;
4612  AA = AA_;
4613  LI = LI_;
4614  DT = DT_;
4615  AC = AC_;
4616  DB = DB_;
4617  DL = &F.getParent()->getDataLayout();
4618 
4619  Stores.clear();
4620  GEPs.clear();
4621  bool Changed = false;
4622 
4623  // If the target claims to have no vector registers don't attempt
4624  // vectorization.
4625  if (!TTI->getNumberOfRegisters(true))
4626  return false;
4627 
4628  // Don't vectorize when the attribute NoImplicitFloat is used.
4629  if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4630  return false;
4631 
4632  LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4633 
4634  // Use the bottom up slp vectorizer to construct chains that start with
4635  // store instructions.
4636  BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4637 
4638  // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4639  // delete instructions.
4640 
4641  // Scan the blocks in the function in post order.
4642  for (auto BB : post_order(&F.getEntryBlock())) {
4643  collectSeedInstructions(BB);
4644 
4645  // Vectorize trees that end at stores.
4646  if (!Stores.empty()) {
4647  LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4648  << " underlying objects.\n");
4649  Changed |= vectorizeStoreChains(R);
4650  }
4651 
4652  // Vectorize trees that end at reductions.
4653  Changed |= vectorizeChainsInBlock(BB, R);
4654 
4655  // Vectorize the index computations of getelementptr instructions. This
4656  // is primarily intended to catch gather-like idioms ending at
4657  // non-consecutive loads.
4658  if (!GEPs.empty()) {
4659  LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4660  << " underlying objects.\n");
4661  Changed |= vectorizeGEPIndices(BB, R);
4662  }
4663  }
4664 
4665  if (Changed) {
4666  R.optimizeGatherSequence();
4667  LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4669  }
4670  return Changed;
4671 }
4672 
4673 /// Check that the Values in the slice in VL array are still existent in
4674 /// the WeakTrackingVH array.
4675 /// Vectorization of part of the VL array may cause later values in the VL array
4676 /// to become invalid. We track when this has happened in the WeakTrackingVH
4677 /// array.
4679  ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4680  unsigned SliceSize) {
4681  VL = VL.slice(SliceBegin, SliceSize);
4682  VH = VH.slice(SliceBegin, SliceSize);
4683  return !std::equal(VL.begin(), VL.end(), VH.begin());
4684 }
4685 
4686 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4687  unsigned VecRegSize) {
4688  const unsigned ChainLen = Chain.size();
4689  LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4690  << "\n");
4691  const unsigned Sz = R.getVectorElementSize(Chain[0]);
4692  const unsigned VF = VecRegSize / Sz;
4693 
4694  if (!isPowerOf2_32(Sz) || VF < 2)
4695  return false;
4696 
4697  // Keep track of values that were deleted by vectorizing in the loop below.
4698  const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4699 
4700  bool Changed = false;
4701  // Look for profitable vectorizable trees at all offsets, starting at zero.
4702  for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4703 
4704  // Check that a previous iteration of this loop did not delete the Value.
4705  if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4706  continue;
4707 
4708  LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4709  << "\n");
4710  ArrayRef<Value *> Operands = Chain.slice(i, VF);
4711 
4712  R.buildTree(Operands);
4713  if (R.isTreeTinyAndNotFullyVectorizable())
4714  continue;
4715 
4716  R.computeMinimumValueSizes();
4717 
4718  int Cost = R.getTreeCost();
4719 
4720  LLVM_DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF
4721  << "\n");
4722  if (Cost < -SLPCostThreshold) {
4723  LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4724 
4725  using namespace ore;
4726 
4727  R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4728  cast<StoreInst>(Chain[i]))
4729  << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4730  << " and with tree size "
4731  << NV("TreeSize", R.getTreeSize()));
4732 
4733  R.vectorizeTree();
4734 
4735  // Move to the next bundle.
4736  i += VF - 1;
4737  Changed = true;
4738  }
4739  }
4740 
4741  return Changed;
4742 }
4743 
4744 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4745  BoUpSLP &R) {
4746  SetVector<StoreInst *> Heads;
4748  SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4749 
4750  // We may run into multiple chains that merge into a single chain. We mark the
4751  // stores that we vectorized so that we don't visit the same store twice.
4752  BoUpSLP::ValueSet VectorizedStores;
4753  bool Changed = false;
4754 
4755  auto &&FindConsecutiveAccess =
4756  [this, &Stores, &Heads, &Tails, &ConsecutiveChain] (int K, int Idx) {
4757  if (!isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE))
4758  return false;
4759 
4760  Tails.insert(Stores[Idx]);
4761  Heads.insert(Stores[K]);
4762  ConsecutiveChain[Stores[K]] = Stores[Idx];
4763  return true;
4764  };
4765 
4766  // Do a quadratic search on all of the given stores in reverse order and find
4767  // all of the pairs of stores that follow each other.
4768  int E = Stores.size();
4769  for (int Idx = E - 1; Idx >= 0; --Idx) {
4770  // If a store has multiple consecutive store candidates, search according
4771  // to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4772  // This is because usually pairing with immediate succeeding or preceding
4773  // candidate create the best chance to find slp vectorization opportunity.
4774  for (int Offset = 1, F = std::max(E - Idx, Idx + 1); Offset < F; ++Offset)
4775  if ((Idx >= Offset && FindConsecutiveAccess(Idx - Offset, Idx)) ||
4776  (Idx + Offset < E && FindConsecutiveAccess(Idx + Offset, Idx)))
4777  break;
4778  }
4779 
4780  // For stores that start but don't end a link in the chain:
4781  for (auto *SI : llvm::reverse(Heads)) {
4782  if (Tails.count(SI))
4783  continue;
4784 
4785  // We found a store instr that starts a chain. Now follow the chain and try
4786  // to vectorize it.
4787  BoUpSLP::ValueList Operands;
4788  StoreInst *I = SI;
4789  // Collect the chain into a list.
4790  while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4791  Operands.push_back(I);
4792  // Move to the next value in the chain.
4793  I = ConsecutiveChain[I];
4794  }
4795 
4796  // FIXME: Is division-by-2 the correct step? Should we assert that the
4797  // register size is a power-of-2?
4798  for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4799  Size /= 2) {
4800  if (vectorizeStoreChain(Operands, R, Size)) {
4801  // Mark the vectorized stores so that we don't vectorize them again.
4802  VectorizedStores.insert(Operands.begin(), Operands.end());
4803  Changed = true;
4804  break;
4805  }
4806  }
4807  }
4808 
4809  return Changed;
4810 }
4811 
4812 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4813  // Initialize the collections. We will make a single pass over the block.
4814  Stores.clear();
4815  GEPs.clear();
4816 
4817  // Visit the store and getelementptr instructions in BB and organize them in
4818  // Stores and GEPs according to the underlying objects of their pointer
4819  // operands.
4820  for (Instruction &I : *BB) {
4821  // Ignore store instructions that are volatile or have a pointer operand
4822  // that doesn't point to a scalar type.
4823  if (auto *SI = dyn_cast<StoreInst>(&I)) {
4824  if (!SI->isSimple())
4825  continue;
4826  if (!isValidElementType(SI->getValueOperand()->getType()))
4827  continue;
4828  Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4829  }
4830 
4831  // Ignore getelementptr instructions that have more than one index, a
4832  // constant index, or a pointer operand that doesn't point to a scalar
4833  // type.
4834  else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4835  auto Idx = GEP->idx_begin()->get();
4836  if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4837  continue;
4838  if (!isValidElementType(Idx->getType()))
4839  continue;
4840  if (GEP->getType()->isVectorTy())
4841  continue;
4842  GEPs[GEP->getPointerOperand()].push_back(GEP);
4843  }
4844  }
4845 }
4846 
4847 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4848  if (!A || !B)
4849  return false;
4850  Value *VL[] = { A, B };
4851  return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4852 }
4853 
4854 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4855  int UserCost, bool AllowReorder) {
4856  if (VL.size() < 2)
4857  return false;
4858 
4859  LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = "
4860  << VL.size() << ".\n");
4861 
4862  // Check that all of the parts are scalar instructions of the same type,
4863  // we permit an alternate opcode via InstructionsState.
4864  InstructionsState S = getSameOpcode(VL);
4865  if (!S.getOpcode())
4866  return false;
4867 
4868  Instruction *I0 = cast<Instruction>(S.OpValue);
4869  unsigned Sz = R.getVectorElementSize(I0);
4870  unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4871  unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4872  if (MaxVF < 2) {
4873  R.getORE()->emit([&]() {
4874  return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
4875  << "Cannot SLP vectorize list: vectorization factor "
4876  << "less than 2 is not supported";
4877  });
4878  return false;
4879  }
4880 
4881  for (Value *V : VL) {
4882  Type *Ty = V->getType();
4883  if (!isValidElementType(Ty)) {
4884  // NOTE: the following will give user internal llvm type name, which may
4885  // not be useful.
4886  R.getORE()->emit([&]() {
4887  std::string type_str;
4888  llvm::raw_string_ostream rso(type_str);
4889  Ty->print(rso);
4890  return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0)
4891  << "Cannot SLP vectorize list: type "
4892  << rso.str() + " is unsupported by vectorizer";
4893  });
4894  return false;
4895  }
4896  }
4897 
4898  bool Changed = false;
4899  bool CandidateFound = false;
4900  int MinCost = SLPCostThreshold;
4901 
4902  // Keep track of values that were deleted by vectorizing in the loop below.
4903  SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4904 
4905  unsigned NextInst = 0, MaxInst = VL.size();
4906  for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4907  VF /= 2) {
4908  // No actual vectorization should happen, if number of parts is the same as
4909  // provided vectorization factor (i.e. the scalar type is used for vector
4910  // code during codegen).
4911  auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4912  if (TTI->getNumberOfParts(VecTy) == VF)
4913  continue;
4914  for (unsigned I = NextInst; I < MaxInst; ++I) {
4915  unsigned OpsWidth = 0;
4916 
4917  if (I + VF > MaxInst)
4918  OpsWidth = MaxInst - I;
4919  else
4920  OpsWidth = VF;
4921 
4922  if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4923  break;
4924 
4925  // Check that a previous iteration of this loop did not delete the Value.
4926  if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4927  continue;
4928 
4929  LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4930  << "\n");
4931  ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4932 
4933  R.buildTree(Ops);
4934  Optional<ArrayRef<unsigned>> Order = R.bestOrder();
4935  // TODO: check if we can allow reordering for more cases.
4936  if (AllowReorder && Order) {
4937  // TODO: reorder tree nodes without tree rebuilding.
4938  // Conceptually, there is nothing actually preventing us from trying to
4939  // reorder a larger list. In fact, we do exactly this when vectorizing
4940  // reductions. However, at this point, we only expect to get here when
4941  // there are exactly two operations.
4942  assert(Ops.size() == 2);
4943  Value *ReorderedOps[] = {Ops[1], Ops[0]};
4944  R.buildTree(ReorderedOps, None);
4945  }
4946  if (R.isTreeTinyAndNotFullyVectorizable())
4947  continue;
4948 
4949  R.computeMinimumValueSizes();
4950  int Cost = R.getTreeCost() - UserCost;
4951  CandidateFound = true;
4952  MinCost = std::min(MinCost, Cost);
4953 
4954  if (Cost < -SLPCostThreshold) {
4955  LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4956  R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4957  cast<Instruction>(Ops[0]))
4958  << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4959  << " and with tree size "
4960  << ore::NV("TreeSize", R.getTreeSize()));
4961 
4962  R.vectorizeTree();
4963  // Move to the next bundle.
4964  I += VF - 1;
4965  NextInst = I + 1;
4966  Changed = true;
4967  }
4968  }
4969  }
4970 
4971  if (!Changed && CandidateFound) {
4972  R.getORE()->emit([&]() {
4973  return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0)
4974  << "List vectorization was possible but not beneficial with cost "
4975  << ore::NV("Cost", MinCost) << " >= "
4976  << ore::NV("Treshold", -SLPCostThreshold);
4977  });
4978  } else if (!Changed) {
4979  R.getORE()->emit([&]() {
4980  return OptimizationRemarkMissed(SV_NAME, "NotPossible", I0)
4981  << "Cannot SLP vectorize list: vectorization was impossible"
4982  << " with available vectorization factors";
4983  });
4984  }
4985  return Changed;
4986 }
4987 
4988 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4989  if (!I)
4990  return false;
4991 
4992  if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4993  return false;
4994 
4995  Value *P = I->getParent();
4996 
4997  // Vectorize in current basic block only.
4998  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4999  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
5000  if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
5001  return false;
5002 
5003  // Try to vectorize V.
5004  if (tryToVectorizePair(Op0, Op1, R))
5005  return true;
5006 
5007  auto *A = dyn_cast<BinaryOperator>(Op0);
5008  auto *B = dyn_cast<BinaryOperator>(Op1);
5009  // Try to skip B.
5010  if (B && B->hasOneUse()) {
5011  auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
5012  auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
5013  if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
5014  return true;
5015  if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
5016  return true;
5017  }
5018 
5019  // Try to skip A.
5020  if (A && A->hasOneUse()) {
5021  auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
5022  auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
5023  if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
5024  return true;
5025  if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
5026  return true;
5027  }
5028  return false;
5029 }
5030 
5031 /// Generate a shuffle mask to be used in a reduction tree.
5032 ///
5033 /// \param VecLen The length of the vector to be reduced.
5034 /// \param NumEltsToRdx The number of elements that should be reduced in the
5035 /// vector.
5036 /// \param IsPairwise Whether the reduction is a pairwise or splitting
5037 /// reduction. A pairwise reduction will generate a mask of
5038 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
5039 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
5040 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
5041 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
5042  bool IsPairwise, bool IsLeft,
5043  IRBuilder<> &Builder) {
5044  assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
5045 
5046  SmallVector<Constant *, 32> ShuffleMask(
5047  VecLen, UndefValue::get(Builder.getInt32Ty()));
5048 
5049  if (IsPairwise)
5050  // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
5051  for (unsigned i = 0; i != NumEltsToRdx; ++i)
5052  ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
5053  else
5054  // Move the upper half of the vector to the lower half.
5055  for (unsigned i = 0; i != NumEltsToRdx; ++i)
5056  ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
5057 
5058  return ConstantVector::get(ShuffleMask);
5059 }
5060 
5061 namespace {
5062 
5063 /// Model horizontal reductions.
5064 ///
5065 /// A horizontal reduction is a tree of reduction operations (currently add and
5066 /// fadd) that has operations that can be put into a vector as its leaf.
5067 /// For example, this tree:
5068 ///
5069 /// mul mul mul mul
5070 /// \ / \ /
5071 /// + +
5072 /// \ /
5073 /// +
5074 /// This tree has "mul" as its reduced values and "+" as its reduction
5075 /// operations. A reduction might be feeding into a store or a binary operation
5076 /// feeding a phi.
5077 /// ...
5078 /// \ /
5079 /// +
5080 /// |
5081 /// phi +=
5082 ///
5083 /// Or:
5084 /// ...
5085 /// \ /
5086 /// +
5087 /// |
5088 /// *p =
5089 ///
5090 class HorizontalReduction {
5091  using ReductionOpsType = SmallVector<Value *, 16>;
5092  using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
5093  ReductionOpsListType ReductionOps;
5094  SmallVector<Value *, 32> ReducedVals;
5095  // Use map vector to make stable output.
5097 
5098  /// Kind of the reduction data.
5099  enum ReductionKind {
5100  RK_None, /// Not a reduction.
5101  RK_Arithmetic, /// Binary reduction data.
5102  RK_Min, /// Minimum reduction data.
5103  RK_UMin, /// Unsigned minimum reduction data.
5104  RK_Max, /// Maximum reduction data.
5105  RK_UMax, /// Unsigned maximum reduction data.
5106  };
5107 
5108  /// Contains info about operation, like its opcode, left and right operands.
5109  class OperationData {
5110  /// Opcode of the instruction.
5111  unsigned Opcode = 0;
5112 
5113  /// Left operand of the reduction operation.
5114  Value *LHS = nullptr;
5115 
5116  /// Right operand of the reduction operation.
5117  Value *RHS = nullptr;
5118 
5119  /// Kind of the reduction operation.
5120  ReductionKind Kind = RK_None;
5121 
5122  /// True if float point min/max reduction has no NaNs.
5123  bool NoNaN = false;
5124 
5125  /// Checks if the reduction operation can be vectorized.
5126  bool isVectorizable() const {
5127  return LHS && RHS &&
5128  // We currently only support add/mul/logical && min/max reductions.
5129  ((Kind == RK_Arithmetic &&
5130  (Opcode == Instruction::Add || Opcode == Instruction::FAdd ||
5131  Opcode == Instruction::Mul || Opcode == Instruction::FMul ||
5132  Opcode == Instruction::And || Opcode == Instruction::Or ||
5133  Opcode == Instruction::Xor)) ||
5134  ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5135  (Kind == RK_Min || Kind == RK_Max)) ||
5136  (Opcode == Instruction::ICmp &&
5137  (Kind == RK_UMin || Kind == RK_UMax)));
5138  }
5139 
5140  /// Creates reduction operation with the current opcode.
5141  Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5142  assert(isVectorizable() &&
5143  "Expected add|fadd or min/max reduction operation.");
5144  Value *Cmp;
5145  switch (Kind) {
5146  case RK_Arithmetic:
5147  return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5148  Name);
5149  case RK_Min:
5150  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5151  : Builder.CreateFCmpOLT(LHS, RHS);
5152  break;
5153  case RK_Max:
5154  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5155  : Builder.CreateFCmpOGT(LHS, RHS);
5156  break;
5157  case RK_UMin:
5158  assert(Opcode == Instruction::ICmp && "Expected integer types.");
5159  Cmp = Builder.CreateICmpULT(LHS, RHS);
5160  break;
5161  case RK_UMax:
5162  assert(Opcode == Instruction::ICmp && "Expected integer types.");
5163  Cmp = Builder.CreateICmpUGT(LHS, RHS);
5164  break;
5165  case RK_None:
5166  llvm_unreachable("Unknown reduction operation.");
5167  }
5168  return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5169  }
5170 
5171  public:
5172  explicit OperationData() = default;
5173 
5174  /// Construction for reduced values. They are identified by opcode only and
5175  /// don't have associated LHS/RHS values.
5176  explicit OperationData(Value *V) {
5177  if (auto *I = dyn_cast<Instruction>(V))
5178  Opcode = I->getOpcode();
5179  }
5180 
5181  /// Constructor for reduction operations with opcode and its left and
5182  /// right operands.
5183  OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5184  bool NoNaN = false)
5185  : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5186  assert(Kind != RK_None && "One of the reduction operations is expected.");
5187  }
5188 
5189  explicit operator bool() const { return Opcode; }
5190 
5191  /// Get the index of the first operand.
5192  unsigned getFirstOperandIndex() const {
5193  assert(!!*this && "The opcode is not set.");
5194  switch (Kind) {
5195  case RK_Min:
5196  case RK_UMin:
5197  case RK_Max:
5198  case RK_UMax:
5199  return 1;
5200  case RK_Arithmetic:
5201  case RK_None:
5202  break;
5203  }
5204  return 0;
5205  }
5206 
5207  /// Total number of operands in the reduction operation.
5208  unsigned getNumberOfOperands() const {
5209  assert(Kind != RK_None && !!*this && LHS && RHS &&
5210  "Expected reduction operation.");
5211  switch (Kind) {
5212  case RK_Arithmetic:
5213  return 2;
5214  case RK_Min:
5215  case RK_UMin:
5216  case RK_Max:
5217  case RK_UMax:
5218  return 3;
5219  case RK_None:
5220  break;
5221  }
5222  llvm_unreachable("Reduction kind is not set");
5223  }
5224 
5225  /// Checks if the operation has the same parent as \p P.
5226  bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5227  assert(Kind != RK_None && !!*this && LHS && RHS &&
5228  "Expected reduction operation.");
5229  if (!IsRedOp)
5230  return I->getParent() == P;
5231  switch (Kind) {
5232  case RK_Arithmetic:
5233  // Arithmetic reduction operation must be used once only.
5234  return I->getParent() == P;
5235  case RK_Min:
5236  case RK_UMin:
5237  case RK_Max:
5238  case RK_UMax: {
5239  // SelectInst must be used twice while the condition op must have single
5240  // use only.
5241  auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5242  return I->getParent() == P && Cmp && Cmp->getParent() == P;
5243  }
5244  case RK_None:
5245  break;
5246  }
5247  llvm_unreachable("Reduction kind is not set");
5248  }
5249  /// Expected number of uses for reduction operations/reduced values.
5250  bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5251  assert(Kind != RK_None && !!*this && LHS && RHS &&
5252  "Expected reduction operation.");
5253  switch (Kind) {
5254  case RK_Arithmetic:
5255  return I->hasOneUse();
5256  case RK_Min:
5257  case RK_UMin:
5258  case RK_Max:
5259  case RK_UMax:
5260  return I->hasNUses(2) &&
5261  (!IsReductionOp ||
5262  cast<SelectInst>(I)->getCondition()->hasOneUse());
5263  case RK_None:
5264  break;
5265  }
5266  llvm_unreachable("Reduction kind is not set");
5267  }
5268 
5269  /// Initializes the list of reduction operations.
5270  void initReductionOps(ReductionOpsListType &ReductionOps) {
5271  assert(Kind != RK_None && !!*this && LHS && RHS &&
5272  "Expected reduction operation.");
5273  switch (Kind) {
5274  case RK_Arithmetic:
5275  ReductionOps.assign(1, ReductionOpsType());
5276  break;
5277  case RK_Min:
5278  case RK_UMin:
5279  case RK_Max:
5280  case RK_UMax:
5281  ReductionOps.assign(2, ReductionOpsType());
5282  break;
5283  case RK_None:
5284  llvm_unreachable("Reduction kind is not set");
5285  }
5286  }
5287  /// Add all reduction operations for the reduction instruction \p I.
5288  void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5289  assert(Kind != RK_None && !!*this && LHS && RHS &&
5290  "Expected reduction operation.");
5291  switch (Kind) {
5292  case RK_Arithmetic:
5293  ReductionOps[0].emplace_back(I);
5294  break;
5295  case RK_Min:
5296  case RK_UMin:
5297  case RK_Max:
5298  case RK_UMax:
5299  ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5300  ReductionOps[1].emplace_back(I);
5301  break;
5302  case RK_None:
5303  llvm_unreachable("Reduction kind is not set");
5304  }
5305  }
5306 
5307  /// Checks if instruction is associative and can be vectorized.
5308  bool isAssociative(Instruction *I) const {
5309  assert(Kind != RK_None && *this && LHS && RHS &&
5310  "Expected reduction operation.");
5311  switch (Kind) {
5312  case RK_Arithmetic:
5313  return I->isAssociative();
5314  case RK_Min:
5315  case RK_Max:
5316  return Opcode == Instruction::ICmp ||
5317  cast<Instruction>(I->getOperand(0))->isFast();
5318  case RK_UMin:
5319  case RK_UMax:
5320  assert(Opcode == Instruction::ICmp &&
5321  "Only integer compare operation is expected.");
5322  return true;
5323  case RK_None:
5324  break;
5325  }
5326  llvm_unreachable("Reduction kind is not set");
5327  }
5328 
5329  /// Checks if the reduction operation can be vectorized.
5330  bool isVectorizable(Instruction *I) const {
5331  return isVectorizable() && isAssociative(I);
5332  }
5333 
5334  /// Checks if two operation data are both a reduction op or both a reduced
5335  /// value.
5336  bool operator==(const OperationData &OD) {
5337  assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
5338  "One of the comparing operations is incorrect.");
5339  return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5340  }
5341  bool operator!=(const OperationData &OD) { return !(*this == OD); }
5342  void clear() {
5343  Opcode = 0;
5344  LHS = nullptr;
5345  RHS = nullptr;
5346  Kind = RK_None;
5347  NoNaN = false;
5348  }
5349 
5350  /// Get the opcode of the reduction operation.
5351  unsigned getOpcode() const {
5352  assert(isVectorizable() && "Expected vectorizable operation.");
5353  return Opcode;
5354  }
5355 
5356  /// Get kind of reduction data.
5357  ReductionKind getKind() const { return Kind; }
5358  Value *getLHS() const { return LHS; }
5359  Value *getRHS() const { return RHS; }
5360  Type *getConditionType() const {
5361  switch (Kind) {
5362  case RK_Arithmetic:
5363  return nullptr;
5364  case RK_Min:
5365  case RK_Max:
5366  case RK_UMin:
5367  case RK_UMax:
5368  return CmpInst::makeCmpResultType(LHS->getType());
5369  case RK_None:
5370  break;
5371  }
5372  llvm_unreachable("Reduction kind is not set");
5373  }
5374 
5375  /// Creates reduction operation with the current opcode with the IR flags
5376  /// from \p ReductionOps.
5377  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5378  const ReductionOpsListType &ReductionOps) const {
5379  assert(isVectorizable() &&
5380  "Expected add|fadd or min/max reduction operation.");
5381  auto *Op = createOp(Builder, Name);
5382  switch (Kind) {
5383  case RK_Arithmetic:
5384  propagateIRFlags(Op, ReductionOps[0]);
5385  return Op;
5386  case RK_Min:
5387  case RK_Max:
5388  case RK_UMin:
5389  case RK_UMax:
5390  if (auto *SI = dyn_cast<SelectInst>(Op))
5391  propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5392  propagateIRFlags(Op, ReductionOps[1]);
5393  return Op;
5394  case RK_None:
5395  break;
5396  }
5397  llvm_unreachable("Unknown reduction operation.");
5398  }
5399  /// Creates reduction operation with the current opcode with the IR flags
5400  /// from \p I.
5401  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5402  Instruction *I) const {
5403  assert(isVectorizable() &&
5404  "Expected add|fadd or min/max reduction operation.");
5405  auto *Op = createOp(Builder, Name);
5406  switch (Kind) {
5407  case RK_Arithmetic:
5408  propagateIRFlags(Op, I);
5409  return Op;
5410  case RK_Min:
5411  case RK_Max:
5412  case RK_UMin:
5413  case RK_UMax:
5414  if (auto *SI = dyn_cast<SelectInst>(Op)) {
5415  propagateIRFlags(SI->getCondition(),
5416  cast<SelectInst>(I)->getCondition());
5417  }
5418  propagateIRFlags(Op, I);
5419  return Op;
5420  case RK_None:
5421  break;
5422  }
5423  llvm_unreachable("Unknown reduction operation.");
5424  }
5425 
5426  TargetTransformInfo::ReductionFlags getFlags() const {
5428  Flags.NoNaN = NoNaN;
5429  switch (Kind) {
5430  case RK_Arithmetic:
5431  break;
5432  case RK_Min:
5433  Flags.IsSigned = Opcode == Instruction::ICmp;
5434  Flags.IsMaxOp = false;
5435  break;
5436  case RK_Max:
5437  Flags.IsSigned = Opcode == Instruction::ICmp;
5438  Flags.IsMaxOp = true;
5439  break;
5440  case RK_UMin:
5441  Flags.IsSigned = false;
5442  Flags.IsMaxOp = false;
5443  break;
5444  case RK_UMax:
5445  Flags.IsSigned = false;
5446  Flags.IsMaxOp = true;
5447  break;
5448  case RK_None:
5449  llvm_unreachable("Reduction kind is not set");
5450  }
5451  return Flags;
5452  }
5453  };
5454 
5455  WeakTrackingVH ReductionRoot;
5456 
5457  /// The operation data of the reduction operation.
5458  OperationData ReductionData;
5459 
5460  /// The operation data of the values we perform a reduction on.
5461  OperationData ReducedValueData;
5462 
5463  /// Should we model this reduction as a pairwise reduction tree or a tree that
5464  /// splits the vector in halves and adds those halves.
5465  bool IsPairwiseReduction = false;
5466 
5467  /// Checks if the ParentStackElem.first should be marked as a reduction
5468  /// operation with an extra argument or as extra argument itself.
5469  void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5470  Value *ExtraArg) {
5471  if (ExtraArgs.count(ParentStackElem.first)) {
5472  ExtraArgs[ParentStackElem.first] = nullptr;
5473  // We ran into something like:
5474  // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5475  // The whole ParentStackElem.first should be considered as an extra value
5476  // in this case.
5477  // Do not perform analysis of remaining operands of ParentStackElem.first
5478  // instruction, this whole instruction is an extra argument.
5479  ParentStackElem.second = ParentStackElem.first->getNumOperands();
5480  } else {
5481  // We ran into something like:
5482  // ParentStackElem.first += ... + ExtraArg + ...
5483  ExtraArgs[ParentStackElem.first] = ExtraArg;
5484  }
5485  }
5486 
5487  static OperationData getOperationData(Value *V) {
5488  if (!V)
5489  return OperationData();
5490 
5491  Value *LHS;
5492  Value *RHS;
5493  if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5494  return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5495  RK_Arithmetic);
5496  }
5497  if (auto *Select = dyn_cast<SelectInst>(V)) {
5498  // Look for a min/max pattern.
5499  if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5500  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5501  } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5502  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5503  } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5504  m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5505  return OperationData(
5506  Instruction::FCmp, LHS, RHS, RK_Min,
5507  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5508  } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5509  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5510  } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5511  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5512  } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5513  m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5514  return OperationData(
5515  Instruction::FCmp, LHS, RHS, RK_Max,
5516  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5517  } else {
5518  // Try harder: look for min/max pattern based on instructions producing
5519  // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2).
5520  // During the intermediate stages of SLP, it's very common to have
5521  // pattern like this (since optimizeGatherSequence is run only once
5522  // at the end):
5523  // %1 = extractelement <2 x i32> %a, i32 0
5524  // %2 = extractelement <2 x i32> %a, i32 1
5525  // %cond = icmp sgt i32 %1, %2
5526  // %3 = extractelement <2 x i32> %a, i32 0
5527  // %4 = extractelement <2 x i32> %a, i32 1
5528  // %select = select i1 %cond, i32 %3, i32 %4
5529  CmpInst::Predicate Pred;
5530  Instruction *L1;
5531  Instruction *L2;
5532 
5533  LHS = Select->getTrueValue();
5534  RHS = Select->getFalseValue();
5535  Value *Cond = Select->getCondition();
5536 
5537  // TODO: Support inverse predicates.
5538  if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) {
5539  if (!isa<ExtractElementInst>(RHS) ||
5540  !L2->isIdenticalTo(cast<Instruction>(RHS)))
5541  return OperationData(V);
5542  } else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) {
5543  if (!isa<ExtractElementInst>(LHS) ||
5544  !L1->isIdenticalTo(cast<Instruction>(LHS)))
5545  return OperationData(V);
5546  } else {
5547  if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS))
5548  return OperationData(V);
5549  if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) ||
5550  !L1->isIdenticalTo(cast<Instruction>(LHS)) ||
5551  !L2->isIdenticalTo(cast<Instruction>(RHS)))
5552  return OperationData(V);
5553  }
5554  switch (Pred) {
5555  default:
5556  return OperationData(V);
5557 
5558  case CmpInst::ICMP_ULT:
5559  case CmpInst::ICMP_ULE:
5560  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5561 
5562  case CmpInst::ICMP_SLT:
5563  case CmpInst::ICMP_SLE:
5564  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5565 
5566  case CmpInst::FCMP_OLT:
5567  case CmpInst::FCMP_OLE:
5568  case CmpInst::FCMP_ULT:
5569  case CmpInst::FCMP_ULE:
5570  return OperationData(Instruction::FCmp, LHS, RHS, RK_Min,
5571  cast<Instruction>(Cond)->hasNoNaNs());
5572 
5573  case CmpInst::ICMP_UGT:
5574  case CmpInst::ICMP_UGE:
5575  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5576 
5577  case CmpInst::ICMP_SGT:
5578  case CmpInst::ICMP_SGE:
5579  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5580 
5581  case CmpInst::FCMP_OGT:
5582  case CmpInst::FCMP_OGE:
5583  case CmpInst::FCMP_UGT:
5584  case CmpInst::FCMP_UGE:
5585  return OperationData(Instruction::FCmp, LHS, RHS, RK_Max,
5586  cast<Instruction>(Cond)->hasNoNaNs());
5587  }
5588  }
5589  }
5590  return OperationData(V);
5591  }
5592 
5593 public:
5594  HorizontalReduction() = default;
5595 
5596  /// Try to find a reduction tree.
5597  bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5598  assert((!Phi || is_contained(Phi->operands(), B)) &&
5599  "Thi phi needs to use the binary operator");
5600 
5601  ReductionData = getOperationData(B);
5602 
5603  // We could have a initial reductions that is not an add.
5604  // r *= v1 + v2 + v3 + v4
5605  // In such a case start looking for a tree rooted in the first '+'.
5606  if (Phi) {
5607  if (ReductionData.getLHS() == Phi) {
5608  Phi = nullptr;
5609  B = dyn_cast<Instruction>(ReductionData.getRHS());
5610  ReductionData = getOperationData(B);
5611  } else if (ReductionData.getRHS() == Phi) {
5612  Phi = nullptr;
5613  B = dyn_cast<Instruction>(ReductionData.getLHS());
5614  ReductionData = getOperationData(B);
5615  }
5616  }
5617 
5618  if (!ReductionData.isVectorizable(B))
5619  return false;
5620 
5621  Type *Ty = B->getType();
5622  if (!isValidElementType(Ty))
5623  return false;
5624  if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy())
5625  return false;
5626 
5627  ReducedValueData.clear();
5628  ReductionRoot = B;
5629 
5630  // Post order traverse the reduction tree starting at B. We only handle true
5631  // trees containing only binary operators.
5633  Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5634  ReductionData.initReductionOps(ReductionOps);
5635  while (!Stack.empty()) {
5636  Instruction *TreeN = Stack.back().first;
5637  unsigned EdgeToVist = Stack.back().second++;
5638  OperationData OpData = getOperationData(TreeN);
5639  bool IsReducedValue = OpData != ReductionData;
5640 
5641  // Postorder vist.
5642  if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5643  if (IsReducedValue)
5644  ReducedVals.push_back(TreeN);
5645  else {
5646  auto I = ExtraArgs.find(TreeN);
5647  if (I != ExtraArgs.end() && !I->second) {
5648  // Check if TreeN is an extra argument of its parent operation.
5649  if (Stack.size() <= 1) {
5650  // TreeN can't be an extra argument as it is a root reduction
5651  // operation.
5652  return false;
5653  }
5654  // Yes, TreeN is an extra argument, do not add it to a list of
5655  // reduction operations.
5656  // Stack[Stack.size() - 2] always points to the parent operation.
5657  markExtraArg(Stack[Stack.size() - 2], TreeN);
5658  ExtraArgs.erase(TreeN);
5659  } else
5660  ReductionData.addReductionOps(TreeN, ReductionOps);
5661  }
5662  // Retract.
5663  Stack.pop_back();
5664  continue;
5665  }
5666 
5667  // Visit left or right.
5668  Value *NextV = TreeN->getOperand(EdgeToVist);
5669  if (NextV != Phi) {
5670  auto *I = dyn_cast<Instruction>(NextV);
5671  OpData = getOperationData(I);
5672  // Continue analysis if the next operand is a reduction operation or
5673  // (possibly) a reduced value. If the reduced value opcode is not set,
5674  // the first met operation != reduction operation is considered as the
5675  // reduced value class.
5676  if (I && (!ReducedValueData || OpData == ReducedValueData ||
5677  OpData == ReductionData)) {
5678  const bool IsReductionOperation = OpData == ReductionData;
5679  // Only handle trees in the current basic block.
5680  if (!ReductionData.hasSameParent(I, B->getParent(),
5681  IsReductionOperation)) {
5682  // I is an extra argument for TreeN (its parent operation).
5683  markExtraArg(Stack.back(), I);
5684  continue;
5685  }
5686 
5687  // Each tree node needs to have minimal number of users except for the
5688  // ultimate reduction.
5689  if (!ReductionData.hasRequiredNumberOfUses(I,
5690  OpData == ReductionData) &&
5691  I != B) {
5692  // I is an extra argument for TreeN (its parent operation).
5693  markExtraArg(Stack.back(), I);
5694  continue;
5695  }
5696 
5697  if (IsReductionOperation) {
5698  // We need to be able to reassociate the reduction operations.
5699  if (!OpData.isAssociative(I)) {
5700  // I is an extra argument for TreeN (its parent operation).
5701  markExtraArg(Stack.back(), I);
5702  continue;
5703  }
5704  } else if (ReducedValueData &&
5705  ReducedValueData != OpData) {
5706  // Make sure that the opcodes of the operations that we are going to
5707  // reduce match.
5708  // I is an extra argument for TreeN (its parent operation).
5709  markExtraArg(Stack.back(), I);
5710  continue;
5711  } else if (!ReducedValueData)
5712  ReducedValueData = OpData;
5713 
5714  Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5715  continue;
5716  }
5717  }
5718  // NextV is an extra argument for TreeN (its parent operation).
5719  markExtraArg(Stack.back(), NextV);
5720  }
5721  return true;
5722  }
5723 
5724  /// Attempt to vectorize the tree found by
5725  /// matchAssociativeReduction.
5726  bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5727  if (ReducedVals.empty())
5728  return false;
5729 
5730  // If there is a sufficient number of reduction values, reduce
5731  // to a nearby power-of-2. Can safely generate oversized
5732  // vectors and rely on the backend to split them to legal sizes.
5733  unsigned NumReducedVals = ReducedVals.size();
5734  if (NumReducedVals < 4)
5735  return false;
5736 
5737  unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5738 
5739  Value *VectorizedTree = nullptr;
5740  IRBuilder<> Builder(cast<Instruction>(ReductionRoot));
5741  FastMathFlags Unsafe;
5742  Unsafe.setFast();
5743  Builder.setFastMathFlags(Unsafe);
5744  unsigned i = 0;
5745 
5746  BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5747  // The same extra argument may be used several time, so log each attempt
5748  // to use it.
5749  for (auto &Pair : ExtraArgs) {
5750  assert(Pair.first && "DebugLoc must be set.");
5751  ExternallyUsedValues[Pair.second].push_back(Pair.first);
5752  }
5753  // The reduction root is used as the insertion point for new instructions,
5754  // so set it as externally used to prevent it from being deleted.
5755  ExternallyUsedValues[ReductionRoot];
5756  SmallVector<Value *, 16> IgnoreList;
5757  for (auto &V : ReductionOps)
5758  IgnoreList.append(V.begin(), V.end());
5759  while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5760  auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5761  V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5762  Optional<ArrayRef<unsigned>> Order = V.bestOrder();
5763  // TODO: Handle orders of size less than number of elements in the vector.
5764  if (Order && Order->size() == VL.size()) {
5765  // TODO: reorder tree nodes without tree rebuilding.
5766  SmallVector<Value *, 4> ReorderedOps(VL.size());
5767  llvm::transform(*Order, ReorderedOps.begin(),
5768  [VL](const unsigned Idx) { return VL[Idx]; });
5769  V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList);
5770  }
5771  if (V.isTreeTinyAndNotFullyVectorizable())
5772  break;
5773 
5774  V.computeMinimumValueSizes();
5775 
5776  // Estimate cost.
5777  int TreeCost = V.getTreeCost();
5778  int ReductionCost = getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5779  int Cost = TreeCost + ReductionCost;
5780  if (Cost >= -SLPCostThreshold) {
5781  V.getORE()->emit([&]() {
5782  return OptimizationRemarkMissed(
5783  SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5784  << "Vectorizing horizontal reduction is possible"
5785  << "but not beneficial with cost "
5786  << ore::NV("Cost", Cost) << " and threshold "
5787  << ore::NV("Threshold", -SLPCostThreshold);
5788  });
5789  break;
5790  }
5791 
5792  LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
5793  << Cost << ". (HorRdx)\n");
5794  V.getORE()->emit([&]() {
5795  return OptimizationRemark(
5796  SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5797  << "Vectorized horizontal reduction with cost "
5798  << ore::NV("Cost", Cost) << " and with tree size "
5799  << ore::NV("TreeSize", V.getTreeSize());
5800  });
5801 
5802  // Vectorize a tree.
5803  DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5804  Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5805 
5806  // Emit a reduction.
5807  Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
5808  Value *ReducedSubTree =
5809  emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5810  if (VectorizedTree) {
5811  Builder.SetCurrentDebugLocation(Loc);
5812  OperationData VectReductionData(ReductionData.getOpcode(),
5813  VectorizedTree, ReducedSubTree,
5814  ReductionData.getKind());
5815  VectorizedTree =
5816  VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5817  } else
5818  VectorizedTree = ReducedSubTree;
5819  i += ReduxWidth;
5820  ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5821  }
5822 
5823  if (VectorizedTree) {
5824  // Finish the reduction.
5825  for (; i < NumReducedVals; ++i) {
5826  auto *I = cast<Instruction>(ReducedVals[i]);
5827  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5828  OperationData VectReductionData(ReductionData.getOpcode(),
5829  VectorizedTree, I,
5830  ReductionData.getKind());
5831  VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5832  }
5833  for (auto &Pair : ExternallyUsedValues) {
5834  // Add each externally used value to the final reduction.
5835  for (auto *I : Pair.second) {
5836  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5837  OperationData VectReductionData(ReductionData.getOpcode(),
5838  VectorizedTree, Pair.first,
5839  ReductionData.getKind());
5840  VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5841  }
5842  }
5843  // Update users.
5844  ReductionRoot->replaceAllUsesWith(VectorizedTree);
5845  }
5846  return VectorizedTree != nullptr;
5847  }
5848 
5849  unsigned numReductionValues() const {
5850  return ReducedVals.size();
5851  }
5852 
5853 private:
5854  /// Calculate the cost of a reduction.
5855  int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5856  unsigned ReduxWidth) {
5857  Type *ScalarTy = FirstReducedVal->getType();
5858  Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5859 
5860  int PairwiseRdxCost;
5861  int SplittingRdxCost;
5862  switch (ReductionData.getKind()) {
5863  case RK_Arithmetic:
5864  PairwiseRdxCost =
5865  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5866  /*IsPairwiseForm=*/true);
5867  SplittingRdxCost =
5868  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,