LLVM  9.0.0svn
SLPVectorizer.cpp
Go to the documentation of this file.
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10 // stores that can be put together into vector-stores. Next, it attempts to
11 // construct vectorizable tree using the use-def chains. If a profitable tree
12 // was found, the SLP vectorizer performs vectorization on the tree.
13 //
14 // The pass is inspired by the work described in the paper:
15 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16 //
17 //===----------------------------------------------------------------------===//
18 
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/DenseSet.h"
23 #include "llvm/ADT/MapVector.h"
24 #include "llvm/ADT/None.h"
25 #include "llvm/ADT/Optional.h"
27 #include "llvm/ADT/STLExtras.h"
28 #include "llvm/ADT/SetVector.h"
29 #include "llvm/ADT/SmallPtrSet.h"
30 #include "llvm/ADT/SmallSet.h"
31 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/ADT/Statistic.h"
33 #include "llvm/ADT/iterator.h"
40 #include "llvm/Analysis/LoopInfo.h"
49 #include "llvm/IR/Attributes.h"
50 #include "llvm/IR/BasicBlock.h"
51 #include "llvm/IR/Constant.h"
52 #include "llvm/IR/Constants.h"
53 #include "llvm/IR/DataLayout.h"
54 #include "llvm/IR/DebugLoc.h"
55 #include "llvm/IR/DerivedTypes.h"
56 #include "llvm/IR/Dominators.h"
57 #include "llvm/IR/Function.h"
58 #include "llvm/IR/IRBuilder.h"
59 #include "llvm/IR/InstrTypes.h"
60 #include "llvm/IR/Instruction.h"
61 #include "llvm/IR/Instructions.h"
62 #include "llvm/IR/IntrinsicInst.h"
63 #include "llvm/IR/Intrinsics.h"
64 #include "llvm/IR/Module.h"
65 #include "llvm/IR/NoFolder.h"
66 #include "llvm/IR/Operator.h"
67 #include "llvm/IR/PassManager.h"
68 #include "llvm/IR/PatternMatch.h"
69 #include "llvm/IR/Type.h"
70 #include "llvm/IR/Use.h"
71 #include "llvm/IR/User.h"
72 #include "llvm/IR/Value.h"
73 #include "llvm/IR/ValueHandle.h"
74 #include "llvm/IR/Verifier.h"
75 #include "llvm/Pass.h"
76 #include "llvm/Support/Casting.h"
78 #include "llvm/Support/Compiler.h"
80 #include "llvm/Support/Debug.h"
83 #include "llvm/Support/KnownBits.h"
88 #include <algorithm>
89 #include <cassert>
90 #include <cstdint>
91 #include <iterator>
92 #include <memory>
93 #include <set>
94 #include <string>
95 #include <tuple>
96 #include <utility>
97 #include <vector>
98 
99 using namespace llvm;
100 using namespace llvm::PatternMatch;
101 using namespace slpvectorizer;
102 
103 #define SV_NAME "slp-vectorizer"
104 #define DEBUG_TYPE "SLP"
105 
106 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
107 
108 static cl::opt<int>
109  SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
110  cl::desc("Only vectorize if you gain more than this "
111  "number "));
112 
113 static cl::opt<bool>
114 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
115  cl::desc("Attempt to vectorize horizontal reductions"));
116 
118  "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
119  cl::desc(
120  "Attempt to vectorize horizontal reductions feeding into a store"));
121 
122 static cl::opt<int>
123 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
124  cl::desc("Attempt to vectorize for this register size in bits"));
125 
126 /// Limits the size of scheduling regions in a block.
127 /// It avoid long compile times for _very_ large blocks where vector
128 /// instructions are spread over a wide range.
129 /// This limit is way higher than needed by real-world functions.
130 static cl::opt<int>
131 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
132  cl::desc("Limit the size of the SLP scheduling region per block"));
133 
135  "slp-min-reg-size", cl::init(128), cl::Hidden,
136  cl::desc("Attempt to vectorize for this register size in bits"));
137 
139  "slp-recursion-max-depth", cl::init(12), cl::Hidden,
140  cl::desc("Limit the recursion depth when building a vectorizable tree"));
141 
143  "slp-min-tree-size", cl::init(3), cl::Hidden,
144  cl::desc("Only vectorize small trees if they are fully vectorizable"));
145 
146 static cl::opt<bool>
147  ViewSLPTree("view-slp-tree", cl::Hidden,
148  cl::desc("Display the SLP trees with Graphviz"));
149 
150 // Limit the number of alias checks. The limit is chosen so that
151 // it has no negative effect on the llvm benchmarks.
152 static const unsigned AliasedCheckLimit = 10;
153 
154 // Another limit for the alias checks: The maximum distance between load/store
155 // instructions where alias checks are done.
156 // This limit is useful for very large basic blocks.
157 static const unsigned MaxMemDepDistance = 160;
158 
159 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
160 /// regions to be handled.
161 static const int MinScheduleRegionSize = 16;
162 
163 /// Predicate for the element types that the SLP vectorizer supports.
164 ///
165 /// The most important thing to filter here are types which are invalid in LLVM
166 /// vectors. We also filter target specific types which have absolutely no
167 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
168 /// avoids spending time checking the cost model and realizing that they will
169 /// be inevitably scalarized.
170 static bool isValidElementType(Type *Ty) {
171  return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
172  !Ty->isPPC_FP128Ty();
173 }
174 
175 /// \returns true if all of the instructions in \p VL are in the same block or
176 /// false otherwise.
178  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
179  if (!I0)
180  return false;
181  BasicBlock *BB = I0->getParent();
182  for (int i = 1, e = VL.size(); i < e; i++) {
183  Instruction *I = dyn_cast<Instruction>(VL[i]);
184  if (!I)
185  return false;
186 
187  if (BB != I->getParent())
188  return false;
189  }
190  return true;
191 }
192 
193 /// \returns True if all of the values in \p VL are constants.
195  for (Value *i : VL)
196  if (!isa<Constant>(i))
197  return false;
198  return true;
199 }
200 
201 /// \returns True if all of the values in \p VL are identical.
202 static bool isSplat(ArrayRef<Value *> VL) {
203  for (unsigned i = 1, e = VL.size(); i < e; ++i)
204  if (VL[i] != VL[0])
205  return false;
206  return true;
207 }
208 
209 /// Checks if the vector of instructions can be represented as a shuffle, like:
210 /// %x0 = extractelement <4 x i8> %x, i32 0
211 /// %x3 = extractelement <4 x i8> %x, i32 3
212 /// %y1 = extractelement <4 x i8> %y, i32 1
213 /// %y2 = extractelement <4 x i8> %y, i32 2
214 /// %x0x0 = mul i8 %x0, %x0
215 /// %x3x3 = mul i8 %x3, %x3
216 /// %y1y1 = mul i8 %y1, %y1
217 /// %y2y2 = mul i8 %y2, %y2
218 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
219 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
220 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
221 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
222 /// ret <4 x i8> %ins4
223 /// can be transformed into:
224 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
225 /// i32 6>
226 /// %2 = mul <4 x i8> %1, %1
227 /// ret <4 x i8> %2
228 /// We convert this initially to something like:
229 /// %x0 = extractelement <4 x i8> %x, i32 0
230 /// %x3 = extractelement <4 x i8> %x, i32 3
231 /// %y1 = extractelement <4 x i8> %y, i32 1
232 /// %y2 = extractelement <4 x i8> %y, i32 2
233 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
234 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
235 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
236 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
237 /// %5 = mul <4 x i8> %4, %4
238 /// %6 = extractelement <4 x i8> %5, i32 0
239 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
240 /// %7 = extractelement <4 x i8> %5, i32 1
241 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
242 /// %8 = extractelement <4 x i8> %5, i32 2
243 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
244 /// %9 = extractelement <4 x i8> %5, i32 3
245 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
246 /// ret <4 x i8> %ins4
247 /// InstCombiner transforms this into a shuffle and vector mul
248 /// TODO: Can we split off and reuse the shuffle mask detection from
249 /// TargetTransformInfo::getInstructionThroughput?
252  auto *EI0 = cast<ExtractElementInst>(VL[0]);
253  unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
254  Value *Vec1 = nullptr;
255  Value *Vec2 = nullptr;
256  enum ShuffleMode { Unknown, Select, Permute };
257  ShuffleMode CommonShuffleMode = Unknown;
258  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
259  auto *EI = cast<ExtractElementInst>(VL[I]);
260  auto *Vec = EI->getVectorOperand();
261  // All vector operands must have the same number of vector elements.
262  if (Vec->getType()->getVectorNumElements() != Size)
263  return None;
264  auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
265  if (!Idx)
266  return None;
267  // Undefined behavior if Idx is negative or >= Size.
268  if (Idx->getValue().uge(Size))
269  continue;
270  unsigned IntIdx = Idx->getValue().getZExtValue();
271  // We can extractelement from undef vector.
272  if (isa<UndefValue>(Vec))
273  continue;
274  // For correct shuffling we have to have at most 2 different vector operands
275  // in all extractelement instructions.
276  if (!Vec1 || Vec1 == Vec)
277  Vec1 = Vec;
278  else if (!Vec2 || Vec2 == Vec)
279  Vec2 = Vec;
280  else
281  return None;
282  if (CommonShuffleMode == Permute)
283  continue;
284  // If the extract index is not the same as the operation number, it is a
285  // permutation.
286  if (IntIdx != I) {
287  CommonShuffleMode = Permute;
288  continue;
289  }
290  CommonShuffleMode = Select;
291  }
292  // If we're not crossing lanes in different vectors, consider it as blending.
293  if (CommonShuffleMode == Select && Vec2)
295  // If Vec2 was never used, we have a permutation of a single vector, otherwise
296  // we have permutation of 2 vectors.
299 }
300 
301 namespace {
302 
303 /// Main data required for vectorization of instructions.
304 struct InstructionsState {
305  /// The very first instruction in the list with the main opcode.
306  Value *OpValue = nullptr;
307 
308  /// The main/alternate instruction.
309  Instruction *MainOp = nullptr;
310  Instruction *AltOp = nullptr;
311 
312  /// The main/alternate opcodes for the list of instructions.
313  unsigned getOpcode() const {
314  return MainOp ? MainOp->getOpcode() : 0;
315  }
316 
317  unsigned getAltOpcode() const {
318  return AltOp ? AltOp->getOpcode() : 0;
319  }
320 
321  /// Some of the instructions in the list have alternate opcodes.
322  bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
323 
324  bool isOpcodeOrAlt(Instruction *I) const {
325  unsigned CheckedOpcode = I->getOpcode();
326  return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
327  }
328 
329  InstructionsState() = delete;
330  InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
331  : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
332 };
333 
334 } // end anonymous namespace
335 
336 /// Chooses the correct key for scheduling data. If \p Op has the same (or
337 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
338 /// OpValue.
339 static Value *isOneOf(const InstructionsState &S, Value *Op) {
340  auto *I = dyn_cast<Instruction>(Op);
341  if (I && S.isOpcodeOrAlt(I))
342  return Op;
343  return S.OpValue;
344 }
345 
346 /// \returns analysis of the Instructions in \p VL described in
347 /// InstructionsState, the Opcode that we suppose the whole list
348 /// could be vectorized even if its structure is diverse.
349 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
350  unsigned BaseIndex = 0) {
351  // Make sure these are all Instructions.
352  if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
353  return InstructionsState(VL[BaseIndex], nullptr, nullptr);
354 
355  bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
356  bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
357  unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
358  unsigned AltOpcode = Opcode;
359  unsigned AltIndex = BaseIndex;
360 
361  // Check for one alternate opcode from another BinaryOperator.
362  // TODO - generalize to support all operators (types, calls etc.).
363  for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
364  unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
365  if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
366  if (InstOpcode == Opcode || InstOpcode == AltOpcode)
367  continue;
368  if (Opcode == AltOpcode) {
369  AltOpcode = InstOpcode;
370  AltIndex = Cnt;
371  continue;
372  }
373  } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
374  Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
375  Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
376  if (Ty0 == Ty1) {
377  if (InstOpcode == Opcode || InstOpcode == AltOpcode)
378  continue;
379  if (Opcode == AltOpcode) {
380  AltOpcode = InstOpcode;
381  AltIndex = Cnt;
382  continue;
383  }
384  }
385  } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
386  continue;
387  return InstructionsState(VL[BaseIndex], nullptr, nullptr);
388  }
389 
390  return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
391  cast<Instruction>(VL[AltIndex]));
392 }
393 
394 /// \returns true if all of the values in \p VL have the same type or false
395 /// otherwise.
397  Type *Ty = VL[0]->getType();
398  for (int i = 1, e = VL.size(); i < e; i++)
399  if (VL[i]->getType() != Ty)
400  return false;
401 
402  return true;
403 }
404 
405 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
407  unsigned Opcode = E->getOpcode();
408  assert((Opcode == Instruction::ExtractElement ||
409  Opcode == Instruction::ExtractValue) &&
410  "Expected extractelement or extractvalue instruction.");
411  if (Opcode == Instruction::ExtractElement) {
412  auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
413  if (!CI)
414  return None;
415  return CI->getZExtValue();
416  }
417  ExtractValueInst *EI = cast<ExtractValueInst>(E);
418  if (EI->getNumIndices() != 1)
419  return None;
420  return *EI->idx_begin();
421 }
422 
423 /// \returns True if in-tree use also needs extract. This refers to
424 /// possible scalar operand in vectorized instruction.
426  TargetLibraryInfo *TLI) {
427  unsigned Opcode = UserInst->getOpcode();
428  switch (Opcode) {
429  case Instruction::Load: {
430  LoadInst *LI = cast<LoadInst>(UserInst);
431  return (LI->getPointerOperand() == Scalar);
432  }
433  case Instruction::Store: {
434  StoreInst *SI = cast<StoreInst>(UserInst);
435  return (SI->getPointerOperand() == Scalar);
436  }
437  case Instruction::Call: {
438  CallInst *CI = cast<CallInst>(UserInst);
440  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
441  return (CI->getArgOperand(1) == Scalar);
442  }
444  }
445  default:
446  return false;
447  }
448 }
449 
450 /// \returns the AA location that is being access by the instruction.
452  if (StoreInst *SI = dyn_cast<StoreInst>(I))
453  return MemoryLocation::get(SI);
454  if (LoadInst *LI = dyn_cast<LoadInst>(I))
455  return MemoryLocation::get(LI);
456  return MemoryLocation();
457 }
458 
459 /// \returns True if the instruction is not a volatile or atomic load/store.
460 static bool isSimple(Instruction *I) {
461  if (LoadInst *LI = dyn_cast<LoadInst>(I))
462  return LI->isSimple();
463  if (StoreInst *SI = dyn_cast<StoreInst>(I))
464  return SI->isSimple();
465  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
466  return !MI->isVolatile();
467  return true;
468 }
469 
470 namespace llvm {
471 
472 namespace slpvectorizer {
473 
474 /// Bottom Up SLP Vectorizer.
475 class BoUpSLP {
476 public:
483 
487  const DataLayout *DL, OptimizationRemarkEmitter *ORE)
488  : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
489  DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
490  CodeMetrics::collectEphemeralValues(F, AC, EphValues);
491  // Use the vector register size specified by the target unless overridden
492  // by a command-line option.
493  // TODO: It would be better to limit the vectorization factor based on
494  // data type rather than just register size. For example, x86 AVX has
495  // 256-bit registers, but it does not support integer operations
496  // at that width (that requires AVX2).
497  if (MaxVectorRegSizeOption.getNumOccurrences())
498  MaxVecRegSize = MaxVectorRegSizeOption;
499  else
500  MaxVecRegSize = TTI->getRegisterBitWidth(true);
501 
502  if (MinVectorRegSizeOption.getNumOccurrences())
503  MinVecRegSize = MinVectorRegSizeOption;
504  else
505  MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
506  }
507 
508  /// Vectorize the tree that starts with the elements in \p VL.
509  /// Returns the vectorized root.
510  Value *vectorizeTree();
511 
512  /// Vectorize the tree but with the list of externally used values \p
513  /// ExternallyUsedValues. Values in this MapVector can be replaced but the
514  /// generated extractvalue instructions.
515  Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
516 
517  /// \returns the cost incurred by unwanted spills and fills, caused by
518  /// holding live values over call sites.
519  int getSpillCost();
520 
521  /// \returns the vectorization cost of the subtree that starts at \p VL.
522  /// A negative number means that this is profitable.
523  int getTreeCost();
524 
525  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
526  /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
527  void buildTree(ArrayRef<Value *> Roots,
528  ArrayRef<Value *> UserIgnoreLst = None);
529 
530  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
531  /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
532  /// into account (anf updating it, if required) list of externally used
533  /// values stored in \p ExternallyUsedValues.
534  void buildTree(ArrayRef<Value *> Roots,
535  ExtraValueToDebugLocsMap &ExternallyUsedValues,
536  ArrayRef<Value *> UserIgnoreLst = None);
537 
538  /// Clear the internal data structures that are created by 'buildTree'.
539  void deleteTree() {
540  VectorizableTree.clear();
541  ScalarToTreeEntry.clear();
542  MustGather.clear();
543  ExternalUses.clear();
544  NumOpsWantToKeepOrder.clear();
545  NumOpsWantToKeepOriginalOrder = 0;
546  for (auto &Iter : BlocksSchedules) {
547  BlockScheduling *BS = Iter.second.get();
548  BS->clear();
549  }
550  MinBWs.clear();
551  }
552 
553  unsigned getTreeSize() const { return VectorizableTree.size(); }
554 
555  /// Perform LICM and CSE on the newly generated gather sequences.
556  void optimizeGatherSequence();
557 
558  /// \returns The best order of instructions for vectorization.
560  auto I = std::max_element(
561  NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(),
562  [](const decltype(NumOpsWantToKeepOrder)::value_type &D1,
563  const decltype(NumOpsWantToKeepOrder)::value_type &D2) {
564  return D1.second < D2.second;
565  });
566  if (I == NumOpsWantToKeepOrder.end() ||
567  I->getSecond() <= NumOpsWantToKeepOriginalOrder)
568  return None;
569 
570  return makeArrayRef(I->getFirst());
571  }
572 
573  /// \return The vector element size in bits to use when vectorizing the
574  /// expression tree ending at \p V. If V is a store, the size is the width of
575  /// the stored value. Otherwise, the size is the width of the largest loaded
576  /// value reaching V. This method is used by the vectorizer to calculate
577  /// vectorization factors.
578  unsigned getVectorElementSize(Value *V);
579 
580  /// Compute the minimum type sizes required to represent the entries in a
581  /// vectorizable tree.
583 
584  // \returns maximum vector register size as set by TTI or overridden by cl::opt.
585  unsigned getMaxVecRegSize() const {
586  return MaxVecRegSize;
587  }
588 
589  // \returns minimum vector register size as set by cl::opt.
590  unsigned getMinVecRegSize() const {
591  return MinVecRegSize;
592  }
593 
594  /// Check if ArrayType or StructType is isomorphic to some VectorType.
595  ///
596  /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
597  unsigned canMapToVector(Type *T, const DataLayout &DL) const;
598 
599  /// \returns True if the VectorizableTree is both tiny and not fully
600  /// vectorizable. We do not vectorize such trees.
601  bool isTreeTinyAndNotFullyVectorizable();
602 
604 
605 private:
606  struct TreeEntry;
607 
608  /// Checks if all users of \p I are the part of the vectorization tree.
609  bool areAllUsersVectorized(Instruction *I) const;
610 
611  /// \returns the cost of the vectorizable entry.
612  int getEntryCost(TreeEntry *E);
613 
614  /// This is the recursive part of buildTree.
615  void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
616 
617  /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
618  /// be vectorized to use the original vector (or aggregate "bitcast" to a
619  /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
620  /// returns false, setting \p CurrentOrder to either an empty vector or a
621  /// non-identity permutation that allows to reuse extract instructions.
622  bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
623  SmallVectorImpl<unsigned> &CurrentOrder) const;
624 
625  /// Vectorize a single entry in the tree.
626  Value *vectorizeTree(TreeEntry *E);
627 
628  /// Vectorize a single entry in the tree, starting in \p VL.
629  Value *vectorizeTree(ArrayRef<Value *> VL);
630 
631  /// \returns the scalarization cost for this type. Scalarization in this
632  /// context means the creation of vectors from a group of scalars.
633  int getGatherCost(Type *Ty, const DenseSet<unsigned> &ShuffledIndices);
634 
635  /// \returns the scalarization cost for this list of values. Assuming that
636  /// this subtree gets vectorized, we may need to extract the values from the
637  /// roots. This method calculates the cost of extracting the values.
638  int getGatherCost(ArrayRef<Value *> VL);
639 
640  /// Set the Builder insert point to one after the last instruction in
641  /// the bundle
642  void setInsertPointAfterBundle(ArrayRef<Value *> VL,
643  const InstructionsState &S);
644 
645  /// \returns a vector from a collection of scalars in \p VL.
646  Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
647 
648  /// \returns whether the VectorizableTree is fully vectorizable and will
649  /// be beneficial even the tree height is tiny.
650  bool isFullyVectorizableTinyTree();
651 
652  /// \reorder commutative operands in alt shuffle if they result in
653  /// vectorized code.
654  void reorderAltShuffleOperands(const InstructionsState &S,
658 
659  /// \reorder commutative operands to get better probability of
660  /// generating vectorized code.
661  void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
663  SmallVectorImpl<Value *> &Right);
664  struct TreeEntry {
665  TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
666 
667  /// \returns true if the scalars in VL are equal to this entry.
668  bool isSame(ArrayRef<Value *> VL) const {
669  if (VL.size() == Scalars.size())
670  return std::equal(VL.begin(), VL.end(), Scalars.begin());
671  return VL.size() == ReuseShuffleIndices.size() &&
672  std::equal(
673  VL.begin(), VL.end(), ReuseShuffleIndices.begin(),
674  [this](Value *V, unsigned Idx) { return V == Scalars[Idx]; });
675  }
676 
677  /// A vector of scalars.
678  ValueList Scalars;
679 
680  /// The Scalars are vectorized into this value. It is initialized to Null.
681  Value *VectorizedValue = nullptr;
682 
683  /// Do we need to gather this sequence ?
684  bool NeedToGather = false;
685 
686  /// Does this sequence require some shuffling?
687  SmallVector<unsigned, 4> ReuseShuffleIndices;
688 
689  /// Does this entry require reordering?
690  ArrayRef<unsigned> ReorderIndices;
691 
692  /// Points back to the VectorizableTree.
693  ///
694  /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
695  /// to be a pointer and needs to be able to initialize the child iterator.
696  /// Thus we need a reference back to the container to translate the indices
697  /// to entries.
698  std::vector<TreeEntry> &Container;
699 
700  /// The TreeEntry index containing the user of this entry. We can actually
701  /// have multiple users so the data structure is not truly a tree.
702  SmallVector<int, 1> UserTreeIndices;
703  };
704 
705  /// Create a new VectorizableTree entry.
706  void newTreeEntry(ArrayRef<Value *> VL, bool Vectorized, int &UserTreeIdx,
707  ArrayRef<unsigned> ReuseShuffleIndices = None,
708  ArrayRef<unsigned> ReorderIndices = None) {
709  VectorizableTree.emplace_back(VectorizableTree);
710  int idx = VectorizableTree.size() - 1;
711  TreeEntry *Last = &VectorizableTree[idx];
712  Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
713  Last->NeedToGather = !Vectorized;
714  Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
715  ReuseShuffleIndices.end());
716  Last->ReorderIndices = ReorderIndices;
717  if (Vectorized) {
718  for (int i = 0, e = VL.size(); i != e; ++i) {
719  assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
720  ScalarToTreeEntry[VL[i]] = idx;
721  }
722  } else {
723  MustGather.insert(VL.begin(), VL.end());
724  }
725 
726  if (UserTreeIdx >= 0)
727  Last->UserTreeIndices.push_back(UserTreeIdx);
728  UserTreeIdx = idx;
729  }
730 
731  /// -- Vectorization State --
732  /// Holds all of the tree entries.
733  std::vector<TreeEntry> VectorizableTree;
734 
735  TreeEntry *getTreeEntry(Value *V) {
736  auto I = ScalarToTreeEntry.find(V);
737  if (I != ScalarToTreeEntry.end())
738  return &VectorizableTree[I->second];
739  return nullptr;
740  }
741 
742  /// Maps a specific scalar to its tree entry.
743  SmallDenseMap<Value*, int> ScalarToTreeEntry;
744 
745  /// A list of scalars that we found that we need to keep as scalars.
746  ValueSet MustGather;
747 
748  /// This POD struct describes one external user in the vectorized tree.
749  struct ExternalUser {
750  ExternalUser(Value *S, llvm::User *U, int L)
751  : Scalar(S), User(U), Lane(L) {}
752 
753  // Which scalar in our function.
754  Value *Scalar;
755 
756  // Which user that uses the scalar.
757  llvm::User *User;
758 
759  // Which lane does the scalar belong to.
760  int Lane;
761  };
763 
764  /// Checks if two instructions may access the same memory.
765  ///
766  /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
767  /// is invariant in the calling loop.
768  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
769  Instruction *Inst2) {
770  // First check if the result is already in the cache.
771  AliasCacheKey key = std::make_pair(Inst1, Inst2);
772  Optional<bool> &result = AliasCache[key];
773  if (result.hasValue()) {
774  return result.getValue();
775  }
776  MemoryLocation Loc2 = getLocation(Inst2, AA);
777  bool aliased = true;
778  if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
779  // Do the alias check.
780  aliased = AA->alias(Loc1, Loc2);
781  }
782  // Store the result in the cache.
783  result = aliased;
784  return aliased;
785  }
786 
787  using AliasCacheKey = std::pair<Instruction *, Instruction *>;
788 
789  /// Cache for alias results.
790  /// TODO: consider moving this to the AliasAnalysis itself.
792 
793  /// Removes an instruction from its block and eventually deletes it.
794  /// It's like Instruction::eraseFromParent() except that the actual deletion
795  /// is delayed until BoUpSLP is destructed.
796  /// This is required to ensure that there are no incorrect collisions in the
797  /// AliasCache, which can happen if a new instruction is allocated at the
798  /// same address as a previously deleted instruction.
799  void eraseInstruction(Instruction *I) {
800  I->removeFromParent();
801  I->dropAllReferences();
802  DeletedInstructions.emplace_back(I);
803  }
804 
805  /// Temporary store for deleted instructions. Instructions will be deleted
806  /// eventually when the BoUpSLP is destructed.
807  SmallVector<unique_value, 8> DeletedInstructions;
808 
809  /// A list of values that need to extracted out of the tree.
810  /// This list holds pairs of (Internal Scalar : External User). External User
811  /// can be nullptr, it means that this Internal Scalar will be used later,
812  /// after vectorization.
813  UserList ExternalUses;
814 
815  /// Values used only by @llvm.assume calls.
817 
818  /// Holds all of the instructions that we gathered.
819  SetVector<Instruction *> GatherSeq;
820 
821  /// A list of blocks that we are going to CSE.
822  SetVector<BasicBlock *> CSEBlocks;
823 
824  /// Contains all scheduling relevant data for an instruction.
825  /// A ScheduleData either represents a single instruction or a member of an
826  /// instruction bundle (= a group of instructions which is combined into a
827  /// vector instruction).
828  struct ScheduleData {
829  // The initial value for the dependency counters. It means that the
830  // dependencies are not calculated yet.
831  enum { InvalidDeps = -1 };
832 
833  ScheduleData() = default;
834 
835  void init(int BlockSchedulingRegionID, Value *OpVal) {
836  FirstInBundle = this;
837  NextInBundle = nullptr;
838  NextLoadStore = nullptr;
839  IsScheduled = false;
840  SchedulingRegionID = BlockSchedulingRegionID;
841  UnscheduledDepsInBundle = UnscheduledDeps;
842  clearDependencies();
843  OpValue = OpVal;
844  }
845 
846  /// Returns true if the dependency information has been calculated.
847  bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
848 
849  /// Returns true for single instructions and for bundle representatives
850  /// (= the head of a bundle).
851  bool isSchedulingEntity() const { return FirstInBundle == this; }
852 
853  /// Returns true if it represents an instruction bundle and not only a
854  /// single instruction.
855  bool isPartOfBundle() const {
856  return NextInBundle != nullptr || FirstInBundle != this;
857  }
858 
859  /// Returns true if it is ready for scheduling, i.e. it has no more
860  /// unscheduled depending instructions/bundles.
861  bool isReady() const {
862  assert(isSchedulingEntity() &&
863  "can't consider non-scheduling entity for ready list");
864  return UnscheduledDepsInBundle == 0 && !IsScheduled;
865  }
866 
867  /// Modifies the number of unscheduled dependencies, also updating it for
868  /// the whole bundle.
869  int incrementUnscheduledDeps(int Incr) {
870  UnscheduledDeps += Incr;
871  return FirstInBundle->UnscheduledDepsInBundle += Incr;
872  }
873 
874  /// Sets the number of unscheduled dependencies to the number of
875  /// dependencies.
876  void resetUnscheduledDeps() {
877  incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
878  }
879 
880  /// Clears all dependency information.
881  void clearDependencies() {
882  Dependencies = InvalidDeps;
883  resetUnscheduledDeps();
884  MemoryDependencies.clear();
885  }
886 
887  void dump(raw_ostream &os) const {
888  if (!isSchedulingEntity()) {
889  os << "/ " << *Inst;
890  } else if (NextInBundle) {
891  os << '[' << *Inst;
892  ScheduleData *SD = NextInBundle;
893  while (SD) {
894  os << ';' << *SD->Inst;
895  SD = SD->NextInBundle;
896  }
897  os << ']';
898  } else {
899  os << *Inst;
900  }
901  }
902 
903  Instruction *Inst = nullptr;
904 
905  /// Points to the head in an instruction bundle (and always to this for
906  /// single instructions).
907  ScheduleData *FirstInBundle = nullptr;
908 
909  /// Single linked list of all instructions in a bundle. Null if it is a
910  /// single instruction.
911  ScheduleData *NextInBundle = nullptr;
912 
913  /// Single linked list of all memory instructions (e.g. load, store, call)
914  /// in the block - until the end of the scheduling region.
915  ScheduleData *NextLoadStore = nullptr;
916 
917  /// The dependent memory instructions.
918  /// This list is derived on demand in calculateDependencies().
919  SmallVector<ScheduleData *, 4> MemoryDependencies;
920 
921  /// This ScheduleData is in the current scheduling region if this matches
922  /// the current SchedulingRegionID of BlockScheduling.
923  int SchedulingRegionID = 0;
924 
925  /// Used for getting a "good" final ordering of instructions.
926  int SchedulingPriority = 0;
927 
928  /// The number of dependencies. Constitutes of the number of users of the
929  /// instruction plus the number of dependent memory instructions (if any).
930  /// This value is calculated on demand.
931  /// If InvalidDeps, the number of dependencies is not calculated yet.
932  int Dependencies = InvalidDeps;
933 
934  /// The number of dependencies minus the number of dependencies of scheduled
935  /// instructions. As soon as this is zero, the instruction/bundle gets ready
936  /// for scheduling.
937  /// Note that this is negative as long as Dependencies is not calculated.
938  int UnscheduledDeps = InvalidDeps;
939 
940  /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
941  /// single instructions.
942  int UnscheduledDepsInBundle = InvalidDeps;
943 
944  /// True if this instruction is scheduled (or considered as scheduled in the
945  /// dry-run).
946  bool IsScheduled = false;
947 
948  /// Opcode of the current instruction in the schedule data.
949  Value *OpValue = nullptr;
950  };
951 
952 #ifndef NDEBUG
953  friend inline raw_ostream &operator<<(raw_ostream &os,
954  const BoUpSLP::ScheduleData &SD) {
955  SD.dump(os);
956  return os;
957  }
958 #endif
959 
960  friend struct GraphTraits<BoUpSLP *>;
961  friend struct DOTGraphTraits<BoUpSLP *>;
962 
963  /// Contains all scheduling data for a basic block.
964  struct BlockScheduling {
965  BlockScheduling(BasicBlock *BB)
966  : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
967 
968  void clear() {
969  ReadyInsts.clear();
970  ScheduleStart = nullptr;
971  ScheduleEnd = nullptr;
972  FirstLoadStoreInRegion = nullptr;
973  LastLoadStoreInRegion = nullptr;
974 
975  // Reduce the maximum schedule region size by the size of the
976  // previous scheduling run.
977  ScheduleRegionSizeLimit -= ScheduleRegionSize;
978  if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
979  ScheduleRegionSizeLimit = MinScheduleRegionSize;
980  ScheduleRegionSize = 0;
981 
982  // Make a new scheduling region, i.e. all existing ScheduleData is not
983  // in the new region yet.
984  ++SchedulingRegionID;
985  }
986 
987  ScheduleData *getScheduleData(Value *V) {
988  ScheduleData *SD = ScheduleDataMap[V];
989  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
990  return SD;
991  return nullptr;
992  }
993 
994  ScheduleData *getScheduleData(Value *V, Value *Key) {
995  if (V == Key)
996  return getScheduleData(V);
997  auto I = ExtraScheduleDataMap.find(V);
998  if (I != ExtraScheduleDataMap.end()) {
999  ScheduleData *SD = I->second[Key];
1000  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1001  return SD;
1002  }
1003  return nullptr;
1004  }
1005 
1006  bool isInSchedulingRegion(ScheduleData *SD) {
1007  return SD->SchedulingRegionID == SchedulingRegionID;
1008  }
1009 
1010  /// Marks an instruction as scheduled and puts all dependent ready
1011  /// instructions into the ready-list.
1012  template <typename ReadyListType>
1013  void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1014  SD->IsScheduled = true;
1015  LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1016 
1017  ScheduleData *BundleMember = SD;
1018  while (BundleMember) {
1019  if (BundleMember->Inst != BundleMember->OpValue) {
1020  BundleMember = BundleMember->NextInBundle;
1021  continue;
1022  }
1023  // Handle the def-use chain dependencies.
1024  for (Use &U : BundleMember->Inst->operands()) {
1025  auto *I = dyn_cast<Instruction>(U.get());
1026  if (!I)
1027  continue;
1028  doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1029  if (OpDef && OpDef->hasValidDependencies() &&
1030  OpDef->incrementUnscheduledDeps(-1) == 0) {
1031  // There are no more unscheduled dependencies after
1032  // decrementing, so we can put the dependent instruction
1033  // into the ready list.
1034  ScheduleData *DepBundle = OpDef->FirstInBundle;
1035  assert(!DepBundle->IsScheduled &&
1036  "already scheduled bundle gets ready");
1037  ReadyList.insert(DepBundle);
1038  LLVM_DEBUG(dbgs()
1039  << "SLP: gets ready (def): " << *DepBundle << "\n");
1040  }
1041  });
1042  }
1043  // Handle the memory dependencies.
1044  for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1045  if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1046  // There are no more unscheduled dependencies after decrementing,
1047  // so we can put the dependent instruction into the ready list.
1048  ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1049  assert(!DepBundle->IsScheduled &&
1050  "already scheduled bundle gets ready");
1051  ReadyList.insert(DepBundle);
1052  LLVM_DEBUG(dbgs()
1053  << "SLP: gets ready (mem): " << *DepBundle << "\n");
1054  }
1055  }
1056  BundleMember = BundleMember->NextInBundle;
1057  }
1058  }
1059 
1060  void doForAllOpcodes(Value *V,
1061  function_ref<void(ScheduleData *SD)> Action) {
1062  if (ScheduleData *SD = getScheduleData(V))
1063  Action(SD);
1064  auto I = ExtraScheduleDataMap.find(V);
1065  if (I != ExtraScheduleDataMap.end())
1066  for (auto &P : I->second)
1067  if (P.second->SchedulingRegionID == SchedulingRegionID)
1068  Action(P.second);
1069  }
1070 
1071  /// Put all instructions into the ReadyList which are ready for scheduling.
1072  template <typename ReadyListType>
1073  void initialFillReadyList(ReadyListType &ReadyList) {
1074  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1075  doForAllOpcodes(I, [&](ScheduleData *SD) {
1076  if (SD->isSchedulingEntity() && SD->isReady()) {
1077  ReadyList.insert(SD);
1078  LLVM_DEBUG(dbgs()
1079  << "SLP: initially in ready list: " << *I << "\n");
1080  }
1081  });
1082  }
1083  }
1084 
1085  /// Checks if a bundle of instructions can be scheduled, i.e. has no
1086  /// cyclic dependencies. This is only a dry-run, no instructions are
1087  /// actually moved at this stage.
1088  bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
1089  const InstructionsState &S);
1090 
1091  /// Un-bundles a group of instructions.
1092  void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1093 
1094  /// Allocates schedule data chunk.
1095  ScheduleData *allocateScheduleDataChunks();
1096 
1097  /// Extends the scheduling region so that V is inside the region.
1098  /// \returns true if the region size is within the limit.
1099  bool extendSchedulingRegion(Value *V, const InstructionsState &S);
1100 
1101  /// Initialize the ScheduleData structures for new instructions in the
1102  /// scheduling region.
1103  void initScheduleData(Instruction *FromI, Instruction *ToI,
1104  ScheduleData *PrevLoadStore,
1105  ScheduleData *NextLoadStore);
1106 
1107  /// Updates the dependency information of a bundle and of all instructions/
1108  /// bundles which depend on the original bundle.
1109  void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1110  BoUpSLP *SLP);
1111 
1112  /// Sets all instruction in the scheduling region to un-scheduled.
1113  void resetSchedule();
1114 
1115  BasicBlock *BB;
1116 
1117  /// Simple memory allocation for ScheduleData.
1118  std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1119 
1120  /// The size of a ScheduleData array in ScheduleDataChunks.
1121  int ChunkSize;
1122 
1123  /// The allocator position in the current chunk, which is the last entry
1124  /// of ScheduleDataChunks.
1125  int ChunkPos;
1126 
1127  /// Attaches ScheduleData to Instruction.
1128  /// Note that the mapping survives during all vectorization iterations, i.e.
1129  /// ScheduleData structures are recycled.
1130  DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1131 
1132  /// Attaches ScheduleData to Instruction with the leading key.
1134  ExtraScheduleDataMap;
1135 
1136  struct ReadyList : SmallVector<ScheduleData *, 8> {
1137  void insert(ScheduleData *SD) { push_back(SD); }
1138  };
1139 
1140  /// The ready-list for scheduling (only used for the dry-run).
1141  ReadyList ReadyInsts;
1142 
1143  /// The first instruction of the scheduling region.
1144  Instruction *ScheduleStart = nullptr;
1145 
1146  /// The first instruction _after_ the scheduling region.
1147  Instruction *ScheduleEnd = nullptr;
1148 
1149  /// The first memory accessing instruction in the scheduling region
1150  /// (can be null).
1151  ScheduleData *FirstLoadStoreInRegion = nullptr;
1152 
1153  /// The last memory accessing instruction in the scheduling region
1154  /// (can be null).
1155  ScheduleData *LastLoadStoreInRegion = nullptr;
1156 
1157  /// The current size of the scheduling region.
1158  int ScheduleRegionSize = 0;
1159 
1160  /// The maximum size allowed for the scheduling region.
1161  int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1162 
1163  /// The ID of the scheduling region. For a new vectorization iteration this
1164  /// is incremented which "removes" all ScheduleData from the region.
1165  // Make sure that the initial SchedulingRegionID is greater than the
1166  // initial SchedulingRegionID in ScheduleData (which is 0).
1167  int SchedulingRegionID = 1;
1168  };
1169 
1170  /// Attaches the BlockScheduling structures to basic blocks.
1172 
1173  /// Performs the "real" scheduling. Done before vectorization is actually
1174  /// performed in a basic block.
1175  void scheduleBlock(BlockScheduling *BS);
1176 
1177  /// List of users to ignore during scheduling and that don't need extracting.
1178  ArrayRef<Value *> UserIgnoreList;
1179 
1181  /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
1182  /// sorted SmallVectors of unsigned.
1183  struct OrdersTypeDenseMapInfo {
1184  static OrdersType getEmptyKey() {
1185  OrdersType V;
1186  V.push_back(~1U);
1187  return V;
1188  }
1189 
1190  static OrdersType getTombstoneKey() {
1191  OrdersType V;
1192  V.push_back(~2U);
1193  return V;
1194  }
1195 
1196  static unsigned getHashValue(const OrdersType &V) {
1197  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1198  }
1199 
1200  static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
1201  return LHS == RHS;
1202  }
1203  };
1204 
1205  /// Contains orders of operations along with the number of bundles that have
1206  /// operations in this order. It stores only those orders that require
1207  /// reordering, if reordering is not required it is counted using \a
1208  /// NumOpsWantToKeepOriginalOrder.
1210  /// Number of bundles that do not require reordering.
1211  unsigned NumOpsWantToKeepOriginalOrder = 0;
1212 
1213  // Analysis and block reference.
1214  Function *F;
1215  ScalarEvolution *SE;
1216  TargetTransformInfo *TTI;
1217  TargetLibraryInfo *TLI;
1218  AliasAnalysis *AA;
1219  LoopInfo *LI;
1220  DominatorTree *DT;
1221  AssumptionCache *AC;
1222  DemandedBits *DB;
1223  const DataLayout *DL;
1225 
1226  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1227  unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1228 
1229  /// Instruction builder to construct the vectorized tree.
1230  IRBuilder<> Builder;
1231 
1232  /// A map of scalar integer values to the smallest bit width with which they
1233  /// can legally be represented. The values map to (width, signed) pairs,
1234  /// where "width" indicates the minimum bit width and "signed" is True if the
1235  /// value must be signed-extended, rather than zero-extended, back to its
1236  /// original width.
1238 };
1239 
1240 } // end namespace slpvectorizer
1241 
1242 template <> struct GraphTraits<BoUpSLP *> {
1243  using TreeEntry = BoUpSLP::TreeEntry;
1244 
1245  /// NodeRef has to be a pointer per the GraphWriter.
1246  using NodeRef = TreeEntry *;
1247 
1248  /// Add the VectorizableTree to the index iterator to be able to return
1249  /// TreeEntry pointers.
1250  struct ChildIteratorType
1251  : public iterator_adaptor_base<ChildIteratorType,
1252  SmallVector<int, 1>::iterator> {
1253  std::vector<TreeEntry> &VectorizableTree;
1254 
1256  std::vector<TreeEntry> &VT)
1257  : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1258 
1259  NodeRef operator*() { return &VectorizableTree[*I]; }
1260  };
1261 
1262  static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1263 
1264  static ChildIteratorType child_begin(NodeRef N) {
1265  return {N->UserTreeIndices.begin(), N->Container};
1266  }
1267 
1268  static ChildIteratorType child_end(NodeRef N) {
1269  return {N->UserTreeIndices.end(), N->Container};
1270  }
1271 
1272  /// For the node iterator we just need to turn the TreeEntry iterator into a
1273  /// TreeEntry* iterator so that it dereferences to NodeRef.
1275 
1276  static nodes_iterator nodes_begin(BoUpSLP *R) {
1277  return nodes_iterator(R->VectorizableTree.begin());
1278  }
1279 
1280  static nodes_iterator nodes_end(BoUpSLP *R) {
1281  return nodes_iterator(R->VectorizableTree.end());
1282  }
1283 
1284  static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1285 };
1286 
1287 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1288  using TreeEntry = BoUpSLP::TreeEntry;
1289 
1291 
1292  std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1293  std::string Str;
1294  raw_string_ostream OS(Str);
1295  if (isSplat(Entry->Scalars)) {
1296  OS << "<splat> " << *Entry->Scalars[0];
1297  return Str;
1298  }
1299  for (auto V : Entry->Scalars) {
1300  OS << *V;
1301  if (std::any_of(
1302  R->ExternalUses.begin(), R->ExternalUses.end(),
1303  [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1304  OS << " <extract>";
1305  OS << "\n";
1306  }
1307  return Str;
1308  }
1309 
1310  static std::string getNodeAttributes(const TreeEntry *Entry,
1311  const BoUpSLP *) {
1312  if (Entry->NeedToGather)
1313  return "color=red";
1314  return "";
1315  }
1316 };
1317 
1318 } // end namespace llvm
1319 
1320 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1321  ArrayRef<Value *> UserIgnoreLst) {
1322  ExtraValueToDebugLocsMap ExternallyUsedValues;
1323  buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1324 }
1325 
1326 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1327  ExtraValueToDebugLocsMap &ExternallyUsedValues,
1328  ArrayRef<Value *> UserIgnoreLst) {
1329  deleteTree();
1330  UserIgnoreList = UserIgnoreLst;
1331  if (!allSameType(Roots))
1332  return;
1333  buildTree_rec(Roots, 0, -1);
1334 
1335  // Collect the values that we need to extract from the tree.
1336  for (TreeEntry &EIdx : VectorizableTree) {
1337  TreeEntry *Entry = &EIdx;
1338 
1339  // No need to handle users of gathered values.
1340  if (Entry->NeedToGather)
1341  continue;
1342 
1343  // For each lane:
1344  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1345  Value *Scalar = Entry->Scalars[Lane];
1346  int FoundLane = Lane;
1347  if (!Entry->ReuseShuffleIndices.empty()) {
1348  FoundLane =
1349  std::distance(Entry->ReuseShuffleIndices.begin(),
1350  llvm::find(Entry->ReuseShuffleIndices, FoundLane));
1351  }
1352 
1353  // Check if the scalar is externally used as an extra arg.
1354  auto ExtI = ExternallyUsedValues.find(Scalar);
1355  if (ExtI != ExternallyUsedValues.end()) {
1356  LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
1357  << Lane << " from " << *Scalar << ".\n");
1358  ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
1359  }
1360  for (User *U : Scalar->users()) {
1361  LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1362 
1363  Instruction *UserInst = dyn_cast<Instruction>(U);
1364  if (!UserInst)
1365  continue;
1366 
1367  // Skip in-tree scalars that become vectors
1368  if (TreeEntry *UseEntry = getTreeEntry(U)) {
1369  Value *UseScalar = UseEntry->Scalars[0];
1370  // Some in-tree scalars will remain as scalar in vectorized
1371  // instructions. If that is the case, the one in Lane 0 will
1372  // be used.
1373  if (UseScalar != U ||
1374  !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1375  LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1376  << ".\n");
1377  assert(!UseEntry->NeedToGather && "Bad state");
1378  continue;
1379  }
1380  }
1381 
1382  // Ignore users in the user ignore list.
1383  if (is_contained(UserIgnoreList, UserInst))
1384  continue;
1385 
1386  LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "
1387  << Lane << " from " << *Scalar << ".\n");
1388  ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
1389  }
1390  }
1391  }
1392 }
1393 
1394 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1395  int UserTreeIdx) {
1396  assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1397 
1398  InstructionsState S = getSameOpcode(VL);
1399  if (Depth == RecursionMaxDepth) {
1400  LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1401  newTreeEntry(VL, false, UserTreeIdx);
1402  return;
1403  }
1404 
1405  // Don't handle vectors.
1406  if (S.OpValue->getType()->isVectorTy()) {
1407  LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1408  newTreeEntry(VL, false, UserTreeIdx);
1409  return;
1410  }
1411 
1412  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1413  if (SI->getValueOperand()->getType()->isVectorTy()) {
1414  LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1415  newTreeEntry(VL, false, UserTreeIdx);
1416  return;
1417  }
1418 
1419  // If all of the operands are identical or constant we have a simple solution.
1420  if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
1421  LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1422  newTreeEntry(VL, false, UserTreeIdx);
1423  return;
1424  }
1425 
1426  // We now know that this is a vector of instructions of the same type from
1427  // the same block.
1428 
1429  // Don't vectorize ephemeral values.
1430  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1431  if (EphValues.count(VL[i])) {
1432  LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]
1433  << ") is ephemeral.\n");
1434  newTreeEntry(VL, false, UserTreeIdx);
1435  return;
1436  }
1437  }
1438 
1439  // Check if this is a duplicate of another entry.
1440  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1441  LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n");
1442  if (!E->isSame(VL)) {
1443  LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1444  newTreeEntry(VL, false, UserTreeIdx);
1445  return;
1446  }
1447  // Record the reuse of the tree node. FIXME, currently this is only used to
1448  // properly draw the graph rather than for the actual vectorization.
1449  E->UserTreeIndices.push_back(UserTreeIdx);
1450  LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue
1451  << ".\n");
1452  return;
1453  }
1454 
1455  // Check that none of the instructions in the bundle are already in the tree.
1456  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1457  auto *I = dyn_cast<Instruction>(VL[i]);
1458  if (!I)
1459  continue;
1460  if (getTreeEntry(I)) {
1461  LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *VL[i]
1462  << ") is already in tree.\n");
1463  newTreeEntry(VL, false, UserTreeIdx);
1464  return;
1465  }
1466  }
1467 
1468  // If any of the scalars is marked as a value that needs to stay scalar, then
1469  // we need to gather the scalars.
1470  // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
1471  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1472  if (MustGather.count(VL[i]) || is_contained(UserIgnoreList, VL[i])) {
1473  LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1474  newTreeEntry(VL, false, UserTreeIdx);
1475  return;
1476  }
1477  }
1478 
1479  // Check that all of the users of the scalars that we want to vectorize are
1480  // schedulable.
1481  auto *VL0 = cast<Instruction>(S.OpValue);
1482  BasicBlock *BB = VL0->getParent();
1483 
1484  if (!DT->isReachableFromEntry(BB)) {
1485  // Don't go into unreachable blocks. They may contain instructions with
1486  // dependency cycles which confuse the final scheduling.
1487  LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1488  newTreeEntry(VL, false, UserTreeIdx);
1489  return;
1490  }
1491 
1492  // Check that every instruction appears once in this bundle.
1493  SmallVector<unsigned, 4> ReuseShuffleIndicies;
1494  SmallVector<Value *, 4> UniqueValues;
1495  DenseMap<Value *, unsigned> UniquePositions;
1496  for (Value *V : VL) {
1497  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
1498  ReuseShuffleIndicies.emplace_back(Res.first->second);
1499  if (Res.second)
1500  UniqueValues.emplace_back(V);
1501  }
1502  if (UniqueValues.size() == VL.size()) {
1503  ReuseShuffleIndicies.clear();
1504  } else {
1505  LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
1506  if (UniqueValues.size() <= 1 || !llvm::isPowerOf2_32(UniqueValues.size())) {
1507  LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1508  newTreeEntry(VL, false, UserTreeIdx);
1509  return;
1510  }
1511  VL = UniqueValues;
1512  }
1513 
1514  auto &BSRef = BlocksSchedules[BB];
1515  if (!BSRef)
1516  BSRef = llvm::make_unique<BlockScheduling>(BB);
1517 
1518  BlockScheduling &BS = *BSRef.get();
1519 
1520  if (!BS.tryScheduleBundle(VL, this, S)) {
1521  LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1522  assert((!BS.getScheduleData(VL0) ||
1523  !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1524  "tryScheduleBundle should cancelScheduling on failure");
1525  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1526  return;
1527  }
1528  LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1529 
1530  unsigned ShuffleOrOp = S.isAltShuffle() ?
1531  (unsigned) Instruction::ShuffleVector : S.getOpcode();
1532  switch (ShuffleOrOp) {
1533  case Instruction::PHI: {
1534  PHINode *PH = dyn_cast<PHINode>(VL0);
1535 
1536  // Check for terminator values (e.g. invoke).
1537  for (unsigned j = 0; j < VL.size(); ++j)
1538  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1539  Instruction *Term = dyn_cast<Instruction>(
1540  cast<PHINode>(VL[j])->getIncomingValueForBlock(
1541  PH->getIncomingBlock(i)));
1542  if (Term && Term->isTerminator()) {
1543  LLVM_DEBUG(dbgs()
1544  << "SLP: Need to swizzle PHINodes (terminator use).\n");
1545  BS.cancelScheduling(VL, VL0);
1546  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1547  return;
1548  }
1549  }
1550 
1551  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1552  LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1553 
1554  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1555  ValueList Operands;
1556  // Prepare the operand vector.
1557  for (Value *j : VL)
1558  Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1559  PH->getIncomingBlock(i)));
1560 
1561  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1562  }
1563  return;
1564  }
1565  case Instruction::ExtractValue:
1566  case Instruction::ExtractElement: {
1567  OrdersType CurrentOrder;
1568  bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
1569  if (Reuse) {
1570  LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
1571  ++NumOpsWantToKeepOriginalOrder;
1572  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1573  ReuseShuffleIndicies);
1574  return;
1575  }
1576  if (!CurrentOrder.empty()) {
1577  LLVM_DEBUG({
1578  dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
1579  "with order";
1580  for (unsigned Idx : CurrentOrder)
1581  dbgs() << " " << Idx;
1582  dbgs() << "\n";
1583  });
1584  // Insert new order with initial value 0, if it does not exist,
1585  // otherwise return the iterator to the existing one.
1586  auto StoredCurrentOrderAndNum =
1587  NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1588  ++StoredCurrentOrderAndNum->getSecond();
1589  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx, ReuseShuffleIndicies,
1590  StoredCurrentOrderAndNum->getFirst());
1591  return;
1592  }
1593  LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
1594  newTreeEntry(VL, /*Vectorized=*/false, UserTreeIdx, ReuseShuffleIndicies);
1595  BS.cancelScheduling(VL, VL0);
1596  return;
1597  }
1598  case Instruction::Load: {
1599  // Check that a vectorized load would load the same memory as a scalar
1600  // load. For example, we don't want to vectorize loads that are smaller
1601  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1602  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1603  // from such a struct, we read/write packed bits disagreeing with the
1604  // unvectorized version.
1605  Type *ScalarTy = VL0->getType();
1606 
1607  if (DL->getTypeSizeInBits(ScalarTy) !=
1608  DL->getTypeAllocSizeInBits(ScalarTy)) {
1609  BS.cancelScheduling(VL, VL0);
1610  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1611  LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1612  return;
1613  }
1614 
1615  // Make sure all loads in the bundle are simple - we can't vectorize
1616  // atomic or volatile loads.
1617  SmallVector<Value *, 4> PointerOps(VL.size());
1618  auto POIter = PointerOps.begin();
1619  for (Value *V : VL) {
1620  auto *L = cast<LoadInst>(V);
1621  if (!L->isSimple()) {
1622  BS.cancelScheduling(VL, VL0);
1623  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1624  LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1625  return;
1626  }
1627  *POIter = L->getPointerOperand();
1628  ++POIter;
1629  }
1630 
1631  OrdersType CurrentOrder;
1632  // Check the order of pointer operands.
1633  if (llvm::sortPtrAccesses(PointerOps, *DL, *SE, CurrentOrder)) {
1634  Value *Ptr0;
1635  Value *PtrN;
1636  if (CurrentOrder.empty()) {
1637  Ptr0 = PointerOps.front();
1638  PtrN = PointerOps.back();
1639  } else {
1640  Ptr0 = PointerOps[CurrentOrder.front()];
1641  PtrN = PointerOps[CurrentOrder.back()];
1642  }
1643  const SCEV *Scev0 = SE->getSCEV(Ptr0);
1644  const SCEV *ScevN = SE->getSCEV(PtrN);
1645  const auto *Diff =
1646  dyn_cast<SCEVConstant>(SE->getMinusSCEV(ScevN, Scev0));
1647  uint64_t Size = DL->getTypeAllocSize(ScalarTy);
1648  // Check that the sorted loads are consecutive.
1649  if (Diff && Diff->getAPInt().getZExtValue() == (VL.size() - 1) * Size) {
1650  if (CurrentOrder.empty()) {
1651  // Original loads are consecutive and does not require reordering.
1652  ++NumOpsWantToKeepOriginalOrder;
1653  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1654  ReuseShuffleIndicies);
1655  LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1656  } else {
1657  // Need to reorder.
1658  auto I = NumOpsWantToKeepOrder.try_emplace(CurrentOrder).first;
1659  ++I->getSecond();
1660  newTreeEntry(VL, /*Vectorized=*/true, UserTreeIdx,
1661  ReuseShuffleIndicies, I->getFirst());
1662  LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
1663  }
1664  return;
1665  }
1666  }
1667 
1668  LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1669  BS.cancelScheduling(VL, VL0);
1670  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1671  return;
1672  }
1673  case Instruction::ZExt:
1674  case Instruction::SExt:
1675  case Instruction::FPToUI:
1676  case Instruction::FPToSI:
1677  case Instruction::FPExt:
1678  case Instruction::PtrToInt:
1679  case Instruction::IntToPtr:
1680  case Instruction::SIToFP:
1681  case Instruction::UIToFP:
1682  case Instruction::Trunc:
1683  case Instruction::FPTrunc:
1684  case Instruction::BitCast: {
1685  Type *SrcTy = VL0->getOperand(0)->getType();
1686  for (unsigned i = 0; i < VL.size(); ++i) {
1687  Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1688  if (Ty != SrcTy || !isValidElementType(Ty)) {
1689  BS.cancelScheduling(VL, VL0);
1690  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1691  LLVM_DEBUG(dbgs()
1692  << "SLP: Gathering casts with different src types.\n");
1693  return;
1694  }
1695  }
1696  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1697  LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1698 
1699  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1700  ValueList Operands;
1701  // Prepare the operand vector.
1702  for (Value *j : VL)
1703  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1704 
1705  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1706  }
1707  return;
1708  }
1709  case Instruction::ICmp:
1710  case Instruction::FCmp: {
1711  // Check that all of the compares have the same predicate.
1712  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1713  Type *ComparedTy = VL0->getOperand(0)->getType();
1714  for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1715  CmpInst *Cmp = cast<CmpInst>(VL[i]);
1716  if (Cmp->getPredicate() != P0 ||
1717  Cmp->getOperand(0)->getType() != ComparedTy) {
1718  BS.cancelScheduling(VL, VL0);
1719  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1720  LLVM_DEBUG(dbgs()
1721  << "SLP: Gathering cmp with different predicate.\n");
1722  return;
1723  }
1724  }
1725 
1726  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1727  LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1728 
1729  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1730  ValueList Operands;
1731  // Prepare the operand vector.
1732  for (Value *j : VL)
1733  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1734 
1735  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1736  }
1737  return;
1738  }
1739  case Instruction::Select:
1740  case Instruction::Add:
1741  case Instruction::FAdd:
1742  case Instruction::Sub:
1743  case Instruction::FSub:
1744  case Instruction::Mul:
1745  case Instruction::FMul:
1746  case Instruction::UDiv:
1747  case Instruction::SDiv:
1748  case Instruction::FDiv:
1749  case Instruction::URem:
1750  case Instruction::SRem:
1751  case Instruction::FRem:
1752  case Instruction::Shl:
1753  case Instruction::LShr:
1754  case Instruction::AShr:
1755  case Instruction::And:
1756  case Instruction::Or:
1757  case Instruction::Xor:
1758  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1759  LLVM_DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1760 
1761  // Sort operands of the instructions so that each side is more likely to
1762  // have the same opcode.
1763  if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1764  ValueList Left, Right;
1765  reorderInputsAccordingToOpcode(S.getOpcode(), VL, Left, Right);
1766  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1767  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1768  return;
1769  }
1770 
1771  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1772  ValueList Operands;
1773  // Prepare the operand vector.
1774  for (Value *j : VL)
1775  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1776 
1777  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1778  }
1779  return;
1780 
1781  case Instruction::GetElementPtr: {
1782  // We don't combine GEPs with complicated (nested) indexing.
1783  for (unsigned j = 0; j < VL.size(); ++j) {
1784  if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1785  LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1786  BS.cancelScheduling(VL, VL0);
1787  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1788  return;
1789  }
1790  }
1791 
1792  // We can't combine several GEPs into one vector if they operate on
1793  // different types.
1794  Type *Ty0 = VL0->getOperand(0)->getType();
1795  for (unsigned j = 0; j < VL.size(); ++j) {
1796  Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1797  if (Ty0 != CurTy) {
1798  LLVM_DEBUG(dbgs()
1799  << "SLP: not-vectorizable GEP (different types).\n");
1800  BS.cancelScheduling(VL, VL0);
1801  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1802  return;
1803  }
1804  }
1805 
1806  // We don't combine GEPs with non-constant indexes.
1807  for (unsigned j = 0; j < VL.size(); ++j) {
1808  auto Op = cast<Instruction>(VL[j])->getOperand(1);
1809  if (!isa<ConstantInt>(Op)) {
1810  LLVM_DEBUG(dbgs()
1811  << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1812  BS.cancelScheduling(VL, VL0);
1813  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1814  return;
1815  }
1816  }
1817 
1818  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1819  LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1820  for (unsigned i = 0, e = 2; i < e; ++i) {
1821  ValueList Operands;
1822  // Prepare the operand vector.
1823  for (Value *j : VL)
1824  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1825 
1826  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1827  }
1828  return;
1829  }
1830  case Instruction::Store: {
1831  // Check if the stores are consecutive or of we need to swizzle them.
1832  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1833  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1834  BS.cancelScheduling(VL, VL0);
1835  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1836  LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1837  return;
1838  }
1839 
1840  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1841  LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1842 
1843  ValueList Operands;
1844  for (Value *j : VL)
1845  Operands.push_back(cast<Instruction>(j)->getOperand(0));
1846 
1847  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1848  return;
1849  }
1850  case Instruction::Call: {
1851  // Check if the calls are all to the same vectorizable intrinsic.
1852  CallInst *CI = cast<CallInst>(VL0);
1853  // Check if this is an Intrinsic call or something that can be
1854  // represented by an intrinsic call
1856  if (!isTriviallyVectorizable(ID)) {
1857  BS.cancelScheduling(VL, VL0);
1858  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1859  LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1860  return;
1861  }
1862  Function *Int = CI->getCalledFunction();
1863  Value *A1I = nullptr;
1864  if (hasVectorInstrinsicScalarOpd(ID, 1))
1865  A1I = CI->getArgOperand(1);
1866  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1867  CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1868  if (!CI2 || CI2->getCalledFunction() != Int ||
1869  getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1870  !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1871  BS.cancelScheduling(VL, VL0);
1872  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1873  LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1874  << "\n");
1875  return;
1876  }
1877  // ctlz,cttz and powi are special intrinsics whose second argument
1878  // should be same in order for them to be vectorized.
1879  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1880  Value *A1J = CI2->getArgOperand(1);
1881  if (A1I != A1J) {
1882  BS.cancelScheduling(VL, VL0);
1883  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1884  LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1885  << " argument " << A1I << "!=" << A1J << "\n");
1886  return;
1887  }
1888  }
1889  // Verify that the bundle operands are identical between the two calls.
1890  if (CI->hasOperandBundles() &&
1892  CI->op_begin() + CI->getBundleOperandsEndIndex(),
1893  CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1894  BS.cancelScheduling(VL, VL0);
1895  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1896  LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"
1897  << *CI << "!=" << *VL[i] << '\n');
1898  return;
1899  }
1900  }
1901 
1902  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1903  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1904  ValueList Operands;
1905  // Prepare the operand vector.
1906  for (Value *j : VL) {
1907  CallInst *CI2 = dyn_cast<CallInst>(j);
1908  Operands.push_back(CI2->getArgOperand(i));
1909  }
1910  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1911  }
1912  return;
1913  }
1914  case Instruction::ShuffleVector:
1915  // If this is not an alternate sequence of opcode like add-sub
1916  // then do not vectorize this instruction.
1917  if (!S.isAltShuffle()) {
1918  BS.cancelScheduling(VL, VL0);
1919  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1920  LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1921  return;
1922  }
1923  newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
1924  LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1925 
1926  // Reorder operands if reordering would enable vectorization.
1927  if (isa<BinaryOperator>(VL0)) {
1928  ValueList Left, Right;
1929  reorderAltShuffleOperands(S, VL, Left, Right);
1930  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1931  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1932  return;
1933  }
1934 
1935  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1936  ValueList Operands;
1937  // Prepare the operand vector.
1938  for (Value *j : VL)
1939  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1940 
1941  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1942  }
1943  return;
1944 
1945  default:
1946  BS.cancelScheduling(VL, VL0);
1947  newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
1948  LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1949  return;
1950  }
1951 }
1952 
1953 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1954  unsigned N;
1955  Type *EltTy;
1956  auto *ST = dyn_cast<StructType>(T);
1957  if (ST) {
1958  N = ST->getNumElements();
1959  EltTy = *ST->element_begin();
1960  } else {
1961  N = cast<ArrayType>(T)->getNumElements();
1962  EltTy = cast<ArrayType>(T)->getElementType();
1963  }
1964  if (!isValidElementType(EltTy))
1965  return 0;
1966  uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1967  if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1968  return 0;
1969  if (ST) {
1970  // Check that struct is homogeneous.
1971  for (const auto *Ty : ST->elements())
1972  if (Ty != EltTy)
1973  return 0;
1974  }
1975  return N;
1976 }
1977 
1978 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1979  SmallVectorImpl<unsigned> &CurrentOrder) const {
1980  Instruction *E0 = cast<Instruction>(OpValue);
1981  assert(E0->getOpcode() == Instruction::ExtractElement ||
1982  E0->getOpcode() == Instruction::ExtractValue);
1983  assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode");
1984  // Check if all of the extracts come from the same vector and from the
1985  // correct offset.
1986  Value *Vec = E0->getOperand(0);
1987 
1988  CurrentOrder.clear();
1989 
1990  // We have to extract from a vector/aggregate with the same number of elements.
1991  unsigned NElts;
1992  if (E0->getOpcode() == Instruction::ExtractValue) {
1993  const DataLayout &DL = E0->getModule()->getDataLayout();
1994  NElts = canMapToVector(Vec->getType(), DL);
1995  if (!NElts)
1996  return false;
1997  // Check if load can be rewritten as load of vector.
1998  LoadInst *LI = dyn_cast<LoadInst>(Vec);
1999  if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
2000  return false;
2001  } else {
2002  NElts = Vec->getType()->getVectorNumElements();
2003  }
2004 
2005  if (NElts != VL.size())
2006  return false;
2007 
2008  // Check that all of the indices extract from the correct offset.
2009  bool ShouldKeepOrder = true;
2010  unsigned E = VL.size();
2011  // Assign to all items the initial value E + 1 so we can check if the extract
2012  // instruction index was used already.
2013  // Also, later we can check that all the indices are used and we have a
2014  // consecutive access in the extract instructions, by checking that no
2015  // element of CurrentOrder still has value E + 1.
2016  CurrentOrder.assign(E, E + 1);
2017  unsigned I = 0;
2018  for (; I < E; ++I) {
2019  auto *Inst = cast<Instruction>(VL[I]);
2020  if (Inst->getOperand(0) != Vec)
2021  break;
2022  Optional<unsigned> Idx = getExtractIndex(Inst);
2023  if (!Idx)
2024  break;
2025  const unsigned ExtIdx = *Idx;
2026  if (ExtIdx != I) {
2027  if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
2028  break;
2029  ShouldKeepOrder = false;
2030  CurrentOrder[ExtIdx] = I;
2031  } else {
2032  if (CurrentOrder[I] != E + 1)
2033  break;
2034  CurrentOrder[I] = I;
2035  }
2036  }
2037  if (I < E) {
2038  CurrentOrder.clear();
2039  return false;
2040  }
2041 
2042  return ShouldKeepOrder;
2043 }
2044 
2045 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
2046  return I->hasOneUse() ||
2047  std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
2048  return ScalarToTreeEntry.count(U) > 0;
2049  });
2050 }
2051 
2052 int BoUpSLP::getEntryCost(TreeEntry *E) {
2053  ArrayRef<Value*> VL = E->Scalars;
2054 
2055  Type *ScalarTy = VL[0]->getType();
2056  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2057  ScalarTy = SI->getValueOperand()->getType();
2058  else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
2059  ScalarTy = CI->getOperand(0)->getType();
2060  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2061 
2062  // If we have computed a smaller type for the expression, update VecTy so
2063  // that the costs will be accurate.
2064  if (MinBWs.count(VL[0]))
2065  VecTy = VectorType::get(
2066  IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
2067 
2068  unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
2069  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
2070  int ReuseShuffleCost = 0;
2071  if (NeedToShuffleReuses) {
2072  ReuseShuffleCost =
2073  TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
2074  }
2075  if (E->NeedToGather) {
2076  if (allConstant(VL))
2077  return 0;
2078  if (isSplat(VL)) {
2079  return ReuseShuffleCost +
2080  TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
2081  }
2082  if (getSameOpcode(VL).getOpcode() == Instruction::ExtractElement &&
2083  allSameType(VL) && allSameBlock(VL)) {
2085  if (ShuffleKind.hasValue()) {
2086  int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2087  for (auto *V : VL) {
2088  // If all users of instruction are going to be vectorized and this
2089  // instruction itself is not going to be vectorized, consider this
2090  // instruction as dead and remove its cost from the final cost of the
2091  // vectorized tree.
2092  if (areAllUsersVectorized(cast<Instruction>(V)) &&
2093  !ScalarToTreeEntry.count(V)) {
2094  auto *IO = cast<ConstantInt>(
2095  cast<ExtractElementInst>(V)->getIndexOperand());
2096  Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2097  IO->getZExtValue());
2098  }
2099  }
2100  return ReuseShuffleCost + Cost;
2101  }
2102  }
2103  return ReuseShuffleCost + getGatherCost(VL);
2104  }
2105  InstructionsState S = getSameOpcode(VL);
2106  assert(S.getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2107  Instruction *VL0 = cast<Instruction>(S.OpValue);
2108  unsigned ShuffleOrOp = S.isAltShuffle() ?
2109  (unsigned) Instruction::ShuffleVector : S.getOpcode();
2110  switch (ShuffleOrOp) {
2111  case Instruction::PHI:
2112  return 0;
2113 
2114  case Instruction::ExtractValue:
2115  case Instruction::ExtractElement:
2116  if (NeedToShuffleReuses) {
2117  unsigned Idx = 0;
2118  for (unsigned I : E->ReuseShuffleIndices) {
2119  if (ShuffleOrOp == Instruction::ExtractElement) {
2120  auto *IO = cast<ConstantInt>(
2121  cast<ExtractElementInst>(VL[I])->getIndexOperand());
2122  Idx = IO->getZExtValue();
2123  ReuseShuffleCost -= TTI->getVectorInstrCost(
2124  Instruction::ExtractElement, VecTy, Idx);
2125  } else {
2126  ReuseShuffleCost -= TTI->getVectorInstrCost(
2127  Instruction::ExtractElement, VecTy, Idx);
2128  ++Idx;
2129  }
2130  }
2131  Idx = ReuseShuffleNumbers;
2132  for (Value *V : VL) {
2133  if (ShuffleOrOp == Instruction::ExtractElement) {
2134  auto *IO = cast<ConstantInt>(
2135  cast<ExtractElementInst>(V)->getIndexOperand());
2136  Idx = IO->getZExtValue();
2137  } else {
2138  --Idx;
2139  }
2140  ReuseShuffleCost +=
2141  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
2142  }
2143  }
2144  if (!E->NeedToGather) {
2145  int DeadCost = ReuseShuffleCost;
2146  if (!E->ReorderIndices.empty()) {
2147  // TODO: Merge this shuffle with the ReuseShuffleCost.
2148  DeadCost += TTI->getShuffleCost(
2150  }
2151  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2152  Instruction *E = cast<Instruction>(VL[i]);
2153  // If all users are going to be vectorized, instruction can be
2154  // considered as dead.
2155  // The same, if have only one user, it will be vectorized for sure.
2156  if (areAllUsersVectorized(E)) {
2157  // Take credit for instruction that will become dead.
2158  if (E->hasOneUse()) {
2159  Instruction *Ext = E->user_back();
2160  if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
2161  all_of(Ext->users(),
2162  [](User *U) { return isa<GetElementPtrInst>(U); })) {
2163  // Use getExtractWithExtendCost() to calculate the cost of
2164  // extractelement/ext pair.
2165  DeadCost -= TTI->getExtractWithExtendCost(
2166  Ext->getOpcode(), Ext->getType(), VecTy, i);
2167  // Add back the cost of s|zext which is subtracted separately.
2168  DeadCost += TTI->getCastInstrCost(
2169  Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
2170  continue;
2171  }
2172  }
2173  DeadCost -=
2174  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2175  }
2176  }
2177  return DeadCost;
2178  }
2179  return ReuseShuffleCost + getGatherCost(VL);
2180 
2181  case Instruction::ZExt:
2182  case Instruction::SExt:
2183  case Instruction::FPToUI:
2184  case Instruction::FPToSI:
2185  case Instruction::FPExt:
2186  case Instruction::PtrToInt:
2187  case Instruction::IntToPtr:
2188  case Instruction::SIToFP:
2189  case Instruction::UIToFP:
2190  case Instruction::Trunc:
2191  case Instruction::FPTrunc:
2192  case Instruction::BitCast: {
2193  Type *SrcTy = VL0->getOperand(0)->getType();
2194  int ScalarEltCost =
2195  TTI->getCastInstrCost(S.getOpcode(), ScalarTy, SrcTy, VL0);
2196  if (NeedToShuffleReuses) {
2197  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2198  }
2199 
2200  // Calculate the cost of this instruction.
2201  int ScalarCost = VL.size() * ScalarEltCost;
2202 
2203  VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2204  int VecCost = 0;
2205  // Check if the values are candidates to demote.
2206  if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
2207  VecCost = ReuseShuffleCost +
2208  TTI->getCastInstrCost(S.getOpcode(), VecTy, SrcVecTy, VL0);
2209  }
2210  return VecCost - ScalarCost;
2211  }
2212  case Instruction::FCmp:
2213  case Instruction::ICmp:
2214  case Instruction::Select: {
2215  // Calculate the cost of this instruction.
2216  int ScalarEltCost = TTI->getCmpSelInstrCost(S.getOpcode(), ScalarTy,
2217  Builder.getInt1Ty(), VL0);
2218  if (NeedToShuffleReuses) {
2219  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2220  }
2221  VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2222  int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2223  int VecCost = TTI->getCmpSelInstrCost(S.getOpcode(), VecTy, MaskTy, VL0);
2224  return ReuseShuffleCost + VecCost - ScalarCost;
2225  }
2226  case Instruction::Add:
2227  case Instruction::FAdd:
2228  case Instruction::Sub:
2229  case Instruction::FSub:
2230  case Instruction::Mul:
2231  case Instruction::FMul:
2232  case Instruction::UDiv:
2233  case Instruction::SDiv:
2234  case Instruction::FDiv:
2235  case Instruction::URem:
2236  case Instruction::SRem:
2237  case Instruction::FRem:
2238  case Instruction::Shl:
2239  case Instruction::LShr:
2240  case Instruction::AShr:
2241  case Instruction::And:
2242  case Instruction::Or:
2243  case Instruction::Xor: {
2244  // Certain instructions can be cheaper to vectorize if they have a
2245  // constant second vector operand.
2254 
2255  // If all operands are exactly the same ConstantInt then set the
2256  // operand kind to OK_UniformConstantValue.
2257  // If instead not all operands are constants, then set the operand kind
2258  // to OK_AnyValue. If all operands are constants but not the same,
2259  // then set the operand kind to OK_NonUniformConstantValue.
2260  ConstantInt *CInt0 = nullptr;
2261  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2262  const Instruction *I = cast<Instruction>(VL[i]);
2263  ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(1));
2264  if (!CInt) {
2267  break;
2268  }
2269  if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
2270  !CInt->getValue().isPowerOf2())
2272  if (i == 0) {
2273  CInt0 = CInt;
2274  continue;
2275  }
2276  if (CInt0 != CInt)
2278  }
2279 
2280  SmallVector<const Value *, 4> Operands(VL0->operand_values());
2281  int ScalarEltCost = TTI->getArithmeticInstrCost(
2282  S.getOpcode(), ScalarTy, Op1VK, Op2VK, Op1VP, Op2VP, Operands);
2283  if (NeedToShuffleReuses) {
2284  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2285  }
2286  int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2287  int VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy, Op1VK,
2288  Op2VK, Op1VP, Op2VP, Operands);
2289  return ReuseShuffleCost + VecCost - ScalarCost;
2290  }
2291  case Instruction::GetElementPtr: {
2296 
2297  int ScalarEltCost =
2298  TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2299  if (NeedToShuffleReuses) {
2300  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2301  }
2302  int ScalarCost = VecTy->getNumElements() * ScalarEltCost;
2303  int VecCost =
2304  TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2305  return ReuseShuffleCost + VecCost - ScalarCost;
2306  }
2307  case Instruction::Load: {
2308  // Cost of wide load - cost of scalar loads.
2309  unsigned alignment = cast<LoadInst>(VL0)->getAlignment();
2310  int ScalarEltCost =
2311  TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2312  if (NeedToShuffleReuses) {
2313  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2314  }
2315  int ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
2316  int VecLdCost =
2317  TTI->getMemoryOpCost(Instruction::Load, VecTy, alignment, 0, VL0);
2318  if (!E->ReorderIndices.empty()) {
2319  // TODO: Merge this shuffle with the ReuseShuffleCost.
2320  VecLdCost += TTI->getShuffleCost(
2322  }
2323  return ReuseShuffleCost + VecLdCost - ScalarLdCost;
2324  }
2325  case Instruction::Store: {
2326  // We know that we can merge the stores. Calculate the cost.
2327  unsigned alignment = cast<StoreInst>(VL0)->getAlignment();
2328  int ScalarEltCost =
2329  TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2330  if (NeedToShuffleReuses) {
2331  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2332  }
2333  int ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
2334  int VecStCost =
2335  TTI->getMemoryOpCost(Instruction::Store, VecTy, alignment, 0, VL0);
2336  return ReuseShuffleCost + VecStCost - ScalarStCost;
2337  }
2338  case Instruction::Call: {
2339  CallInst *CI = cast<CallInst>(VL0);
2341 
2342  // Calculate the cost of the scalar and vector calls.
2343  SmallVector<Type *, 4> ScalarTys;
2344  for (unsigned op = 0, opc = CI->getNumArgOperands(); op != opc; ++op)
2345  ScalarTys.push_back(CI->getArgOperand(op)->getType());
2346 
2347  FastMathFlags FMF;
2348  if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2349  FMF = FPMO->getFastMathFlags();
2350 
2351  int ScalarEltCost =
2352  TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2353  if (NeedToShuffleReuses) {
2354  ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
2355  }
2356  int ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
2357 
2359  int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2360  VecTy->getNumElements());
2361 
2362  LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCost
2363  << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2364  << " for " << *CI << "\n");
2365 
2366  return ReuseShuffleCost + VecCallCost - ScalarCallCost;
2367  }
2368  case Instruction::ShuffleVector: {
2369  assert(S.isAltShuffle() &&
2370  ((Instruction::isBinaryOp(S.getOpcode()) &&
2371  Instruction::isBinaryOp(S.getAltOpcode())) ||
2372  (Instruction::isCast(S.getOpcode()) &&
2373  Instruction::isCast(S.getAltOpcode()))) &&
2374  "Invalid Shuffle Vector Operand");
2375  int ScalarCost = 0;
2376  if (NeedToShuffleReuses) {
2377  for (unsigned Idx : E->ReuseShuffleIndices) {
2378  Instruction *I = cast<Instruction>(VL[Idx]);
2379  ReuseShuffleCost -= TTI->getInstructionCost(
2381  }
2382  for (Value *V : VL) {
2383  Instruction *I = cast<Instruction>(V);
2384  ReuseShuffleCost += TTI->getInstructionCost(
2386  }
2387  }
2388  for (Value *i : VL) {
2389  Instruction *I = cast<Instruction>(i);
2390  assert(S.isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
2391  ScalarCost += TTI->getInstructionCost(
2393  }
2394  // VecCost is equal to sum of the cost of creating 2 vectors
2395  // and the cost of creating shuffle.
2396  int VecCost = 0;
2397  if (Instruction::isBinaryOp(S.getOpcode())) {
2398  VecCost = TTI->getArithmeticInstrCost(S.getOpcode(), VecTy);
2399  VecCost += TTI->getArithmeticInstrCost(S.getAltOpcode(), VecTy);
2400  } else {
2401  Type *Src0SclTy = S.MainOp->getOperand(0)->getType();
2402  Type *Src1SclTy = S.AltOp->getOperand(0)->getType();
2403  VectorType *Src0Ty = VectorType::get(Src0SclTy, VL.size());
2404  VectorType *Src1Ty = VectorType::get(Src1SclTy, VL.size());
2405  VecCost = TTI->getCastInstrCost(S.getOpcode(), VecTy, Src0Ty);
2406  VecCost += TTI->getCastInstrCost(S.getAltOpcode(), VecTy, Src1Ty);
2407  }
2408  VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_Select, VecTy, 0);
2409  return ReuseShuffleCost + VecCost - ScalarCost;
2410  }
2411  default:
2412  llvm_unreachable("Unknown instruction");
2413  }
2414 }
2415 
2416 bool BoUpSLP::isFullyVectorizableTinyTree() {
2417  LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "
2418  << VectorizableTree.size() << " is fully vectorizable .\n");
2419 
2420  // We only handle trees of heights 1 and 2.
2421  if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2422  return true;
2423 
2424  if (VectorizableTree.size() != 2)
2425  return false;
2426 
2427  // Handle splat and all-constants stores.
2428  if (!VectorizableTree[0].NeedToGather &&
2429  (allConstant(VectorizableTree[1].Scalars) ||
2430  isSplat(VectorizableTree[1].Scalars)))
2431  return true;
2432 
2433  // Gathering cost would be too much for tiny trees.
2434  if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2435  return false;
2436 
2437  return true;
2438 }
2439 
2440 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2441  // We can vectorize the tree if its size is greater than or equal to the
2442  // minimum size specified by the MinTreeSize command line option.
2443  if (VectorizableTree.size() >= MinTreeSize)
2444  return false;
2445 
2446  // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2447  // can vectorize it if we can prove it fully vectorizable.
2448  if (isFullyVectorizableTinyTree())
2449  return false;
2450 
2451  assert(VectorizableTree.empty()
2452  ? ExternalUses.empty()
2453  : true && "We shouldn't have any external users");
2454 
2455  // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2456  // vectorizable.
2457  return true;
2458 }
2459 
2460 int BoUpSLP::getSpillCost() {
2461  // Walk from the bottom of the tree to the top, tracking which values are
2462  // live. When we see a call instruction that is not part of our tree,
2463  // query TTI to see if there is a cost to keeping values live over it
2464  // (for example, if spills and fills are required).
2465  unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2466  int Cost = 0;
2467 
2468  SmallPtrSet<Instruction*, 4> LiveValues;
2469  Instruction *PrevInst = nullptr;
2470 
2471  for (const auto &N : VectorizableTree) {
2472  Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2473  if (!Inst)
2474  continue;
2475 
2476  if (!PrevInst) {
2477  PrevInst = Inst;
2478  continue;
2479  }
2480 
2481  // Update LiveValues.
2482  LiveValues.erase(PrevInst);
2483  for (auto &J : PrevInst->operands()) {
2484  if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2485  LiveValues.insert(cast<Instruction>(&*J));
2486  }
2487 
2488  LLVM_DEBUG({
2489  dbgs() << "SLP: #LV: " << LiveValues.size();
2490  for (auto *X : LiveValues)
2491  dbgs() << " " << X->getName();
2492  dbgs() << ", Looking at ";
2493  Inst->dump();
2494  });
2495 
2496  // Now find the sequence of instructions between PrevInst and Inst.
2498  PrevInstIt =
2499  PrevInst->getIterator().getReverse();
2500  while (InstIt != PrevInstIt) {
2501  if (PrevInstIt == PrevInst->getParent()->rend()) {
2502  PrevInstIt = Inst->getParent()->rbegin();
2503  continue;
2504  }
2505 
2506  // Debug informations don't impact spill cost.
2507  if ((isa<CallInst>(&*PrevInstIt) &&
2508  !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
2509  &*PrevInstIt != PrevInst) {
2511  for (auto *II : LiveValues)
2512  V.push_back(VectorType::get(II->getType(), BundleWidth));
2513  Cost += TTI->getCostOfKeepingLiveOverCall(V);
2514  }
2515 
2516  ++PrevInstIt;
2517  }
2518 
2519  PrevInst = Inst;
2520  }
2521 
2522  return Cost;
2523 }
2524 
2525 int BoUpSLP::getTreeCost() {
2526  int Cost = 0;
2527  LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
2528  << VectorizableTree.size() << ".\n");
2529 
2530  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2531 
2532  for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
2533  TreeEntry &TE = VectorizableTree[I];
2534 
2535  // We create duplicate tree entries for gather sequences that have multiple
2536  // uses. However, we should not compute the cost of duplicate sequences.
2537  // For example, if we have a build vector (i.e., insertelement sequence)
2538  // that is used by more than one vector instruction, we only need to
2539  // compute the cost of the insertelement instructions once. The redundant
2540  // instructions will be eliminated by CSE.
2541  //
2542  // We should consider not creating duplicate tree entries for gather
2543  // sequences, and instead add additional edges to the tree representing
2544  // their uses. Since such an approach results in fewer total entries,
2545  // existing heuristics based on tree size may yield different results.
2546  //
2547  if (TE.NeedToGather &&
2548  std::any_of(std::next(VectorizableTree.begin(), I + 1),
2549  VectorizableTree.end(), [TE](TreeEntry &Entry) {
2550  return Entry.NeedToGather && Entry.isSame(TE.Scalars);
2551  }))
2552  continue;
2553 
2554  int C = getEntryCost(&TE);
2555  LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
2556  << " for bundle that starts with " << *TE.Scalars[0]
2557  << ".\n");
2558  Cost += C;
2559  }
2560 
2561  SmallPtrSet<Value *, 16> ExtractCostCalculated;
2562  int ExtractCost = 0;
2563  for (ExternalUser &EU : ExternalUses) {
2564  // We only add extract cost once for the same scalar.
2565  if (!ExtractCostCalculated.insert(EU.Scalar).second)
2566  continue;
2567 
2568  // Uses by ephemeral values are free (because the ephemeral value will be
2569  // removed prior to code generation, and so the extraction will be
2570  // removed as well).
2571  if (EphValues.count(EU.User))
2572  continue;
2573 
2574  // If we plan to rewrite the tree in a smaller type, we will need to sign
2575  // extend the extracted value back to the original type. Here, we account
2576  // for the extract and the added cost of the sign extend if needed.
2577  auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2578  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2579  if (MinBWs.count(ScalarRoot)) {
2580  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2581  auto Extend =
2582  MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2583  VecTy = VectorType::get(MinTy, BundleWidth);
2584  ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2585  VecTy, EU.Lane);
2586  } else {
2587  ExtractCost +=
2588  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2589  }
2590  }
2591 
2592  int SpillCost = getSpillCost();
2593  Cost += SpillCost + ExtractCost;
2594 
2595  std::string Str;
2596  {
2597  raw_string_ostream OS(Str);
2598  OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2599  << "SLP: Extract Cost = " << ExtractCost << ".\n"
2600  << "SLP: Total Cost = " << Cost << ".\n";
2601  }
2602  LLVM_DEBUG(dbgs() << Str);
2603 
2604  if (ViewSLPTree)
2605  ViewGraph(this, "SLP" + F->getName(), false, Str);
2606 
2607  return Cost;
2608 }
2609 
2610 int BoUpSLP::getGatherCost(Type *Ty,
2611  const DenseSet<unsigned> &ShuffledIndices) {
2612  int Cost = 0;
2613  for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2614  if (!ShuffledIndices.count(i))
2615  Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2616  if (!ShuffledIndices.empty())
2617  Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, Ty);
2618  return Cost;
2619 }
2620 
2621 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2622  // Find the type of the operands in VL.
2623  Type *ScalarTy = VL[0]->getType();
2624  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2625  ScalarTy = SI->getValueOperand()->getType();
2626  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2627  // Find the cost of inserting/extracting values from the vector.
2628  // Check if the same elements are inserted several times and count them as
2629  // shuffle candidates.
2630  DenseSet<unsigned> ShuffledElements;
2631  DenseSet<Value *> UniqueElements;
2632  // Iterate in reverse order to consider insert elements with the high cost.
2633  for (unsigned I = VL.size(); I > 0; --I) {
2634  unsigned Idx = I - 1;
2635  if (!UniqueElements.insert(VL[Idx]).second)
2636  ShuffledElements.insert(Idx);
2637  }
2638  return getGatherCost(VecTy, ShuffledElements);
2639 }
2640 
2641 // Reorder commutative operations in alternate shuffle if the resulting vectors
2642 // are consecutive loads. This would allow us to vectorize the tree.
2643 // If we have something like-
2644 // load a[0] - load b[0]
2645 // load b[1] + load a[1]
2646 // load a[2] - load b[2]
2647 // load a[3] + load b[3]
2648 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2649 // code.
2650 void BoUpSLP::reorderAltShuffleOperands(const InstructionsState &S,
2651  ArrayRef<Value *> VL,
2654  // Push left and right operands of binary operation into Left and Right
2655  for (Value *V : VL) {
2656  auto *I = cast<Instruction>(V);
2657  assert(S.isOpcodeOrAlt(I) && "Incorrect instruction in vector");
2658  Left.push_back(I->getOperand(0));
2659  Right.push_back(I->getOperand(1));
2660  }
2661 
2662  // Reorder if we have a commutative operation and consecutive access
2663  // are on either side of the alternate instructions.
2664  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2665  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2666  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2667  Instruction *VL1 = cast<Instruction>(VL[j]);
2668  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2669  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2670  std::swap(Left[j], Right[j]);
2671  continue;
2672  } else if (VL2->isCommutative() &&
2673  isConsecutiveAccess(L, L1, *DL, *SE)) {
2674  std::swap(Left[j + 1], Right[j + 1]);
2675  continue;
2676  }
2677  // else unchanged
2678  }
2679  }
2680  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2681  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2682  Instruction *VL1 = cast<Instruction>(VL[j]);
2683  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2684  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2685  std::swap(Left[j], Right[j]);
2686  continue;
2687  } else if (VL2->isCommutative() &&
2688  isConsecutiveAccess(L, L1, *DL, *SE)) {
2689  std::swap(Left[j + 1], Right[j + 1]);
2690  continue;
2691  }
2692  // else unchanged
2693  }
2694  }
2695  }
2696 }
2697 
2698 // Return true if I should be commuted before adding it's left and right
2699 // operands to the arrays Left and Right.
2700 //
2701 // The vectorizer is trying to either have all elements one side being
2702 // instruction with the same opcode to enable further vectorization, or having
2703 // a splat to lower the vectorizing cost.
2705  int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2706  ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2707  bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2708  VLeft = I.getOperand(0);
2709  VRight = I.getOperand(1);
2710  // If we have "SplatRight", try to see if commuting is needed to preserve it.
2711  if (SplatRight) {
2712  if (VRight == Right[i - 1])
2713  // Preserve SplatRight
2714  return false;
2715  if (VLeft == Right[i - 1]) {
2716  // Commuting would preserve SplatRight, but we don't want to break
2717  // SplatLeft either, i.e. preserve the original order if possible.
2718  // (FIXME: why do we care?)
2719  if (SplatLeft && VLeft == Left[i - 1])
2720  return false;
2721  return true;
2722  }
2723  }
2724  // Symmetrically handle Right side.
2725  if (SplatLeft) {
2726  if (VLeft == Left[i - 1])
2727  // Preserve SplatLeft
2728  return false;
2729  if (VRight == Left[i - 1])
2730  return true;
2731  }
2732 
2733  Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2734  Instruction *IRight = dyn_cast<Instruction>(VRight);
2735 
2736  // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2737  // it and not the right, in this case we want to commute.
2738  if (AllSameOpcodeRight) {
2739  unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2740  if (IRight && RightPrevOpcode == IRight->getOpcode())
2741  // Do not commute, a match on the right preserves AllSameOpcodeRight
2742  return false;
2743  if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2744  // We have a match and may want to commute, but first check if there is
2745  // not also a match on the existing operands on the Left to preserve
2746  // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2747  // (FIXME: why do we care?)
2748  if (AllSameOpcodeLeft && ILeft &&
2749  cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2750  return false;
2751  return true;
2752  }
2753  }
2754  // Symmetrically handle Left side.
2755  if (AllSameOpcodeLeft) {
2756  unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2757  if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2758  return false;
2759  if (IRight && LeftPrevOpcode == IRight->getOpcode())
2760  return true;
2761  }
2762  return false;
2763 }
2764 
2765 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2766  ArrayRef<Value *> VL,
2768  SmallVectorImpl<Value *> &Right) {
2769  if (!VL.empty()) {
2770  // Peel the first iteration out of the loop since there's nothing
2771  // interesting to do anyway and it simplifies the checks in the loop.
2772  auto *I = cast<Instruction>(VL[0]);
2773  Value *VLeft = I->getOperand(0);
2774  Value *VRight = I->getOperand(1);
2775  if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2776  // Favor having instruction to the right. FIXME: why?
2777  std::swap(VLeft, VRight);
2778  Left.push_back(VLeft);
2779  Right.push_back(VRight);
2780  }
2781 
2782  // Keep track if we have instructions with all the same opcode on one side.
2783  bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2784  bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2785  // Keep track if we have one side with all the same value (broadcast).
2786  bool SplatLeft = true;
2787  bool SplatRight = true;
2788 
2789  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2790  Instruction *I = cast<Instruction>(VL[i]);
2791  assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2792  (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2793  "Can only process commutative instruction");
2794  // Commute to favor either a splat or maximizing having the same opcodes on
2795  // one side.
2796  Value *VLeft;
2797  Value *VRight;
2798  if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2799  AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2800  VRight)) {
2801  Left.push_back(VRight);
2802  Right.push_back(VLeft);
2803  } else {
2804  Left.push_back(VLeft);
2805  Right.push_back(VRight);
2806  }
2807  // Update Splat* and AllSameOpcode* after the insertion.
2808  SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2809  SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2810  AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2811  (cast<Instruction>(Left[i - 1])->getOpcode() ==
2812  cast<Instruction>(Left[i])->getOpcode());
2813  AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2814  (cast<Instruction>(Right[i - 1])->getOpcode() ==
2815  cast<Instruction>(Right[i])->getOpcode());
2816  }
2817 
2818  // If one operand end up being broadcast, return this operand order.
2819  if (SplatRight || SplatLeft)
2820  return;
2821 
2822  // Finally check if we can get longer vectorizable chain by reordering
2823  // without breaking the good operand order detected above.
2824  // E.g. If we have something like-
2825  // load a[0] load b[0]
2826  // load b[1] load a[1]
2827  // load a[2] load b[2]
2828  // load a[3] load b[3]
2829  // Reordering the second load b[1] load a[1] would allow us to vectorize
2830  // this code and we still retain AllSameOpcode property.
2831  // FIXME: This load reordering might break AllSameOpcode in some rare cases
2832  // such as-
2833  // add a[0],c[0] load b[0]
2834  // add a[1],c[2] load b[1]
2835  // b[2] load b[2]
2836  // add a[3],c[3] load b[3]
2837  for (unsigned j = 0, e = VL.size() - 1; j < e; ++j) {
2838  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2839  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2840  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2841  std::swap(Left[j + 1], Right[j + 1]);
2842  continue;
2843  }
2844  }
2845  }
2846  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2847  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2848  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2849  std::swap(Left[j + 1], Right[j + 1]);
2850  continue;
2851  }
2852  }
2853  }
2854  // else unchanged
2855  }
2856 }
2857 
2858 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL,
2859  const InstructionsState &S) {
2860  // Get the basic block this bundle is in. All instructions in the bundle
2861  // should be in this block.
2862  auto *Front = cast<Instruction>(S.OpValue);
2863  auto *BB = Front->getParent();
2864  assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2865  auto *I = cast<Instruction>(V);
2866  return !S.isOpcodeOrAlt(I) || I->getParent() == BB;
2867  }));
2868 
2869  // The last instruction in the bundle in program order.
2870  Instruction *LastInst = nullptr;
2871 
2872  // Find the last instruction. The common case should be that BB has been
2873  // scheduled, and the last instruction is VL.back(). So we start with
2874  // VL.back() and iterate over schedule data until we reach the end of the
2875  // bundle. The end of the bundle is marked by null ScheduleData.
2876  if (BlocksSchedules.count(BB)) {
2877  auto *Bundle =
2878  BlocksSchedules[BB]->getScheduleData(isOneOf(S, VL.back()));
2879  if (Bundle && Bundle->isPartOfBundle())
2880  for (; Bundle; Bundle = Bundle->NextInBundle)
2881  if (Bundle->OpValue == Bundle->Inst)
2882  LastInst = Bundle->Inst;
2883  }
2884 
2885  // LastInst can still be null at this point if there's either not an entry
2886  // for BB in BlocksSchedules or there's no ScheduleData available for
2887  // VL.back(). This can be the case if buildTree_rec aborts for various
2888  // reasons (e.g., the maximum recursion depth is reached, the maximum region
2889  // size is reached, etc.). ScheduleData is initialized in the scheduling
2890  // "dry-run".
2891  //
2892  // If this happens, we can still find the last instruction by brute force. We
2893  // iterate forwards from Front (inclusive) until we either see all
2894  // instructions in the bundle or reach the end of the block. If Front is the
2895  // last instruction in program order, LastInst will be set to Front, and we
2896  // will visit all the remaining instructions in the block.
2897  //
2898  // One of the reasons we exit early from buildTree_rec is to place an upper
2899  // bound on compile-time. Thus, taking an additional compile-time hit here is
2900  // not ideal. However, this should be exceedingly rare since it requires that
2901  // we both exit early from buildTree_rec and that the bundle be out-of-order
2902  // (causing us to iterate all the way to the end of the block).
2903  if (!LastInst) {
2904  SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2905  for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2906  if (Bundle.erase(&I) && S.isOpcodeOrAlt(&I))
2907  LastInst = &I;
2908  if (Bundle.empty())
2909  break;
2910  }
2911  }
2912 
2913  // Set the insertion point after the last instruction in the bundle. Set the
2914  // debug location to Front.
2915  Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2916  Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2917 }
2918 
2919 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2920  Value *Vec = UndefValue::get(Ty);
2921  // Generate the 'InsertElement' instruction.
2922  for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2923  Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2924  if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2925  GatherSeq.insert(Insrt);
2926  CSEBlocks.insert(Insrt->getParent());
2927 
2928  // Add to our 'need-to-extract' list.
2929  if (TreeEntry *E = getTreeEntry(VL[i])) {
2930  // Find which lane we need to extract.
2931  int FoundLane = -1;
2932  for (unsigned Lane = 0, LE = E->Scalars.size(); Lane != LE; ++Lane) {
2933  // Is this the lane of the scalar that we are looking for ?
2934  if (E->Scalars[Lane] == VL[i]) {
2935  FoundLane = Lane;
2936  break;
2937  }
2938  }
2939  assert(FoundLane >= 0 && "Could not find the correct lane");
2940  if (!E->ReuseShuffleIndices.empty()) {
2941  FoundLane =
2942  std::distance(E->ReuseShuffleIndices.begin(),
2943  llvm::find(E->ReuseShuffleIndices, FoundLane));
2944  }
2945  ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2946  }
2947  }
2948  }
2949 
2950  return Vec;
2951 }
2952 
2953 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2954  InstructionsState S = getSameOpcode(VL);
2955  if (S.getOpcode()) {
2956  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2957  if (E->isSame(VL)) {
2958  Value *V = vectorizeTree(E);
2959  if (VL.size() == E->Scalars.size() && !E->ReuseShuffleIndices.empty()) {
2960  // We need to get the vectorized value but without shuffle.
2961  if (auto *SV = dyn_cast<ShuffleVectorInst>(V)) {
2962  V = SV->getOperand(0);
2963  } else {
2964  // Reshuffle to get only unique values.
2965  SmallVector<unsigned, 4> UniqueIdxs;
2966  SmallSet<unsigned, 4> UsedIdxs;
2967  for(unsigned Idx : E->ReuseShuffleIndices)
2968  if (UsedIdxs.insert(Idx).second)
2969  UniqueIdxs.emplace_back(Idx);
2970  V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
2971  UniqueIdxs);
2972  }
2973  }
2974  return V;
2975  }
2976  }
2977  }
2978 
2979  Type *ScalarTy = S.OpValue->getType();
2980  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2981  ScalarTy = SI->getValueOperand()->getType();
2982 
2983  // Check that every instruction appears once in this bundle.
2984  SmallVector<unsigned, 4> ReuseShuffleIndicies;
2985  SmallVector<Value *, 4> UniqueValues;
2986  if (VL.size() > 2) {
2987  DenseMap<Value *, unsigned> UniquePositions;
2988  for (Value *V : VL) {
2989  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
2990  ReuseShuffleIndicies.emplace_back(Res.first->second);
2991  if (Res.second || isa<Constant>(V))
2992  UniqueValues.emplace_back(V);
2993  }
2994  // Do not shuffle single element or if number of unique values is not power
2995  // of 2.
2996  if (UniqueValues.size() == VL.size() || UniqueValues.size() <= 1 ||
2997  !llvm::isPowerOf2_32(UniqueValues.size()))
2998  ReuseShuffleIndicies.clear();
2999  else
3000  VL = UniqueValues;
3001  }
3002  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
3003 
3004  Value *V = Gather(VL, VecTy);
3005  if (!ReuseShuffleIndicies.empty()) {
3006  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3007  ReuseShuffleIndicies, "shuffle");
3008  if (auto *I = dyn_cast<Instruction>(V)) {
3009  GatherSeq.insert(I);
3010  CSEBlocks.insert(I->getParent());
3011  }
3012  }
3013  return V;
3014 }
3015 
3018  Mask.clear();
3019  const unsigned E = Indices.size();
3020  Mask.resize(E);
3021  for (unsigned I = 0; I < E; ++I)
3022  Mask[Indices[I]] = I;
3023 }
3024 
3025 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
3026  IRBuilder<>::InsertPointGuard Guard(Builder);
3027 
3028  if (E->VectorizedValue) {
3029  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
3030  return E->VectorizedValue;
3031  }
3032 
3033  InstructionsState S = getSameOpcode(E->Scalars);
3034  Instruction *VL0 = cast<Instruction>(S.OpValue);
3035  Type *ScalarTy = VL0->getType();
3036  if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
3037  ScalarTy = SI->getValueOperand()->getType();
3038  VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
3039 
3040  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
3041 
3042  if (E->NeedToGather) {
3043  setInsertPointAfterBundle(E->Scalars, S);
3044  auto *V = Gather(E->Scalars, VecTy);
3045  if (NeedToShuffleReuses) {
3046  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3047  E->ReuseShuffleIndices, "shuffle");
3048  if (auto *I = dyn_cast<Instruction>(V)) {
3049  GatherSeq.insert(I);
3050  CSEBlocks.insert(I->getParent());
3051  }
3052  }
3053  E->VectorizedValue = V;
3054  return V;
3055  }
3056 
3057  unsigned ShuffleOrOp = S.isAltShuffle() ?
3058  (unsigned) Instruction::ShuffleVector : S.getOpcode();
3059  switch (ShuffleOrOp) {
3060  case Instruction::PHI: {
3061  PHINode *PH = dyn_cast<PHINode>(VL0);
3062  Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
3063  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3064  PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
3065  Value *V = NewPhi;
3066  if (NeedToShuffleReuses) {
3067  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3068  E->ReuseShuffleIndices, "shuffle");
3069  }
3070  E->VectorizedValue = V;
3071 
3072  // PHINodes may have multiple entries from the same block. We want to
3073  // visit every block once.
3074  SmallPtrSet<BasicBlock*, 4> VisitedBBs;
3075 
3076  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
3077  ValueList Operands;
3078  BasicBlock *IBB = PH->getIncomingBlock(i);
3079 
3080  if (!VisitedBBs.insert(IBB).second) {
3081  NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
3082  continue;
3083  }
3084 
3085  // Prepare the operand vector.
3086  for (Value *V : E->Scalars)
3087  Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
3088 
3089  Builder.SetInsertPoint(IBB->getTerminator());
3090  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
3091  Value *Vec = vectorizeTree(Operands);
3092  NewPhi->addIncoming(Vec, IBB);
3093  }
3094 
3095  assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
3096  "Invalid number of incoming values");
3097  return V;
3098  }
3099 
3100  case Instruction::ExtractElement: {
3101  if (!E->NeedToGather) {
3102  Value *V = VL0->getOperand(0);
3103  if (!E->ReorderIndices.empty()) {
3104  OrdersType Mask;
3105  inversePermutation(E->ReorderIndices, Mask);
3106  Builder.SetInsertPoint(VL0);
3107  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy), Mask,
3108  "reorder_shuffle");
3109  }
3110  if (NeedToShuffleReuses) {
3111  // TODO: Merge this shuffle with the ReorderShuffleMask.
3112  if (E->ReorderIndices.empty())
3113  Builder.SetInsertPoint(VL0);
3114  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3115  E->ReuseShuffleIndices, "shuffle");
3116  }
3117  E->VectorizedValue = V;
3118  return V;
3119  }
3120  setInsertPointAfterBundle(E->Scalars, S);
3121  auto *V = Gather(E->Scalars, VecTy);
3122  if (NeedToShuffleReuses) {
3123  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3124  E->ReuseShuffleIndices, "shuffle");
3125  if (auto *I = dyn_cast<Instruction>(V)) {
3126  GatherSeq.insert(I);
3127  CSEBlocks.insert(I->getParent());
3128  }
3129  }
3130  E->VectorizedValue = V;
3131  return V;
3132  }
3133  case Instruction::ExtractValue: {
3134  if (!E->NeedToGather) {
3135  LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
3136  Builder.SetInsertPoint(LI);
3137  PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
3138  Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
3139  LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
3140  Value *NewV = propagateMetadata(V, E->Scalars);
3141  if (!E->ReorderIndices.empty()) {
3142  OrdersType Mask;
3143  inversePermutation(E->ReorderIndices, Mask);
3144  NewV = Builder.CreateShuffleVector(NewV, UndefValue::get(VecTy), Mask,
3145  "reorder_shuffle");
3146  }
3147  if (NeedToShuffleReuses) {
3148  // TODO: Merge this shuffle with the ReorderShuffleMask.
3149  NewV = Builder.CreateShuffleVector(
3150  NewV, UndefValue::get(VecTy), E->ReuseShuffleIndices, "shuffle");
3151  }
3152  E->VectorizedValue = NewV;
3153  return NewV;
3154  }
3155  setInsertPointAfterBundle(E->Scalars, S);
3156  auto *V = Gather(E->Scalars, VecTy);
3157  if (NeedToShuffleReuses) {
3158  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3159  E->ReuseShuffleIndices, "shuffle");
3160  if (auto *I = dyn_cast<Instruction>(V)) {
3161  GatherSeq.insert(I);
3162  CSEBlocks.insert(I->getParent());
3163  }
3164  }
3165  E->VectorizedValue = V;
3166  return V;
3167  }
3168  case Instruction::ZExt:
3169  case Instruction::SExt:
3170  case Instruction::FPToUI:
3171  case Instruction::FPToSI:
3172  case Instruction::FPExt:
3173  case Instruction::PtrToInt:
3174  case Instruction::IntToPtr:
3175  case Instruction::SIToFP:
3176  case Instruction::UIToFP:
3177  case Instruction::Trunc:
3178  case Instruction::FPTrunc:
3179  case Instruction::BitCast: {
3180  ValueList INVL;
3181  for (Value *V : E->Scalars)
3182  INVL.push_back(cast<Instruction>(V)->getOperand(0));
3183 
3184  setInsertPointAfterBundle(E->Scalars, S);
3185 
3186  Value *InVec = vectorizeTree(INVL);
3187 
3188  if (E->VectorizedValue) {
3189  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3190  return E->VectorizedValue;
3191  }
3192 
3193  CastInst *CI = dyn_cast<CastInst>(VL0);
3194  Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
3195  if (NeedToShuffleReuses) {
3196  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3197  E->ReuseShuffleIndices, "shuffle");
3198  }
3199  E->VectorizedValue = V;
3200  ++NumVectorInstructions;
3201  return V;
3202  }
3203  case Instruction::FCmp:
3204  case Instruction::ICmp: {
3205  ValueList LHSV, RHSV;
3206  for (Value *V : E->Scalars) {
3207  LHSV.push_back(cast<Instruction>(V)->getOperand(0));
3208  RHSV.push_back(cast<Instruction>(V)->getOperand(1));
3209  }
3210 
3211  setInsertPointAfterBundle(E->Scalars, S);
3212 
3213  Value *L = vectorizeTree(LHSV);
3214  Value *R = vectorizeTree(RHSV);
3215 
3216  if (E->VectorizedValue) {
3217  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3218  return E->VectorizedValue;
3219  }
3220 
3221  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3222  Value *V;
3223  if (S.getOpcode() == Instruction::FCmp)
3224  V = Builder.CreateFCmp(P0, L, R);
3225  else
3226  V = Builder.CreateICmp(P0, L, R);
3227 
3228  propagateIRFlags(V, E->Scalars, VL0);
3229  if (NeedToShuffleReuses) {
3230  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3231  E->ReuseShuffleIndices, "shuffle");
3232  }
3233  E->VectorizedValue = V;
3234  ++NumVectorInstructions;
3235  return V;
3236  }
3237  case Instruction::Select: {
3238  ValueList TrueVec, FalseVec, CondVec;
3239  for (Value *V : E->Scalars) {
3240  CondVec.push_back(cast<Instruction>(V)->getOperand(0));
3241  TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
3242  FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
3243  }
3244 
3245  setInsertPointAfterBundle(E->Scalars, S);
3246 
3247  Value *Cond = vectorizeTree(CondVec);
3248  Value *True = vectorizeTree(TrueVec);
3249  Value *False = vectorizeTree(FalseVec);
3250 
3251  if (E->VectorizedValue) {
3252  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3253  return E->VectorizedValue;
3254  }
3255 
3256  Value *V = Builder.CreateSelect(Cond, True, False);
3257  if (NeedToShuffleReuses) {
3258  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3259  E->ReuseShuffleIndices, "shuffle");
3260  }
3261  E->VectorizedValue = V;
3262  ++NumVectorInstructions;
3263  return V;
3264  }
3265  case Instruction::Add:
3266  case Instruction::FAdd:
3267  case Instruction::Sub:
3268  case Instruction::FSub:
3269  case Instruction::Mul:
3270  case Instruction::FMul:
3271  case Instruction::UDiv:
3272  case Instruction::SDiv:
3273  case Instruction::FDiv:
3274  case Instruction::URem:
3275  case Instruction::SRem:
3276  case Instruction::FRem:
3277  case Instruction::Shl:
3278  case Instruction::LShr:
3279  case Instruction::AShr:
3280  case Instruction::And:
3281  case Instruction::Or:
3282  case Instruction::Xor: {
3283  ValueList LHSVL, RHSVL;
3284  if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
3285  reorderInputsAccordingToOpcode(S.getOpcode(), E->Scalars, LHSVL,
3286  RHSVL);
3287  else
3288  for (Value *V : E->Scalars) {
3289  auto *I = cast<Instruction>(V);
3290  LHSVL.push_back(I->getOperand(0));
3291  RHSVL.push_back(I->getOperand(1));
3292  }
3293 
3294  setInsertPointAfterBundle(E->Scalars, S);
3295 
3296  Value *LHS = vectorizeTree(LHSVL);
3297  Value *RHS = vectorizeTree(RHSVL);
3298 
3299  if (E->VectorizedValue) {
3300  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3301  return E->VectorizedValue;
3302  }
3303 
3304  Value *V = Builder.CreateBinOp(
3305  static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3306  propagateIRFlags(V, E->Scalars, VL0);
3307  if (auto *I = dyn_cast<Instruction>(V))
3308  V = propagateMetadata(I, E->Scalars);
3309 
3310  if (NeedToShuffleReuses) {
3311  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3312  E->ReuseShuffleIndices, "shuffle");
3313  }
3314  E->VectorizedValue = V;
3315  ++NumVectorInstructions;
3316 
3317  return V;
3318  }
3319  case Instruction::Load: {
3320  // Loads are inserted at the head of the tree because we don't want to
3321  // sink them all the way down past store instructions.
3322  bool IsReorder = !E->ReorderIndices.empty();
3323  if (IsReorder) {
3324  S = getSameOpcode(E->Scalars, E->ReorderIndices.front());
3325  VL0 = cast<Instruction>(S.OpValue);
3326  }
3327  setInsertPointAfterBundle(E->Scalars, S);
3328 
3329  LoadInst *LI = cast<LoadInst>(VL0);
3330  Type *ScalarLoadTy = LI->getType();
3331  unsigned AS = LI->getPointerAddressSpace();
3332 
3333  Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
3334  VecTy->getPointerTo(AS));
3335 
3336  // The pointer operand uses an in-tree scalar so we add the new BitCast to
3337  // ExternalUses list to make sure that an extract will be generated in the
3338  // future.
3339  Value *PO = LI->getPointerOperand();
3340  if (getTreeEntry(PO))
3341  ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
3342 
3343  unsigned Alignment = LI->getAlignment();
3344  LI = Builder.CreateLoad(VecPtr);
3345  if (!Alignment) {
3346  Alignment = DL->getABITypeAlignment(ScalarLoadTy);
3347  }
3348  LI->setAlignment(Alignment);
3349  Value *V = propagateMetadata(LI, E->Scalars);
3350  if (IsReorder) {
3351  OrdersType Mask;
3352  inversePermutation(E->ReorderIndices, Mask);
3353  V = Builder.CreateShuffleVector(V, UndefValue::get(V->getType()),
3354  Mask, "reorder_shuffle");
3355  }
3356  if (NeedToShuffleReuses) {
3357  // TODO: Merge this shuffle with the ReorderShuffleMask.
3358  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3359  E->ReuseShuffleIndices, "shuffle");
3360  }
3361  E->VectorizedValue = V;
3362  ++NumVectorInstructions;
3363  return V;
3364  }
3365  case Instruction::Store: {
3366  StoreInst *SI = cast<StoreInst>(VL0);
3367  unsigned Alignment = SI->getAlignment();
3368  unsigned AS = SI->getPointerAddressSpace();
3369 
3370  ValueList ScalarStoreValues;
3371  for (Value *V : E->Scalars)
3372  ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3373 
3374  setInsertPointAfterBundle(E->Scalars, S);
3375 
3376  Value *VecValue = vectorizeTree(ScalarStoreValues);
3377  Value *ScalarPtr = SI->getPointerOperand();
3378  Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3379  StoreInst *ST = Builder.CreateStore(VecValue, VecPtr);
3380 
3381  // The pointer operand uses an in-tree scalar, so add the new BitCast to
3382  // ExternalUses to make sure that an extract will be generated in the
3383  // future.
3384  if (getTreeEntry(ScalarPtr))
3385  ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3386 
3387  if (!Alignment)
3388  Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3389 
3390  ST->setAlignment(Alignment);
3391  Value *V = propagateMetadata(ST, E->Scalars);
3392  if (NeedToShuffleReuses) {
3393  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3394  E->ReuseShuffleIndices, "shuffle");
3395  }
3396  E->VectorizedValue = V;
3397  ++NumVectorInstructions;
3398  return V;
3399  }
3400  case Instruction::GetElementPtr: {
3401  setInsertPointAfterBundle(E->Scalars, S);
3402 
3403  ValueList Op0VL;
3404  for (Value *V : E->Scalars)
3405  Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3406 
3407  Value *Op0 = vectorizeTree(Op0VL);
3408 
3409  std::vector<Value *> OpVecs;
3410  for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3411  ++j) {
3412  ValueList OpVL;
3413  for (Value *V : E->Scalars)
3414  OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3415 
3416  Value *OpVec = vectorizeTree(OpVL);
3417  OpVecs.push_back(OpVec);
3418  }
3419 
3420  Value *V = Builder.CreateGEP(
3421  cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3422  if (Instruction *I = dyn_cast<Instruction>(V))
3423  V = propagateMetadata(I, E->Scalars);
3424 
3425  if (NeedToShuffleReuses) {
3426  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3427  E->ReuseShuffleIndices, "shuffle");
3428  }
3429  E->VectorizedValue = V;
3430  ++NumVectorInstructions;
3431 
3432  return V;
3433  }
3434  case Instruction::Call: {
3435  CallInst *CI = cast<CallInst>(VL0);
3436  setInsertPointAfterBundle(E->Scalars, S);
3437  Function *FI;
3439  Value *ScalarArg = nullptr;
3440  if (CI && (FI = CI->getCalledFunction())) {
3441  IID = FI->getIntrinsicID();
3442  }
3443  std::vector<Value *> OpVecs;
3444  for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3445  ValueList OpVL;
3446  // ctlz,cttz and powi are special intrinsics whose second argument is
3447  // a scalar. This argument should not be vectorized.
3448  if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3449  CallInst *CEI = cast<CallInst>(VL0);
3450  ScalarArg = CEI->getArgOperand(j);
3451  OpVecs.push_back(CEI->getArgOperand(j));
3452  continue;
3453  }
3454  for (Value *V : E->Scalars) {
3455  CallInst *CEI = cast<CallInst>(V);
3456  OpVL.push_back(CEI->getArgOperand(j));
3457  }
3458 
3459  Value *OpVec = vectorizeTree(OpVL);
3460  LLVM_DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3461  OpVecs.push_back(OpVec);
3462  }
3463 
3464  Module *M = F->getParent();
3466  Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3467  Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3469  CI->getOperandBundlesAsDefs(OpBundles);
3470  Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3471 
3472  // The scalar argument uses an in-tree scalar so we add the new vectorized
3473  // call to ExternalUses list to make sure that an extract will be
3474  // generated in the future.
3475  if (ScalarArg && getTreeEntry(ScalarArg))
3476  ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3477 
3478  propagateIRFlags(V, E->Scalars, VL0);
3479  if (NeedToShuffleReuses) {
3480  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3481  E->ReuseShuffleIndices, "shuffle");
3482  }
3483  E->VectorizedValue = V;
3484  ++NumVectorInstructions;
3485  return V;
3486  }
3487  case Instruction::ShuffleVector: {
3488  ValueList LHSVL, RHSVL;
3489  assert(S.isAltShuffle() &&
3490  ((Instruction::isBinaryOp(S.getOpcode()) &&
3491  Instruction::isBinaryOp(S.getAltOpcode())) ||
3492  (Instruction::isCast(S.getOpcode()) &&
3493  Instruction::isCast(S.getAltOpcode()))) &&
3494  "Invalid Shuffle Vector Operand");
3495 
3496  Value *LHS, *RHS;
3497  if (Instruction::isBinaryOp(S.getOpcode())) {
3498  reorderAltShuffleOperands(S, E->Scalars, LHSVL, RHSVL);
3499  setInsertPointAfterBundle(E->Scalars, S);
3500  LHS = vectorizeTree(LHSVL);
3501  RHS = vectorizeTree(RHSVL);
3502  } else {
3503  ValueList INVL;
3504  for (Value *V : E->Scalars)
3505  INVL.push_back(cast<Instruction>(V)->getOperand(0));
3506  setInsertPointAfterBundle(E->Scalars, S);
3507  LHS = vectorizeTree(INVL);
3508  }
3509 
3510  if (E->VectorizedValue) {
3511  LLVM_DEBUG(dbgs() << "SLP: Diamond merged for " << *VL0 << ".\n");
3512  return E->VectorizedValue;
3513  }
3514 
3515  Value *V0, *V1;
3516  if (Instruction::isBinaryOp(S.getOpcode())) {
3517  V0 = Builder.CreateBinOp(
3518  static_cast<Instruction::BinaryOps>(S.getOpcode()), LHS, RHS);
3519  V1 = Builder.CreateBinOp(
3520  static_cast<Instruction::BinaryOps>(S.getAltOpcode()), LHS, RHS);
3521  } else {
3522  V0 = Builder.CreateCast(
3523  static_cast<Instruction::CastOps>(S.getOpcode()), LHS, VecTy);
3524  V1 = Builder.CreateCast(
3525  static_cast<Instruction::CastOps>(S.getAltOpcode()), LHS, VecTy);
3526  }
3527 
3528  // Create shuffle to take alternate operations from the vector.
3529  // Also, gather up main and alt scalar ops to propagate IR flags to
3530  // each vector operation.
3531  ValueList OpScalars, AltScalars;
3532  unsigned e = E->Scalars.size();
3534  for (unsigned i = 0; i < e; ++i) {
3535  auto *OpInst = cast<Instruction>(E->Scalars[i]);
3536  assert(S.isOpcodeOrAlt(OpInst) && "Unexpected main/alternate opcode");
3537  if (OpInst->getOpcode() == S.getAltOpcode()) {
3538  Mask[i] = Builder.getInt32(e + i);
3539  AltScalars.push_back(E->Scalars[i]);
3540  } else {
3541  Mask[i] = Builder.getInt32(i);
3542  OpScalars.push_back(E->Scalars[i]);
3543  }
3544  }
3545 
3546  Value *ShuffleMask = ConstantVector::get(Mask);
3547  propagateIRFlags(V0, OpScalars);
3548  propagateIRFlags(V1, AltScalars);
3549 
3550  Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3551  if (Instruction *I = dyn_cast<Instruction>(V))
3552  V = propagateMetadata(I, E->Scalars);
3553  if (NeedToShuffleReuses) {
3554  V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
3555  E->ReuseShuffleIndices, "shuffle");
3556  }
3557  E->VectorizedValue = V;
3558  ++NumVectorInstructions;
3559 
3560  return V;
3561  }
3562  default:
3563  llvm_unreachable("unknown inst");
3564  }
3565  return nullptr;
3566 }
3567 
3568 Value *BoUpSLP::vectorizeTree() {
3569  ExtraValueToDebugLocsMap ExternallyUsedValues;
3570  return vectorizeTree(ExternallyUsedValues);
3571 }
3572 
3573 Value *
3574 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3575  // All blocks must be scheduled before any instructions are inserted.
3576  for (auto &BSIter : BlocksSchedules) {
3577  scheduleBlock(BSIter.second.get());
3578  }
3579 
3580  Builder.SetInsertPoint(&F->getEntryBlock().front());
3581  auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3582 
3583  // If the vectorized tree can be rewritten in a smaller type, we truncate the
3584  // vectorized root. InstCombine will then rewrite the entire expression. We
3585  // sign extend the extracted values below.
3586  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3587  if (MinBWs.count(ScalarRoot)) {
3588  if (auto *I = dyn_cast<Instruction>(VectorRoot))
3589  Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3590  auto BundleWidth = VectorizableTree[0].Scalars.size();
3591  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3592  auto *VecTy = VectorType::get(MinTy, BundleWidth);
3593  auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3594  VectorizableTree[0].VectorizedValue = Trunc;
3595  }
3596 
3597  LLVM_DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size()
3598  << " values .\n");
3599 
3600  // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3601  // specified by ScalarType.
3602  auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3603  if (!MinBWs.count(ScalarRoot))
3604  return Ex;
3605  if (MinBWs[ScalarRoot].second)
3606  return Builder.CreateSExt(Ex, ScalarType);
3607  return Builder.CreateZExt(Ex, ScalarType);
3608  };
3609 
3610  // Extract all of the elements with the external uses.
3611  for (const auto &ExternalUse : ExternalUses) {
3612  Value *Scalar = ExternalUse.Scalar;
3613  llvm::User *User = ExternalUse.User;
3614 
3615  // Skip users that we already RAUW. This happens when one instruction
3616  // has multiple uses of the same value.
3617  if (User && !is_contained(Scalar->users(), User))
3618  continue;
3619  TreeEntry *E = getTreeEntry(Scalar);
3620  assert(E && "Invalid scalar");
3621  assert(!E->NeedToGather && "Extracting from a gather list");
3622 
3623  Value *Vec = E->VectorizedValue;
3624  assert(Vec && "Can't find vectorizable value");
3625 
3626  Value *Lane = Builder.getInt32(ExternalUse.Lane);
3627  // If User == nullptr, the Scalar is used as extra arg. Generate
3628  // ExtractElement instruction and update the record for this scalar in
3629  // ExternallyUsedValues.
3630  if (!User) {
3631  assert(ExternallyUsedValues.count(Scalar) &&
3632  "Scalar with nullptr as an external user must be registered in "
3633  "ExternallyUsedValues map");
3634  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3635  Builder.SetInsertPoint(VecI->getParent(),
3636  std::next(VecI->getIterator()));
3637  } else {
3638  Builder.SetInsertPoint(&F->getEntryBlock().front());
3639  }
3640  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3641  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3642  CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3643  auto &Locs = ExternallyUsedValues[Scalar];
3644  ExternallyUsedValues.insert({Ex, Locs});
3645  ExternallyUsedValues.erase(Scalar);
3646  // Required to update internally referenced instructions.
3647  Scalar->replaceAllUsesWith(Ex);
3648  continue;
3649  }
3650 
3651  // Generate extracts for out-of-tree users.
3652  // Find the insertion point for the extractelement lane.
3653  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3654  if (PHINode *PH = dyn_cast<PHINode>(User)) {
3655  for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3656  if (PH->getIncomingValue(i) == Scalar) {
3657  Instruction *IncomingTerminator =
3658  PH->getIncomingBlock(i)->getTerminator();
3659  if (isa<CatchSwitchInst>(IncomingTerminator)) {
3660  Builder.SetInsertPoint(VecI->getParent(),
3661  std::next(VecI->getIterator()));
3662  } else {
3663  Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3664  }
3665  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3666  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3667  CSEBlocks.insert(PH->getIncomingBlock(i));
3668  PH->setOperand(i, Ex);
3669  }
3670  }
3671  } else {
3672  Builder.SetInsertPoint(cast<Instruction>(User));
3673  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3674  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3675  CSEBlocks.insert(cast<Instruction>(User)->getParent());
3676  User->replaceUsesOfWith(Scalar, Ex);
3677  }
3678  } else {
3679  Builder.SetInsertPoint(&F->getEntryBlock().front());
3680  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3681  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3682  CSEBlocks.insert(&F->getEntryBlock());
3683  User->replaceUsesOfWith(Scalar, Ex);
3684  }
3685 
3686  LLVM_DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3687  }
3688 
3689  // For each vectorized value:
3690  for (TreeEntry &EIdx : VectorizableTree) {
3691  TreeEntry *Entry = &EIdx;
3692 
3693  // No need to handle users of gathered values.
3694  if (Entry->NeedToGather)
3695  continue;
3696 
3697  assert(Entry->VectorizedValue && "Can't find vectorizable value");
3698 
3699  // For each lane:
3700  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3701  Value *Scalar = Entry->Scalars[Lane];
3702 
3703  Type *Ty = Scalar->getType();
3704  if (!Ty->isVoidTy()) {
3705 #ifndef NDEBUG
3706  for (User *U : Scalar->users()) {
3707  LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3708 
3709  // It is legal to replace users in the ignorelist by undef.
3710  assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3711  "Replacing out-of-tree value with undef");
3712  }
3713 #endif
3714  Value *Undef = UndefValue::get(Ty);
3715  Scalar->replaceAllUsesWith(Undef);
3716  }
3717  LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3718  eraseInstruction(cast<Instruction>(Scalar));
3719  }
3720  }
3721 
3722  Builder.ClearInsertionPoint();
3723 
3724  return VectorizableTree[0].VectorizedValue;
3725 }
3726 
3727 void BoUpSLP::optimizeGatherSequence() {
3728  LLVM_DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3729  << " gather sequences instructions.\n");
3730  // LICM InsertElementInst sequences.
3731  for (Instruction *I : GatherSeq) {
3732  if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
3733  continue;
3734 
3735  // Check if this block is inside a loop.
3736  Loop *L = LI->getLoopFor(I->getParent());
3737  if (!L)
3738  continue;
3739 
3740  // Check if it has a preheader.
3741  BasicBlock *PreHeader = L->getLoopPreheader();
3742  if (!PreHeader)
3743  continue;
3744 
3745  // If the vector or the element that we insert into it are
3746  // instructions that are defined in this basic block then we can't
3747  // hoist this instruction.
3748  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
3749  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
3750  if (Op0 && L->contains(Op0))
3751  continue;
3752  if (Op1 && L->contains(Op1))
3753  continue;
3754 
3755  // We can hoist this instruction. Move it to the pre-header.
3756  I->moveBefore(PreHeader->getTerminator());
3757  }
3758 
3759  // Make a list of all reachable blocks in our CSE queue.
3761  CSEWorkList.reserve(CSEBlocks.size());
3762  for (BasicBlock *BB : CSEBlocks)
3763  if (DomTreeNode *N = DT->getNode(BB)) {
3764  assert(DT->isReachableFromEntry(N));
3765  CSEWorkList.push_back(N);
3766  }
3767 
3768  // Sort blocks by domination. This ensures we visit a block after all blocks
3769  // dominating it are visited.
3770  std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3771  [this](const DomTreeNode *A, const DomTreeNode *B) {
3772  return DT->properlyDominates(A, B);
3773  });
3774 
3775  // Perform O(N^2) search over the gather sequences and merge identical
3776  // instructions. TODO: We can further optimize this scan if we split the
3777  // instructions into different buckets based on the insert lane.
3779  for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3780  assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
3781  "Worklist not sorted properly!");
3782  BasicBlock *BB = (*I)->getBlock();
3783  // For all instructions in blocks containing gather sequences:
3784  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3785  Instruction *In = &*it++;
3786  if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3787  continue;
3788 
3789  // Check if we can replace this instruction with any of the
3790  // visited instructions.
3791  for (Instruction *v : Visited) {
3792  if (In->isIdenticalTo(v) &&
3793  DT->dominates(v->getParent(), In->getParent())) {
3794  In->replaceAllUsesWith(v);
3795  eraseInstruction(In);
3796  In = nullptr;
3797  break;
3798  }
3799  }
3800  if (In) {
3801  assert(!is_contained(Visited, In));
3802  Visited.push_back(In);
3803  }
3804  }
3805  }
3806  CSEBlocks.clear();
3807  GatherSeq.clear();
3808 }
3809 
3810 // Groups the instructions to a bundle (which is then a single scheduling entity)
3811 // and schedules instructions until the bundle gets ready.
3812 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3813  BoUpSLP *SLP,
3814  const InstructionsState &S) {
3815  if (isa<PHINode>(S.OpValue))
3816  return true;
3817 
3818  // Initialize the instruction bundle.
3819  Instruction *OldScheduleEnd = ScheduleEnd;
3820  ScheduleData *PrevInBundle = nullptr;
3821  ScheduleData *Bundle = nullptr;
3822  bool ReSchedule = false;
3823  LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.OpValue << "\n");
3824 
3825  // Make sure that the scheduling region contains all
3826  // instructions of the bundle.
3827  for (Value *V : VL) {
3828  if (!extendSchedulingRegion(V, S))
3829  return false;
3830  }
3831 
3832  for (Value *V : VL) {
3833  ScheduleData *BundleMember = getScheduleData(V);
3834  assert(BundleMember &&
3835  "no ScheduleData for bundle member (maybe not in same basic block)");
3836  if (BundleMember->IsScheduled) {
3837  // A bundle member was scheduled as single instruction before and now
3838  // needs to be scheduled as part of the bundle. We just get rid of the
3839  // existing schedule.
3840  LLVM_DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3841  << " was already scheduled\n");
3842  ReSchedule = true;
3843  }
3844  assert(BundleMember->isSchedulingEntity() &&
3845  "bundle member already part of other bundle");
3846  if (PrevInBundle) {
3847  PrevInBundle->NextInBundle = BundleMember;
3848  } else {
3849  Bundle = BundleMember;
3850  }
3851  BundleMember->UnscheduledDepsInBundle = 0;
3852  Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3853 
3854  // Group the instructions to a bundle.
3855  BundleMember->FirstInBundle = Bundle;
3856  PrevInBundle = BundleMember;
3857  }
3858  if (ScheduleEnd != OldScheduleEnd) {
3859  // The scheduling region got new instructions at the lower end (or it is a
3860  // new region for the first bundle). This makes it necessary to
3861  // recalculate all dependencies.
3862  // It is seldom that this needs to be done a second time after adding the
3863  // initial bundle to the region.
3864  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3865  doForAllOpcodes(I, [](ScheduleData *SD) {
3866  SD->clearDependencies();
3867  });
3868  }
3869  ReSchedule = true;
3870  }
3871  if (ReSchedule) {
3872  resetSchedule();
3873  initialFillReadyList(ReadyInsts);
3874  }
3875 
3876  LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3877  << BB->getName() << "\n");
3878 
3879  calculateDependencies(Bundle, true, SLP);
3880 
3881  // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3882  // means that there are no cyclic dependencies and we can schedule it.
3883  // Note that's important that we don't "schedule" the bundle yet (see
3884  // cancelScheduling).
3885  while (!Bundle->isReady() && !ReadyInsts.empty()) {
3886 
3887  ScheduleData *pickedSD = ReadyInsts.back();
3888  ReadyInsts.pop_back();
3889 
3890  if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3891  schedule(pickedSD, ReadyInsts);
3892  }
3893  }
3894  if (!Bundle->isReady()) {
3895  cancelScheduling(VL, S.OpValue);
3896  return false;
3897  }
3898  return true;
3899 }
3900 
3901 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3902  Value *OpValue) {
3903  if (isa<PHINode>(OpValue))
3904  return;
3905 
3906  ScheduleData *Bundle = getScheduleData(OpValue);
3907  LLVM_DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3908  assert(!Bundle->IsScheduled &&
3909  "Can't cancel bundle which is already scheduled");
3910  assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3911  "tried to unbundle something which is not a bundle");
3912 
3913  // Un-bundle: make single instructions out of the bundle.
3914  ScheduleData *BundleMember = Bundle;
3915  while (BundleMember) {
3916  assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3917  BundleMember->FirstInBundle = BundleMember;
3918  ScheduleData *Next = BundleMember->NextInBundle;
3919  BundleMember->NextInBundle = nullptr;
3920  BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3921  if (BundleMember->UnscheduledDepsInBundle == 0) {
3922  ReadyInsts.insert(BundleMember);
3923  }
3924  BundleMember = Next;
3925  }
3926 }
3927 
3928 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3929  // Allocate a new ScheduleData for the instruction.
3930  if (ChunkPos >= ChunkSize) {
3931  ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3932  ChunkPos = 0;
3933  }
3934  return &(ScheduleDataChunks.back()[ChunkPos++]);
3935 }
3936 
3937 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3938  const InstructionsState &S) {
3939  if (getScheduleData(V, isOneOf(S, V)))
3940  return true;
3941  Instruction *I = dyn_cast<Instruction>(V);
3942  assert(I && "bundle member must be an instruction");
3943  assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3944  auto &&CheckSheduleForI = [this, &S](Instruction *I) -> bool {
3945  ScheduleData *ISD = getScheduleData(I);
3946  if (!ISD)
3947  return false;
3948  assert(isInSchedulingRegion(ISD) &&
3949  "ScheduleData not in scheduling region");
3950  ScheduleData *SD = allocateScheduleDataChunks();
3951  SD->Inst = I;
3952  SD->init(SchedulingRegionID, S.OpValue);
3953  ExtraScheduleDataMap[I][S.OpValue] = SD;
3954  return true;
3955  };
3956  if (CheckSheduleForI(I))
3957  return true;
3958  if (!ScheduleStart) {
3959  // It's the first instruction in the new region.
3960  initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3961  ScheduleStart = I;
3962  ScheduleEnd = I->getNextNode();
3963  if (isOneOf(S, I) != I)
3964  CheckSheduleForI(I);
3965  assert(ScheduleEnd && "tried to vectorize a terminator?");
3966  LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3967  return true;
3968  }
3969  // Search up and down at the same time, because we don't know if the new
3970  // instruction is above or below the existing scheduling region.
3972  ++ScheduleStart->getIterator().getReverse();
3973  BasicBlock::reverse_iterator UpperEnd = BB->rend();
3974  BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3975  BasicBlock::iterator LowerEnd = BB->end();
3976  while (true) {
3977  if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3978  LLVM_DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3979  return false;
3980  }
3981 
3982  if (UpIter != UpperEnd) {
3983  if (&*UpIter == I) {
3984  initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3985  ScheduleStart = I;
3986  if (isOneOf(S, I) != I)
3987  CheckSheduleForI(I);
3988  LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I
3989  << "\n");
3990  return true;
3991  }
3992  UpIter++;
3993  }
3994  if (DownIter != LowerEnd) {
3995  if (&*DownIter == I) {
3996  initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3997  nullptr);
3998  ScheduleEnd = I->getNextNode();
3999  if (isOneOf(S, I) != I)
4000  CheckSheduleForI(I);
4001  assert(ScheduleEnd && "tried to vectorize a terminator?");
4002  LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I
4003  << "\n");
4004  return true;
4005  }
4006  DownIter++;
4007  }
4008  assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
4009  "instruction not found in block");
4010  }
4011  return true;
4012 }
4013 
4014 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
4015  Instruction *ToI,
4016  ScheduleData *PrevLoadStore,
4017  ScheduleData *NextLoadStore) {
4018  ScheduleData *CurrentLoadStore = PrevLoadStore;
4019  for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
4020  ScheduleData *SD = ScheduleDataMap[I];
4021  if (!SD) {
4022  SD = allocateScheduleDataChunks();
4023  ScheduleDataMap[I] = SD;
4024  SD->Inst = I;
4025  }
4026  assert(!isInSchedulingRegion(SD) &&
4027  "new ScheduleData already in scheduling region");
4028  SD->init(SchedulingRegionID, I);
4029 
4030  if (I->mayReadOrWriteMemory() &&
4031  (!isa<IntrinsicInst>(I) ||
4032  cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
4033  // Update the linked list of memory accessing instructions.
4034  if (CurrentLoadStore) {
4035  CurrentLoadStore->NextLoadStore = SD;
4036  } else {
4037  FirstLoadStoreInRegion = SD;
4038  }
4039  CurrentLoadStore = SD;
4040  }
4041  }
4042  if (NextLoadStore) {
4043  if (CurrentLoadStore)
4044  CurrentLoadStore->NextLoadStore = NextLoadStore;
4045  } else {
4046  LastLoadStoreInRegion = CurrentLoadStore;
4047  }
4048 }
4049 
4050 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
4051  bool InsertInReadyList,
4052  BoUpSLP *SLP) {
4053  assert(SD->isSchedulingEntity());
4054 
4056  WorkList.push_back(SD);
4057 
4058  while (!WorkList.empty()) {
4059  ScheduleData *SD = WorkList.back();
4060  WorkList.pop_back();
4061 
4062  ScheduleData *BundleMember = SD;
4063  while (BundleMember) {
4064  assert(isInSchedulingRegion(BundleMember));
4065  if (!BundleMember->hasValidDependencies()) {
4066 
4067  LLVM_DEBUG(dbgs() << "SLP: update deps of " << *BundleMember
4068  << "\n");
4069  BundleMember->Dependencies = 0;
4070  BundleMember->resetUnscheduledDeps();
4071 
4072  // Handle def-use chain dependencies.
4073  if (BundleMember->OpValue != BundleMember->Inst) {
4074  ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
4075  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4076  BundleMember->Dependencies++;
4077  ScheduleData *DestBundle = UseSD->FirstInBundle;
4078  if (!DestBundle->IsScheduled)
4079  BundleMember->incrementUnscheduledDeps(1);
4080  if (!DestBundle->hasValidDependencies())
4081  WorkList.push_back(DestBundle);
4082  }
4083  } else {
4084  for (User *U : BundleMember->Inst->users()) {
4085  if (isa<Instruction>(U)) {
4086  ScheduleData *UseSD = getScheduleData(U);
4087  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
4088  BundleMember->Dependencies++;
4089  ScheduleData *DestBundle = UseSD->FirstInBundle;
4090  if (!DestBundle->IsScheduled)
4091  BundleMember->incrementUnscheduledDeps(1);
4092  if (!DestBundle->hasValidDependencies())
4093  WorkList.push_back(DestBundle);
4094  }
4095  } else {
4096  // I'm not sure if this can ever happen. But we need to be safe.
4097  // This lets the instruction/bundle never be scheduled and
4098  // eventually disable vectorization.
4099  BundleMember->Dependencies++;
4100  BundleMember->incrementUnscheduledDeps(1);
4101  }
4102  }
4103  }
4104 
4105  // Handle the memory dependencies.
4106  ScheduleData *DepDest = BundleMember->NextLoadStore;
4107  if (DepDest) {
4108  Instruction *SrcInst = BundleMember->Inst;
4109  MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
4110  bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
4111  unsigned numAliased = 0;
4112  unsigned DistToSrc = 1;
4113 
4114  while (DepDest) {
4115  assert(isInSchedulingRegion(DepDest));
4116 
4117  // We have two limits to reduce the complexity:
4118  // 1) AliasedCheckLimit: It's a small limit to reduce calls to
4119  // SLP->isAliased (which is the expensive part in this loop).
4120  // 2) MaxMemDepDistance: It's for very large blocks and it aborts
4121  // the whole loop (even if the loop is fast, it's quadratic).
4122  // It's important for the loop break condition (see below) to
4123  // check this limit even between two read-only instructions.
4124  if (DistToSrc >= MaxMemDepDistance ||
4125  ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
4126  (numAliased >= AliasedCheckLimit ||
4127  SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
4128 
4129  // We increment the counter only if the locations are aliased
4130  // (instead of counting all alias checks). This gives a better
4131  // balance between reduced runtime and accurate dependencies.
4132  numAliased++;
4133 
4134  DepDest->MemoryDependencies.push_back(BundleMember);
4135  BundleMember->Dependencies++;
4136  ScheduleData *DestBundle = DepDest->FirstInBundle;
4137  if (!DestBundle->IsScheduled) {
4138  BundleMember->incrementUnscheduledDeps(1);
4139  }
4140  if (!DestBundle->hasValidDependencies()) {
4141  WorkList.push_back(DestBundle);
4142  }
4143  }
4144  DepDest = DepDest->NextLoadStore;
4145 
4146  // Example, explaining the loop break condition: Let's assume our
4147  // starting instruction is i0 and MaxMemDepDistance = 3.
4148  //
4149  // +--------v--v--v
4150  // i0,i1,i2,i3,i4,i5,i6,i7,i8
4151  // +--------^--^--^
4152  //
4153  // MaxMemDepDistance let us stop alias-checking at i3 and we add
4154  // dependencies from i0 to i3,i4,.. (even if they are not aliased).
4155  // Previously we already added dependencies from i3 to i6,i7,i8
4156  // (because of MaxMemDepDistance). As we added a dependency from
4157  // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
4158  // and we can abort this loop at i6.
4159  if (DistToSrc >= 2 * MaxMemDepDistance)
4160  break;
4161  DistToSrc++;
4162  }
4163  }
4164  }
4165  BundleMember = BundleMember->NextInBundle;
4166  }
4167  if (InsertInReadyList && SD->isReady()) {
4168  ReadyInsts.push_back(SD);
4169  LLVM_DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst
4170  << "\n");
4171  }
4172  }
4173 }
4174 
4175 void BoUpSLP::BlockScheduling::resetSchedule() {
4176  assert(ScheduleStart &&
4177  "tried to reset schedule on block which has not been scheduled");
4178  for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
4179  doForAllOpcodes(I, [&](ScheduleData *SD) {
4180  assert(isInSchedulingRegion(SD) &&
4181  "ScheduleData not in scheduling region");
4182  SD->IsScheduled = false;
4183  SD->resetUnscheduledDeps();
4184  });
4185  }
4186  ReadyInsts.clear();
4187 }
4188 
4189 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
4190  if (!BS->ScheduleStart)
4191  return;
4192 
4193  LLVM_DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
4194 
4195  BS->resetSchedule();
4196 
4197  // For the real scheduling we use a more sophisticated ready-list: it is
4198  // sorted by the original instruction location. This lets the final schedule
4199  // be as close as possible to the original instruction order.
4200  struct ScheduleDataCompare {
4201  bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
4202  return SD2->SchedulingPriority < SD1->SchedulingPriority;
4203  }
4204  };
4205  std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
4206 
4207  // Ensure that all dependency data is updated and fill the ready-list with
4208  // initial instructions.
4209  int Idx = 0;
4210  int NumToSchedule = 0;
4211  for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
4212  I = I->getNextNode()) {
4213  BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
4214  assert(SD->isPartOfBundle() ==
4215  (getTreeEntry(SD->Inst) != nullptr) &&
4216  "scheduler and vectorizer bundle mismatch");
4217  SD->FirstInBundle->SchedulingPriority = Idx++;
4218  if (SD->isSchedulingEntity()) {
4219  BS->calculateDependencies(SD, false, this);
4220  NumToSchedule++;
4221  }
4222  });
4223  }
4224  BS->initialFillReadyList(ReadyInsts);
4225 
4226  Instruction *LastScheduledInst = BS->ScheduleEnd;
4227 
4228  // Do the "real" scheduling.
4229  while (!ReadyInsts.empty()) {
4230  ScheduleData *picked = *ReadyInsts.begin();
4231  ReadyInsts.erase(ReadyInsts.begin());
4232 
4233  // Move the scheduled instruction(s) to their dedicated places, if not
4234  // there yet.
4235  ScheduleData *BundleMember = picked;
4236  while (BundleMember) {
4237  Instruction *pickedInst = BundleMember->Inst;
4238  if (LastScheduledInst->getNextNode() != pickedInst) {
4239  BS->BB->getInstList().remove(pickedInst);
4240  BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
4241  pickedInst);
4242  }
4243  LastScheduledInst = pickedInst;
4244  BundleMember = BundleMember->NextInBundle;
4245  }
4246 
4247  BS->schedule(picked, ReadyInsts);
4248  NumToSchedule--;
4249  }
4250  assert(NumToSchedule == 0 && "could not schedule all instructions");
4251 
4252  // Avoid duplicate scheduling of the block.
4253  BS->ScheduleStart = nullptr;
4254 }
4255 
4256 unsigned BoUpSLP::getVectorElementSize(Value *V) {
4257  // If V is a store, just return the width of the stored value without
4258  // traversing the expression tree. This is the common case.
4259  if (auto *Store = dyn_cast<StoreInst>(V))
4260  return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
4261 
4262  // If V is not a store, we can traverse the expression tree to find loads
4263  // that feed it. The type of the loaded value may indicate a more suitable
4264  // width than V's type. We want to base the vector element size on the width
4265  // of memory operations where possible.
4268  if (auto *I = dyn_cast<Instruction>(V))
4269  Worklist.push_back(I);
4270 
4271  // Traverse the expression tree in bottom-up order looking for loads. If we
4272  // encounter an instruction we don't yet handle, we give up.
4273  auto MaxWidth = 0u;
4274  auto FoundUnknownInst = false;
4275  while (!Worklist.empty() && !FoundUnknownInst) {
4276  auto *I = Worklist.pop_back_val();
4277  Visited.insert(I);
4278 
4279  // We should only be looking at scalar instructions here. If the current
4280  // instruction has a vector type, give up.
4281  auto *Ty = I->getType();
4282  if (isa<VectorType>(Ty))
4283  FoundUnknownInst = true;
4284 
4285  // If the current instruction is a load, update MaxWidth to reflect the
4286  // width of the loaded value.
4287  else if (isa<LoadInst>(I))
4288  MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
4289 
4290  // Otherwise, we need to visit the operands of the instruction. We only
4291  // handle the interesting cases from buildTree here. If an operand is an
4292  // instruction we haven't yet visited, we add it to the worklist.
4293  else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
4294  isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
4295  for (Use &U : I->operands())
4296  if (auto *J = dyn_cast<Instruction>(U.get()))
4297  if (!Visited.count(J))
4298  Worklist.push_back(J);
4299  }
4300 
4301  // If we don't yet handle the instruction, give up.
4302  else
4303  FoundUnknownInst = true;
4304  }
4305 
4306  // If we didn't encounter a memory access in the expression tree, or if we
4307  // gave up for some reason, just return the width of V.
4308  if (!MaxWidth || FoundUnknownInst)
4309  return DL->getTypeSizeInBits(V->getType());
4310 
4311  // Otherwise, return the maximum width we found.
4312  return MaxWidth;
4313 }
4314 
4315 // Determine if a value V in a vectorizable expression Expr can be demoted to a
4316 // smaller type with a truncation. We collect the values that will be demoted
4317 // in ToDemote and additional roots that require investigating in Roots.
4319  SmallVectorImpl<Value *> &ToDemote,
4320  SmallVectorImpl<Value *> &Roots) {
4321  // We can always demote constants.
4322  if (isa<Constant>(V)) {
4323  ToDemote.push_back(V);
4324  return true;
4325  }
4326 
4327  // If the value is not an instruction in the expression with only one use, it
4328  // cannot be demoted.
4329  auto *I = dyn_cast<Instruction>(V);
4330  if (!I || !I->hasOneUse() || !Expr.count(I))
4331  return false;
4332 
4333  switch (I->getOpcode()) {
4334 
4335  // We can always demote truncations and extensions. Since truncations can
4336  // seed additional demotion, we save the truncated value.
4337  case Instruction::Trunc:
4338  Roots.push_back(I->getOperand(0));
4339  break;
4340  case Instruction::ZExt:
4341  case Instruction::SExt:
4342  break;
4343 
4344  // We can demote certain binary operations if we can demote both of their
4345  // operands.
4346  case Instruction::Add:
4347  case Instruction::Sub:
4348  case Instruction::Mul:
4349  case Instruction::And:
4350  case Instruction::Or:
4351  case Instruction::Xor:
4352  if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
4353  !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
4354  return false;
4355  break;
4356 
4357  // We can demote selects if we can demote their true and false values.
4358  case Instruction::Select: {
4359  SelectInst *SI = cast<SelectInst>(I);
4360  if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
4361  !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
4362  return false;
4363  break;
4364  }
4365 
4366  // We can demote phis if we can demote all their incoming operands. Note that
4367  // we don't need to worry about cycles since we ensure single use above.
4368  case Instruction::PHI: {
4369  PHINode *PN = cast<PHINode>(I);
4370  for (Value *IncValue : PN->incoming_values())
4371  if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
4372  return false;
4373  break;
4374  }
4375 
4376  // Otherwise, conservatively give up.
4377  default:
4378  return false;
4379  }
4380 
4381  // Record the value that we can demote.
4382  ToDemote.push_back(V);
4383  return true;
4384 }
4385 
4387  // If there are no external uses, the expression tree must be rooted by a
4388  // store. We can't demote in-memory values, so there is nothing to do here.
4389  if (ExternalUses.empty())
4390  return;
4391 
4392  // We only attempt to truncate integer expressions.
4393  auto &TreeRoot = VectorizableTree[0].Scalars;
4394  auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
4395  if (!TreeRootIT)
4396  return;
4397 
4398  // If the expression is not rooted by a store, these roots should have
4399  // external uses. We will rely on InstCombine to rewrite the expression in
4400  // the narrower type. However, InstCombine only rewrites single-use values.
4401  // This means that if a tree entry other than a root is used externally, it
4402  // must have multiple uses and InstCombine will not rewrite it. The code
4403  // below ensures that only the roots are used externally.
4404  SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
4405  for (auto &EU : ExternalUses)
4406  if (!Expr.erase(EU.Scalar))
4407  return;
4408  if (!Expr.empty())
4409  return;
4410 
4411  // Collect the scalar values of the vectorizable expression. We will use this
4412  // context to determine which values can be demoted. If we see a truncation,
4413  // we mark it as seeding another demotion.
4414  for (auto &Entry : VectorizableTree)
4415  Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4416 
4417  // Ensure the roots of the vectorizable tree don't form a cycle. They must
4418  // have a single external user that is not in the vectorizable tree.
4419  for (auto *Root : TreeRoot)
4420  if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4421  return;
4422 
4423  // Conservatively determine if we can actually truncate the roots of the
4424  // expression. Collect the values that can be demoted in ToDemote and
4425  // additional roots that require investigating in Roots.
4426  SmallVector<Value *, 32> ToDemote;
4428  for (auto *Root : TreeRoot)
4429  if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4430  return;
4431 
4432  // The maximum bit width required to represent all the values that can be
4433  // demoted without loss of precision. It would be safe to truncate the roots
4434  // of the expression to this width.
4435  auto MaxBitWidth = 8u;
4436 
4437  // We first check if all the bits of the roots are demanded. If they're not,
4438  // we can truncate the roots to this narrower type.
4439  for (auto *Root : TreeRoot) {
4440  auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4441  MaxBitWidth = std::max<unsigned>(
4442  Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4443  }
4444 
4445  // True if the roots can be zero-extended back to their original type, rather
4446  // than sign-extended. We know that if the leading bits are not demanded, we
4447  // can safely zero-extend. So we initialize IsKnownPositive to True.
4448  bool IsKnownPositive = true;
4449 
4450  // If all the bits of the roots are demanded, we can try a little harder to
4451  // compute a narrower type. This can happen, for example, if the roots are
4452  // getelementptr indices. InstCombine promotes these indices to the pointer
4453  // width. Thus, all their bits are technically demanded even though the
4454  // address computation might be vectorized in a smaller type.
4455  //
4456  // We start by looking at each entry that can be demoted. We compute the
4457  // maximum bit width required to store the scalar by using ValueTracking to
4458  // compute the number of high-order bits we can truncate.
4459  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType()) &&
4460  llvm::all_of(TreeRoot, [](Value *R) {
4461  assert(R->hasOneUse() && "Root should have only one use!");
4462  return isa<GetElementPtrInst>(R->user_back());
4463  })) {
4464  MaxBitWidth = 8u;
4465 
4466  // Determine if the sign bit of all the roots is known to be zero. If not,
4467  // IsKnownPositive is set to False.
4468  IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4469  KnownBits Known = computeKnownBits(R, *DL);
4470  return Known.isNonNegative();
4471  });
4472 
4473  // Determine the maximum number of bits required to store the scalar
4474  // values.
4475  for (auto *Scalar : ToDemote) {
4476  auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4477  auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4478  MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4479  }
4480 
4481  // If we can't prove that the sign bit is zero, we must add one to the
4482  // maximum bit width to account for the unknown sign bit. This preserves
4483  // the existing sign bit so we can safely sign-extend the root back to the
4484  // original type. Otherwise, if we know the sign bit is zero, we will
4485  // zero-extend the root instead.
4486  //
4487  // FIXME: This is somewhat suboptimal, as there will be cases where adding
4488  // one to the maximum bit width will yield a larger-than-necessary
4489  // type. In general, we need to add an extra bit only if we can't
4490  // prove that the upper bit of the original type is equal to the
4491  // upper bit of the proposed smaller type. If these two bits are the
4492  // same (either zero or one) we know that sign-extending from the
4493  // smaller type will result in the same value. Here, since we can't
4494  // yet prove this, we are just making the proposed smaller type
4495  // larger to ensure correctness.
4496  if (!IsKnownPositive)
4497  ++MaxBitWidth;
4498  }
4499 
4500  // Round MaxBitWidth up to the next power-of-two.
4501  if (!isPowerOf2_64(MaxBitWidth))
4502  MaxBitWidth = NextPowerOf2(MaxBitWidth);
4503 
4504  // If the maximum bit width we compute is less than the with of the roots'
4505  // type, we can proceed with the narrowing. Otherwise, do nothing.
4506  if (MaxBitWidth >= TreeRootIT->getBitWidth())
4507  return;
4508 
4509  // If we can truncate the root, we must collect additional values that might
4510  // be demoted as a result. That is, those seeded by truncations we will
4511  // modify.
4512  while (!Roots.empty())
4513  collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4514 
4515  // Finally, map the values we can demote to the maximum bit with we computed.
4516  for (auto *Scalar : ToDemote)
4517  MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4518 }
4519 
4520 namespace {
4521 
4522 /// The SLPVectorizer Pass.
4523 struct SLPVectorizer : public FunctionPass {
4524  SLPVectorizerPass Impl;
4525 
4526  /// Pass identification, replacement for typeid
4527  static char ID;
4528 
4529  explicit SLPVectorizer() : FunctionPass(ID) {
4531  }
4532 
4533  bool doInitialization(Module &M) override {
4534  return false;
4535  }
4536 
4537  bool runOnFunction(Function &F) override {
4538  if (skipFunction(F))
4539  return false;
4540 
4541  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4542  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4543  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4544  auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4545  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4546  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4547  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4548  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4549  auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4550  auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4551 
4552  return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4553  }
4554 
4555  void getAnalysisUsage(AnalysisUsage &AU) const override {
4569  AU.setPreservesCFG();
4570  }
4571 };
4572 
4573 } // end anonymous namespace
4574 
4576  auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4577  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4578  auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4579  auto *AA = &AM.getResult<AAManager>(F);
4580  auto *LI = &AM.getResult<LoopAnalysis>(F);
4581  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4582  auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4583  auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4584  auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4585 
4586  bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4587  if (!Changed)
4588  return PreservedAnalyses::all();
4589 
4590  PreservedAnalyses PA;
4591  PA.preserveSet<CFGAnalyses>();
4592  PA.preserve<AAManager>();
4593  PA.preserve<GlobalsAA>();
4594  return PA;
4595 }
4596 
4598  TargetTransformInfo *TTI_,
4599  TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4600  LoopInfo *LI_, DominatorTree *DT_,
4601  AssumptionCache *AC_, DemandedBits *DB_,
4602  OptimizationRemarkEmitter *ORE_) {
4603  SE = SE_;
4604  TTI = TTI_;
4605  TLI = TLI_;
4606  AA = AA_;
4607  LI = LI_;
4608  DT = DT_;
4609  AC = AC_;
4610  DB = DB_;
4611  DL = &F.getParent()->getDataLayout();
4612 
4613  Stores.clear();
4614  GEPs.clear();
4615  bool Changed = false;
4616 
4617  // If the target claims to have no vector registers don't attempt
4618  // vectorization.
4619  if (!TTI->getNumberOfRegisters(true))
4620  return false;
4621 
4622  // Don't vectorize when the attribute NoImplicitFloat is used.
4623  if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4624  return false;
4625 
4626  LLVM_DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4627 
4628  // Use the bottom up slp vectorizer to construct chains that start with
4629  // store instructions.
4630  BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4631 
4632  // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4633  // delete instructions.
4634 
4635  // Scan the blocks in the function in post order.
4636  for (auto BB : post_order(&F.getEntryBlock())) {
4637  collectSeedInstructions(BB);
4638 
4639  // Vectorize trees that end at stores.
4640  if (!Stores.empty()) {
4641  LLVM_DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4642  << " underlying objects.\n");
4643  Changed |= vectorizeStoreChains(R);
4644  }
4645 
4646  // Vectorize trees that end at reductions.
4647  Changed |= vectorizeChainsInBlock(BB, R);
4648 
4649  // Vectorize the index computations of getelementptr instructions. This
4650  // is primarily intended to catch gather-like idioms ending at
4651  // non-consecutive loads.
4652  if (!GEPs.empty()) {
4653  LLVM_DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4654  << " underlying objects.\n");
4655  Changed |= vectorizeGEPIndices(BB, R);
4656  }
4657  }
4658 
4659  if (Changed) {
4660  R.optimizeGatherSequence();
4661  LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4663  }
4664  return Changed;
4665 }
4666 
4667 /// Check that the Values in the slice in VL array are still existent in
4668 /// the WeakTrackingVH array.
4669 /// Vectorization of part of the VL array may cause later values in the VL array
4670 /// to become invalid. We track when this has happened in the WeakTrackingVH
4671 /// array.
4673  ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4674  unsigned SliceSize) {
4675  VL = VL.slice(SliceBegin, SliceSize);
4676  VH = VH.slice(SliceBegin, SliceSize);
4677  return !std::equal(VL.begin(), VL.end(), VH.begin());
4678 }
4679 
4680 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4681  unsigned VecRegSize) {
4682  const unsigned ChainLen = Chain.size();
4683  LLVM_DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4684  << "\n");
4685  const unsigned Sz = R.getVectorElementSize(Chain[0]);
4686  const unsigned VF = VecRegSize / Sz;
4687 
4688  if (!isPowerOf2_32(Sz) || VF < 2)
4689  return false;
4690 
4691  // Keep track of values that were deleted by vectorizing in the loop below.
4692  const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4693 
4694  bool Changed = false;
4695  // Look for profitable vectorizable trees at all offsets, starting at zero.
4696  for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
4697 
4698  // Check that a previous iteration of this loop did not delete the Value.
4699  if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4700  continue;
4701 
4702  LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4703  << "\n");
4704  ArrayRef<Value *> Operands = Chain.slice(i, VF);
4705 
4706  R.buildTree(Operands);
4707  if (R.isTreeTinyAndNotFullyVectorizable())
4708  continue;
4709 
4710  R.computeMinimumValueSizes();
4711 
4712  int Cost = R.getTreeCost();
4713 
4714  LLVM_DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF
4715  << "\n");
4716  if (Cost < -SLPCostThreshold) {
4717  LLVM_DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4718 
4719  using namespace ore;
4720 
4721  R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4722  cast<StoreInst>(Chain[i]))
4723  << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4724  << " and with tree size "
4725  << NV("TreeSize", R.getTreeSize()));
4726 
4727  R.vectorizeTree();
4728 
4729  // Move to the next bundle.
4730  i += VF - 1;
4731  Changed = true;
4732  }
4733  }
4734 
4735  return Changed;
4736 }
4737 
4738 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4739  BoUpSLP &R) {
4740  SetVector<StoreInst *> Heads;
4742  SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4743 
4744  // We may run into multiple chains that merge into a single chain. We mark the
4745  // stores that we vectorized so that we don't visit the same store twice.
4746  BoUpSLP::ValueSet VectorizedStores;
4747  bool Changed = false;
4748 
4749  // Do a quadratic search on all of the given stores in reverse order and find
4750  // all of the pairs of stores that follow each other.
4751  SmallVector<unsigned, 16> IndexQueue;
4752  unsigned E = Stores.size();
4753  IndexQueue.resize(E - 1);
4754  for (unsigned I = E; I > 0; --I) {
4755  unsigned Idx = I - 1;
4756  // If a store has multiple consecutive store candidates, search Stores
4757  // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4758  // This is because usually pairing with immediate succeeding or preceding
4759  // candidate create the best chance to find slp vectorization opportunity.
4760  unsigned Offset = 1;
4761  unsigned Cnt = 0;
4762  for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4763  if (Idx >= Offset) {
4764  IndexQueue[Cnt] = Idx - Offset;
4765  ++Cnt;
4766  }
4767  if (Idx + Offset < E) {
4768  IndexQueue[Cnt] = Idx + Offset;
4769  ++Cnt;
4770  }
4771  }
4772 
4773  for (auto K : IndexQueue) {
4774  if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4775  Tails.insert(Stores[Idx]);
4776  Heads.insert(Stores[K]);
4777  ConsecutiveChain[Stores[K]] = Stores[Idx];
4778  break;
4779  }
4780  }
4781  }
4782 
4783  // For stores that start but don't end a link in the chain:
4784  for (auto *SI : llvm::reverse(Heads)) {
4785  if (Tails.count(SI))
4786  continue;
4787 
4788  // We found a store instr that starts a chain. Now follow the chain and try
4789  // to vectorize it.
4790  BoUpSLP::ValueList Operands;
4791  StoreInst *I = SI;
4792  // Collect the chain into a list.
4793  while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4794  Operands.push_back(I);
4795  // Move to the next value in the chain.
4796  I = ConsecutiveChain[I];
4797  }
4798 
4799  // FIXME: Is division-by-2 the correct step? Should we assert that the
4800  // register size is a power-of-2?
4801  for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4802  Size /= 2) {
4803  if (vectorizeStoreChain(Operands, R, Size)) {
4804  // Mark the vectorized stores so that we don't vectorize them again.
4805  VectorizedStores.insert(Operands.begin(), Operands.end());
4806  Changed = true;
4807  break;
4808  }
4809  }
4810  }
4811 
4812  return Changed;
4813 }
4814 
4815 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4816  // Initialize the collections. We will make a single pass over the block.
4817  Stores.clear();
4818  GEPs.clear();
4819 
4820  // Visit the store and getelementptr instructions in BB and organize them in
4821  // Stores and GEPs according to the underlying objects of their pointer
4822  // operands.
4823  for (Instruction &I : *BB) {
4824  // Ignore store instructions that are volatile or have a pointer operand
4825  // that doesn't point to a scalar type.
4826  if (auto *SI = dyn_cast<StoreInst>(&I)) {
4827  if (!SI->isSimple())
4828  continue;
4829  if (!isValidElementType(SI->getValueOperand()->getType()))
4830  continue;
4831  Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4832  }
4833 
4834  // Ignore getelementptr instructions that have more than one index, a
4835  // constant index, or a pointer operand that doesn't point to a scalar
4836  // type.
4837  else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4838  auto Idx = GEP->idx_begin()->get();
4839  if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4840  continue;
4841  if (!isValidElementType(Idx->getType()))
4842  continue;
4843  if (GEP->getType()->isVectorTy())
4844  continue;
4845  GEPs[GEP->getPointerOperand()].push_back(GEP);
4846  }
4847  }
4848 }
4849 
4850 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4851  if (!A || !B)
4852  return false;
4853  Value *VL[] = { A, B };
4854  return tryToVectorizeList(VL, R, /*UserCost=*/0, true);
4855 }
4856 
4857 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4858  int UserCost, bool AllowReorder) {
4859  if (VL.size() < 2)
4860  return false;
4861 
4862  LLVM_DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = "
4863  << VL.size() << ".\n");
4864 
4865  // Check that all of the parts are scalar instructions of the same type,
4866  // we permit an alternate opcode via InstructionsState.
4867  InstructionsState S = getSameOpcode(VL);
4868  if (!S.getOpcode())
4869  return false;
4870 
4871  Instruction *I0 = cast<Instruction>(S.OpValue);
4872  unsigned Sz = R.getVectorElementSize(I0);
4873  unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4874  unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4875  if (MaxVF < 2) {
4876  R.getORE()->emit([&]() {
4877  return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
4878  << "Cannot SLP vectorize list: vectorization factor "
4879  << "less than 2 is not supported";
4880  });
4881  return false;
4882  }
4883 
4884  for (Value *V : VL) {
4885  Type *Ty = V->getType();
4886  if (!isValidElementType(Ty)) {
4887  // NOTE: the following will give user internal llvm type name, which may
4888  // not be useful.
4889  R.getORE()->emit([&]() {
4890  std::string type_str;
4891  llvm::raw_string_ostream rso(type_str);
4892  Ty->print(rso);
4893  return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0)
4894  << "Cannot SLP vectorize list: type "
4895  << rso.str() + " is unsupported by vectorizer";
4896  });
4897  return false;
4898  }
4899  }
4900 
4901  bool Changed = false;
4902  bool CandidateFound = false;
4903  int MinCost = SLPCostThreshold;
4904 
4905  // Keep track of values that were deleted by vectorizing in the loop below.
4906  SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4907 
4908  unsigned NextInst = 0, MaxInst = VL.size();
4909  for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4910  VF /= 2) {
4911  // No actual vectorization should happen, if number of parts is the same as
4912  // provided vectorization factor (i.e. the scalar type is used for vector
4913  // code during codegen).
4914  auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4915  if (TTI->getNumberOfParts(VecTy) == VF)
4916  continue;
4917  for (unsigned I = NextInst; I < MaxInst; ++I) {
4918  unsigned OpsWidth = 0;
4919 
4920  if (I + VF > MaxInst)
4921  OpsWidth = MaxInst - I;
4922  else
4923  OpsWidth = VF;
4924 
4925  if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4926  break;
4927 
4928  // Check that a previous iteration of this loop did not delete the Value.
4929  if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4930  continue;
4931 
4932  LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4933  << "\n");
4934  ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4935 
4936  R.buildTree(Ops);
4937  Optional<ArrayRef<unsigned>> Order = R.bestOrder();
4938  // TODO: check if we can allow reordering for more cases.
4939  if (AllowReorder && Order) {
4940  // TODO: reorder tree nodes without tree rebuilding.
4941  // Conceptually, there is nothing actually preventing us from trying to
4942  // reorder a larger list. In fact, we do exactly this when vectorizing
4943  // reductions. However, at this point, we only expect to get here when
4944  // there are exactly two operations.
4945  assert(Ops.size() == 2);
4946  Value *ReorderedOps[] = {Ops[1], Ops[0]};
4947  R.buildTree(ReorderedOps, None);
4948  }
4949  if (R.isTreeTinyAndNotFullyVectorizable())
4950  continue;
4951 
4952  R.computeMinimumValueSizes();
4953  int Cost = R.getTreeCost() - UserCost;
4954  CandidateFound = true;
4955  MinCost = std::min(MinCost, Cost);
4956 
4957  if (Cost < -SLPCostThreshold) {
4958  LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4959  R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4960  cast<Instruction>(Ops[0]))
4961  << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4962  << " and with tree size "
4963  << ore::NV("TreeSize", R.getTreeSize()));
4964 
4965  R.vectorizeTree();
4966  // Move to the next bundle.
4967  I += VF - 1;
4968  NextInst = I + 1;
4969  Changed = true;
4970  }
4971  }
4972  }
4973 
4974  if (!Changed && CandidateFound) {
4975  R.getORE()->emit([&]() {
4976  return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0)
4977  << "List vectorization was possible but not beneficial with cost "
4978  << ore::NV("Cost", MinCost) << " >= "
4979  << ore::NV("Treshold", -SLPCostThreshold);
4980  });
4981  } else if (!Changed) {
4982  R.getORE()->emit([&]() {
4983  return OptimizationRemarkMissed(SV_NAME, "NotPossible", I0)
4984  << "Cannot SLP vectorize list: vectorization was impossible"
4985  << " with available vectorization factors";
4986  });
4987  }
4988  return Changed;
4989 }
4990 
4991 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4992  if (!I)
4993  return false;
4994 
4995  if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4996  return false;
4997 
4998  Value *P = I->getParent();
4999 
5000  // Vectorize in current basic block only.
5001  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
5002  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
5003  if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
5004  return false;
5005 
5006  // Try to vectorize V.
5007  if (tryToVectorizePair(Op0, Op1, R))
5008  return true;
5009 
5010  auto *A = dyn_cast<BinaryOperator>(Op0);
5011  auto *B = dyn_cast<BinaryOperator>(Op1);
5012  // Try to skip B.
5013  if (B && B->hasOneUse()) {
5014  auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
5015  auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
5016  if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
5017  return true;
5018  if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
5019  return true;
5020  }
5021 
5022  // Try to skip A.
5023  if (A && A->hasOneUse()) {
5024  auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
5025  auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
5026  if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
5027  return true;
5028  if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
5029  return true;
5030  }
5031  return false;
5032 }
5033 
5034 /// Generate a shuffle mask to be used in a reduction tree.
5035 ///
5036 /// \param VecLen The length of the vector to be reduced.
5037 /// \param NumEltsToRdx The number of elements that should be reduced in the
5038 /// vector.
5039 /// \param IsPairwise Whether the reduction is a pairwise or splitting
5040 /// reduction. A pairwise reduction will generate a mask of
5041 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
5042 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
5043 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
5044 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
5045  bool IsPairwise, bool IsLeft,
5046  IRBuilder<> &Builder) {
5047  assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
5048 
5049  SmallVector<Constant *, 32> ShuffleMask(
5050  VecLen, UndefValue::get(Builder.getInt32Ty()));
5051 
5052  if (IsPairwise)
5053  // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
5054  for (unsigned i = 0; i != NumEltsToRdx; ++i)
5055  ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
5056  else
5057  // Move the upper half of the vector to the lower half.
5058  for (unsigned i = 0; i != NumEltsToRdx; ++i)
5059  ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
5060 
5061  return ConstantVector::get(ShuffleMask);
5062 }
5063 
5064 namespace {
5065 
5066 /// Model horizontal reductions.
5067 ///
5068 /// A horizontal reduction is a tree of reduction operations (currently add and
5069 /// fadd) that has operations that can be put into a vector as its leaf.
5070 /// For example, this tree:
5071 ///
5072 /// mul mul mul mul
5073 /// \ / \ /
5074 /// + +
5075 /// \ /
5076 /// +
5077 /// This tree has "mul" as its reduced values and "+" as its reduction
5078 /// operations. A reduction might be feeding into a store or a binary operation
5079 /// feeding a phi.
5080 /// ...
5081 /// \ /
5082 /// +
5083 /// |
5084 /// phi +=
5085 ///
5086 /// Or:
5087 /// ...
5088 /// \ /
5089 /// +
5090 /// |
5091 /// *p =
5092 ///
5093 class HorizontalReduction {
5094  using ReductionOpsType = SmallVector<Value *, 16>;
5095  using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
5096  ReductionOpsListType ReductionOps;
5097  SmallVector<Value *, 32> ReducedVals;
5098  // Use map vector to make stable output.
5100 
5101  /// Kind of the reduction data.
5102  enum ReductionKind {
5103  RK_None, /// Not a reduction.
5104  RK_Arithmetic, /// Binary reduction data.
5105  RK_Min, /// Minimum reduction data.
5106  RK_UMin, /// Unsigned minimum reduction data.
5107  RK_Max, /// Maximum reduction data.
5108  RK_UMax, /// Unsigned maximum reduction data.
5109  };
5110 
5111  /// Contains info about operation, like its opcode, left and right operands.
5112  class OperationData {
5113  /// Opcode of the instruction.
5114  unsigned Opcode = 0;
5115 
5116  /// Left operand of the reduction operation.
5117  Value *LHS = nullptr;
5118 
5119  /// Right operand of the reduction operation.
5120  Value *RHS = nullptr;
5121 
5122  /// Kind of the reduction operation.
5123  ReductionKind Kind = RK_None;
5124 
5125  /// True if float point min/max reduction has no NaNs.
5126  bool NoNaN = false;
5127 
5128  /// Checks if the reduction operation can be vectorized.
5129  bool isVectorizable() const {
5130  return LHS && RHS &&
5131  // We currently only support add/mul/logical && min/max reductions.
5132  ((Kind == RK_Arithmetic &&
5133  (Opcode == Instruction::Add || Opcode == Instruction::FAdd ||
5134  Opcode == Instruction::Mul || Opcode == Instruction::FMul ||
5135  Opcode == Instruction::And || Opcode == Instruction::Or ||
5136  Opcode == Instruction::Xor)) ||
5137  ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
5138  (Kind == RK_Min || Kind == RK_Max)) ||
5139  (Opcode == Instruction::ICmp &&
5140  (Kind == RK_UMin || Kind == RK_UMax)));
5141  }
5142 
5143  /// Creates reduction operation with the current opcode.
5144  Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
5145  assert(isVectorizable() &&
5146  "Expected add|fadd or min/max reduction operation.");
5147  Value *Cmp;
5148  switch (Kind) {
5149  case RK_Arithmetic:
5150  return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
5151  Name);
5152  case RK_Min:
5153  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
5154  : Builder.CreateFCmpOLT(LHS, RHS);
5155  break;
5156  case RK_Max:
5157  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
5158  : Builder.CreateFCmpOGT(LHS, RHS);
5159  break;
5160  case RK_UMin:
5161  assert(Opcode == Instruction::ICmp && "Expected integer types.");
5162  Cmp = Builder.CreateICmpULT(LHS, RHS);
5163  break;
5164  case RK_UMax:
5165  assert(Opcode == Instruction::ICmp && "Expected integer types.");
5166  Cmp = Builder.CreateICmpUGT(LHS, RHS);
5167  break;
5168  case RK_None:
5169  llvm_unreachable("Unknown reduction operation.");
5170  }
5171  return Builder.CreateSelect(Cmp, LHS, RHS, Name);
5172  }
5173 
5174  public:
5175  explicit OperationData() = default;
5176 
5177  /// Construction for reduced values. They are identified by opcode only and
5178  /// don't have associated LHS/RHS values.
5179  explicit OperationData(Value *V) {
5180  if (auto *I = dyn_cast<Instruction>(V))
5181  Opcode = I->getOpcode();
5182  }
5183 
5184  /// Constructor for reduction operations with opcode and its left and
5185  /// right operands.
5186  OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
5187  bool NoNaN = false)
5188  : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
5189  assert(Kind != RK_None && "One of the reduction operations is expected.");
5190  }
5191 
5192  explicit operator bool() const { return Opcode; }
5193 
5194  /// Get the index of the first operand.
5195  unsigned getFirstOperandIndex() const {
5196  assert(!!*this && "The opcode is not set.");
5197  switch (Kind) {
5198  case RK_Min:
5199  case RK_UMin:
5200  case RK_Max:
5201  case RK_UMax:
5202  return 1;
5203  case RK_Arithmetic:
5204  case RK_None:
5205  break;
5206  }
5207  return 0;
5208  }
5209 
5210  /// Total number of operands in the reduction operation.
5211  unsigned getNumberOfOperands() const {
5212  assert(Kind != RK_None && !!*this && LHS && RHS &&
5213  "Expected reduction operation.");
5214  switch (Kind) {
5215  case RK_Arithmetic:
5216  return 2;
5217  case RK_Min:
5218  case RK_UMin:
5219  case RK_Max:
5220  case RK_UMax:
5221  return 3;
5222  case RK_None:
5223  break;
5224  }
5225  llvm_unreachable("Reduction kind is not set");
5226  }
5227 
5228  /// Checks if the operation has the same parent as \p P.
5229  bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
5230  assert(Kind != RK_None && !!*this && LHS && RHS &&
5231  "Expected reduction operation.");
5232  if (!IsRedOp)
5233  return I->getParent() == P;
5234  switch (Kind) {
5235  case RK_Arithmetic:
5236  // Arithmetic reduction operation must be used once only.
5237  return I->getParent() == P;
5238  case RK_Min:
5239  case RK_UMin:
5240  case RK_Max:
5241  case RK_UMax: {
5242  // SelectInst must be used twice while the condition op must have single
5243  // use only.
5244  auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
5245  return I->getParent() == P && Cmp && Cmp->getParent() == P;
5246  }
5247  case RK_None:
5248  break;
5249  }
5250  llvm_unreachable("Reduction kind is not set");
5251  }
5252  /// Expected number of uses for reduction operations/reduced values.
5253  bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
5254  assert(Kind != RK_None && !!*this && LHS && RHS &&
5255  "Expected reduction operation.");
5256  switch (Kind) {
5257  case RK_Arithmetic:
5258  return I->hasOneUse();
5259  case RK_Min:
5260  case RK_UMin:
5261  case RK_Max:
5262  case RK_UMax:
5263  return I->hasNUses(2) &&
5264  (!IsReductionOp ||
5265  cast<SelectInst>(I)->getCondition()->hasOneUse());
5266  case RK_None:
5267  break;
5268  }
5269  llvm_unreachable("Reduction kind is not set");
5270  }
5271 
5272  /// Initializes the list of reduction operations.
5273  void initReductionOps(ReductionOpsListType &ReductionOps) {
5274  assert(Kind != RK_None && !!*this && LHS && RHS &&
5275  "Expected reduction operation.");
5276  switch (Kind) {
5277  case RK_Arithmetic:
5278  ReductionOps.assign(1, ReductionOpsType());
5279  break;
5280  case RK_Min:
5281  case RK_UMin:
5282  case RK_Max:
5283  case RK_UMax:
5284  ReductionOps.assign(2, ReductionOpsType());
5285  break;
5286  case RK_None:
5287  llvm_unreachable("Reduction kind is not set");
5288  }
5289  }
5290  /// Add all reduction operations for the reduction instruction \p I.
5291  void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
5292  assert(Kind != RK_None && !!*this && LHS && RHS &&
5293  "Expected reduction operation.");
5294  switch (Kind) {
5295  case RK_Arithmetic:
5296  ReductionOps[0].emplace_back(I);
5297  break;
5298  case RK_Min:
5299  case RK_UMin:
5300  case RK_Max:
5301  case RK_UMax:
5302  ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
5303  ReductionOps[1].emplace_back(I);
5304  break;
5305  case RK_None:
5306  llvm_unreachable("Reduction kind is not set");
5307  }
5308  }
5309 
5310  /// Checks if instruction is associative and can be vectorized.
5311  bool isAssociative(Instruction *I) const {
5312  assert(Kind != RK_None && *this && LHS && RHS &&
5313  "Expected reduction operation.");
5314  switch (Kind) {
5315  case RK_Arithmetic:
5316  return I->isAssociative();
5317  case RK_Min:
5318  case RK_Max:
5319  return Opcode == Instruction::ICmp ||
5320  cast<Instruction>(I->getOperand(0))->isFast();
5321  case RK_UMin:
5322  case RK_UMax:
5323  assert(Opcode == Instruction::ICmp &&
5324  "Only integer compare operation is expected.");
5325  return true;
5326  case RK_None:
5327  break;
5328  }
5329  llvm_unreachable("Reduction kind is not set");
5330  }
5331 
5332  /// Checks if the reduction operation can be vectorized.
5333  bool isVectorizable(Instruction *I) const {
5334  return isVectorizable() && isAssociative(I);
5335  }
5336 
5337  /// Checks if two operation data are both a reduction op or both a reduced
5338  /// value.
5339  bool operator==(const OperationData &OD) {
5340  assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
5341  "One of the comparing operations is incorrect.");
5342  return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
5343  }
5344  bool operator!=(const OperationData &OD) { return !(*this == OD); }
5345  void clear() {
5346  Opcode = 0;
5347  LHS = nullptr;
5348  RHS = nullptr;
5349  Kind = RK_None;
5350  NoNaN = false;
5351  }
5352 
5353  /// Get the opcode of the reduction operation.
5354  unsigned getOpcode() const {
5355  assert(isVectorizable() && "Expected vectorizable operation.");
5356  return Opcode;
5357  }
5358 
5359  /// Get kind of reduction data.
5360  ReductionKind getKind() const { return Kind; }
5361  Value *getLHS() const { return LHS; }
5362  Value *getRHS() const { return RHS; }
5363  Type *getConditionType() const {
5364  switch (Kind) {
5365  case RK_Arithmetic:
5366  return nullptr;
5367  case RK_Min:
5368  case RK_Max:
5369  case RK_UMin:
5370  case RK_UMax:
5371  return CmpInst::makeCmpResultType(LHS->getType());
5372  case RK_None:
5373  break;
5374  }
5375  llvm_unreachable("Reduction kind is not set");
5376  }
5377 
5378  /// Creates reduction operation with the current opcode with the IR flags
5379  /// from \p ReductionOps.
5380  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5381  const ReductionOpsListType &ReductionOps) const {
5382  assert(isVectorizable() &&
5383  "Expected add|fadd or min/max reduction operation.");
5384  auto *Op = createOp(Builder, Name);
5385  switch (Kind) {
5386  case RK_Arithmetic:
5387  propagateIRFlags(Op, ReductionOps[0]);
5388  return Op;
5389  case RK_Min:
5390  case RK_Max:
5391  case RK_UMin:
5392  case RK_UMax:
5393  if (auto *SI = dyn_cast<SelectInst>(Op))
5394  propagateIRFlags(SI->getCondition(), ReductionOps[0]);
5395  propagateIRFlags(Op, ReductionOps[1]);
5396  return Op;
5397  case RK_None:
5398  break;
5399  }
5400  llvm_unreachable("Unknown reduction operation.");
5401  }
5402  /// Creates reduction operation with the current opcode with the IR flags
5403  /// from \p I.
5404  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5405  Instruction *I) const {
5406  assert(isVectorizable() &&
5407  "Expected add|fadd or min/max reduction operation.");
5408  auto *Op = createOp(Builder, Name);
5409  switch (Kind) {
5410  case RK_Arithmetic:
5411  propagateIRFlags(Op, I);
5412  return Op;
5413  case RK_Min:
5414  case RK_Max:
5415  case RK_UMin:
5416  case RK_UMax:
5417  if (auto *SI = dyn_cast<SelectInst>(Op)) {
5418  propagateIRFlags(SI->getCondition(),
5419  cast<SelectInst>(I)->getCondition());
5420  }
5421  propagateIRFlags(Op, I);
5422  return Op;
5423  case RK_None:
5424  break;
5425  }
5426  llvm_unreachable("Unknown reduction operation.");
5427  }
5428 
5429  TargetTransformInfo::ReductionFlags getFlags() const {
5431  Flags.NoNaN = NoNaN;
5432  switch (Kind) {
5433  case RK_Arithmetic:
5434  break;
5435  case RK_Min:
5436  Flags.IsSigned = Opcode == Instruction::ICmp;
5437  Flags.IsMaxOp = false;
5438  break;
5439  case RK_Max:
5440  Flags.IsSigned = Opcode == Instruction::ICmp;
5441  Flags.IsMaxOp = true;
5442  break;
5443  case RK_UMin:
5444  Flags.IsSigned = false;
5445  Flags.IsMaxOp = false;
5446  break;
5447  case RK_UMax:
5448  Flags.IsSigned = false;
5449  Flags.IsMaxOp = true;
5450  break;
5451  case RK_None:
5452  llvm_unreachable("Reduction kind is not set");
5453  }
5454  return Flags;
5455  }
5456  };
5457 
5458  WeakTrackingVH ReductionRoot;
5459 
5460  /// The operation data of the reduction operation.
5461  OperationData ReductionData;
5462 
5463  /// The operation data of the values we perform a reduction on.
5464  OperationData ReducedValueData;
5465 
5466  /// Should we model this reduction as a pairwise reduction tree or a tree that
5467  /// splits the vector in halves and adds those halves.
5468  bool IsPairwiseReduction = false;
5469 
5470  /// Checks if the ParentStackElem.first should be marked as a reduction
5471  /// operation with an extra argument or as extra argument itself.
5472  void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5473  Value *ExtraArg) {
5474  if (ExtraArgs.count(ParentStackElem.first)) {
5475  ExtraArgs[ParentStackElem.first] = nullptr;
5476  // We ran into something like:
5477  // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5478  // The whole ParentStackElem.first should be considered as an extra value
5479  // in this case.
5480  // Do not perform analysis of remaining operands of ParentStackElem.first
5481  // instruction, this whole instruction is an extra argument.
5482  ParentStackElem.second = ParentStackElem.first->getNumOperands();
5483  } else {
5484  // We ran into something like:
5485  // ParentStackElem.first += ... + ExtraArg + ...
5486  ExtraArgs[ParentStackElem.first] = ExtraArg;
5487  }
5488  }
5489 
5490  static OperationData getOperationData(Value *V) {
5491  if (!V)
5492  return OperationData();
5493 
5494  Value *LHS;
5495  Value *RHS;
5496  if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5497  return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5498  RK_Arithmetic);
5499  }
5500  if (auto *Select = dyn_cast<SelectInst>(V)) {
5501  // Look for a min/max pattern.
5502  if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5503  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5504  } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5505  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5506  } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5507  m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5508  return OperationData(
5509  Instruction::FCmp, LHS, RHS, RK_Min,
5510  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5511  } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5512  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5513  } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5514  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5515  } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5516  m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5517  return OperationData(
5518  Instruction::FCmp, LHS, RHS, RK_Max,
5519  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5520  } else {
5521  // Try harder: look for min/max pattern based on instructions producing
5522  // same values such as: select ((cmp Inst1, Inst2), Inst1, Inst2).
5523  // During the intermediate stages of SLP, it's very common to have
5524  // pattern like this (since optimizeGatherSequence is run only once
5525  // at the end):
5526  // %1 = extractelement <2 x i32> %a, i32 0
5527  // %2 = extractelement <2 x i32> %a, i32 1
5528  // %cond = icmp sgt i32 %1, %2
5529  // %3 = extractelement <2 x i32> %a, i32 0
5530  // %4 = extractelement <2 x i32> %a, i32 1
5531  // %select = select i1 %cond, i32 %3, i32 %4
5532  CmpInst::Predicate Pred;
5533  Instruction *L1;
5534  Instruction *L2;
5535 
5536  LHS = Select->getTrueValue();
5537  RHS = Select->getFalseValue();
5538  Value *Cond = Select->getCondition();
5539 
5540  // TODO: Support inverse predicates.
5541  if (match(Cond, m_Cmp(Pred, m_Specific(LHS), m_Instruction(L2)))) {
5542  if (!isa<ExtractElementInst>(RHS) ||
5543  !L2->isIdenticalTo(cast<Instruction>(RHS)))
5544  return OperationData(V);
5545  } else if (match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Specific(RHS)))) {
5546  if (!isa<ExtractElementInst>(LHS) ||
5547  !L1->isIdenticalTo(cast<Instruction>(LHS)))
5548  return OperationData(V);
5549  } else {
5550  if (!isa<ExtractElementInst>(LHS) || !isa<ExtractElementInst>(RHS))
5551  return OperationData(V);
5552  if (!match(Cond, m_Cmp(Pred, m_Instruction(L1), m_Instruction(L2))) ||
5553  !L1->isIdenticalTo(cast<Instruction>(LHS)) ||
5554  !L2->isIdenticalTo(cast<Instruction>(RHS)))
5555  return OperationData(V);
5556  }
5557  switch (Pred) {
5558  default:
5559  return OperationData(V);
5560 
5561  case CmpInst::ICMP_ULT:
5562  case CmpInst::ICMP_ULE:
5563  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5564 
5565  case CmpInst::ICMP_SLT:
5566  case CmpInst::ICMP_SLE:
5567  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5568 
5569  case CmpInst::FCMP_OLT:
5570  case CmpInst::FCMP_OLE:
5571  case CmpInst::FCMP_ULT:
5572  case CmpInst::FCMP_ULE:
5573  return OperationData(Instruction::FCmp, LHS, RHS, RK_Min,
5574  cast<Instruction>(Cond)->hasNoNaNs());
5575 
5576  case CmpInst::ICMP_UGT:
5577  case CmpInst::ICMP_UGE:
5578  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5579 
5580  case CmpInst::ICMP_SGT:
5581  case CmpInst::ICMP_SGE:
5582  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5583 
5584  case CmpInst::FCMP_OGT:
5585  case CmpInst::FCMP_OGE:
5586  case CmpInst::FCMP_UGT:
5587  case CmpInst::FCMP_UGE:
5588  return OperationData(Instruction::FCmp, LHS, RHS, RK_Max,
5589  cast<Instruction>(Cond)->hasNoNaNs());
5590  }
5591  }
5592  }
5593  return OperationData(V);
5594  }
5595 
5596 public:
5597  HorizontalReduction() = default;
5598 
5599  /// Try to find a reduction tree.
5600  bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5601  assert((!Phi || is_contained(Phi->operands(), B)) &&
5602  "Thi phi needs to use the binary operator");
5603 
5604  ReductionData = getOperationData(B);
5605 
5606  // We could have a initial reductions that is not an add.
5607  // r *= v1 + v2 + v3 + v4
5608  // In such a case start looking for a tree rooted in the first '+'.
5609  if (Phi) {
5610  if (ReductionData.getLHS() == Phi) {
5611  Phi = nullptr;
5612  B = dyn_cast<Instruction>(ReductionData.getRHS());
5613  ReductionData = getOperationData(B);
5614  } else if (ReductionData.getRHS() == Phi) {
5615  Phi = nullptr;
5616  B = dyn_cast<Instruction>(ReductionData.getLHS());
5617  ReductionData = getOperationData(B);
5618  }
5619  }
5620 
5621  if (!ReductionData.isVectorizable(B))
5622  return false;
5623 
5624  Type *Ty = B->getType();
5625  if (!isValidElementType(Ty))
5626  return false;
5627  if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy())
5628  return false;
5629 
5630  ReducedValueData.clear();
5631  ReductionRoot = B;
5632 
5633  // Post order traverse the reduction tree starting at B. We only handle true
5634  // trees containing only binary operators.
5636  Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5637  ReductionData.initReductionOps(ReductionOps);
5638  while (!Stack.empty()) {
5639  Instruction *TreeN = Stack.back().first;
5640  unsigned EdgeToVist = Stack.back().second++;
5641  OperationData OpData = getOperationData(TreeN);
5642  bool IsReducedValue = OpData != ReductionData;
5643 
5644  // Postorder vist.
5645  if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5646  if (IsReducedValue)
5647  ReducedVals.push_back(TreeN);
5648  else {
5649  auto I = ExtraArgs.find(TreeN);
5650  if (I != ExtraArgs.end() && !I->second) {
5651  // Check if TreeN is an extra argument of its parent operation.
5652  if (Stack.size() <= 1) {
5653  // TreeN can't be an extra argument as it is a root reduction
5654  // operation.
5655  return false;
5656  }
5657  // Yes, TreeN is an extra argument, do not add it to a list of
5658  // reduction operations.
5659  // Stack[Stack.size() - 2] always points to the parent operation.
5660  markExtraArg(Stack[Stack.size() - 2], TreeN);
5661  ExtraArgs.erase(TreeN);
5662  } else
5663  ReductionData.addReductionOps(TreeN, ReductionOps);
5664  }
5665  // Retract.
5666  Stack.pop_back();
5667  continue;
5668  }
5669 
5670  // Visit left or right.
5671  Value *NextV = TreeN->getOperand(EdgeToVist);
5672  if (NextV != Phi) {
5673  auto *I = dyn_cast<Instruction>(NextV);
5674  OpData = getOperationData(I);
5675  // Continue analysis if the next operand is a reduction operation or
5676  // (possibly) a reduced value. If the reduced value opcode is not set,
5677  // the first met operation != reduction operation is considered as the
5678  // reduced value class.
5679  if (I && (!ReducedValueData || OpData == ReducedValueData ||
5680  OpData == ReductionData)) {
5681  const bool IsReductionOperation = OpData == ReductionData;
5682  // Only handle trees in the current basic block.
5683  if (!ReductionData.hasSameParent(I, B->getParent(),
5684  IsReductionOperation)) {
5685  // I is an extra argument for TreeN (its parent operation).
5686  markExtraArg(Stack.back(), I);
5687  continue;
5688  }
5689 
5690  // Each tree node needs to have minimal number of users except for the
5691  // ultimate reduction.
5692  if (!ReductionData.hasRequiredNumberOfUses(I,
5693  OpData == ReductionData) &&
5694  I != B) {
5695  // I is an extra argument for TreeN (its parent operation).
5696  markExtraArg(Stack.back(), I);
5697  continue;
5698  }
5699 
5700  if (IsReductionOperation) {
5701  // We need to be able to reassociate the reduction operations.
5702  if (!OpData.isAssociative(I)) {
5703  // I is an extra argument for TreeN (its parent operation).
5704  markExtraArg(Stack.back(), I);
5705  continue;
5706  }
5707  } else if (ReducedValueData &&
5708  ReducedValueData != OpData) {
5709  // Make sure that the opcodes of the operations that we are going to
5710  // reduce match.
5711  // I is an extra argument for TreeN (its parent operation).
5712  markExtraArg(Stack.back(), I);
5713  continue;
5714  } else if (!ReducedValueData)
5715  ReducedValueData = OpData;
5716 
5717  Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5718  continue;
5719  }
5720  }
5721  // NextV is an extra argument for TreeN (its parent operation).
5722  markExtraArg(Stack.back(), NextV);
5723  }
5724  return true;
5725  }
5726 
5727  /// Attempt to vectorize the tree found by
5728  /// matchAssociativeReduction.
5729  bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5730  if (ReducedVals.empty())
5731  return false;
5732 
5733  // If there is a sufficient number of reduction values, reduce
5734  // to a nearby power-of-2. Can safely generate oversized
5735  // vectors and rely on the backend to split them to legal sizes.
5736  unsigned NumReducedVals = ReducedVals.size();
5737  if (NumReducedVals < 4)
5738  return false;
5739 
5740  unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5741 
5742  Value *VectorizedTree = nullptr;
5743  IRBuilder<> Builder(cast<Instruction>(ReductionRoot));
5744  FastMathFlags Unsafe;
5745  Unsafe.setFast();
5746  Builder.setFastMathFlags(Unsafe);
5747  unsigned i = 0;
5748 
5749  BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5750  // The same extra argument may be used several time, so log each attempt
5751  // to use it.
5752  for (auto &Pair : ExtraArgs) {
5753  assert(Pair.first && "DebugLoc must be set.");
5754  ExternallyUsedValues[Pair.second].push_back(Pair.first);
5755  }
5756  // The reduction root is used as the insertion point for new instructions,
5757  // so set it as externally used to prevent it from being deleted.
5758  ExternallyUsedValues[ReductionRoot];
5759  SmallVector<Value *, 16> IgnoreList;
5760  for (auto &V : ReductionOps)
5761  IgnoreList.append(V.begin(), V.end());
5762  while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5763  auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5764  V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5765  Optional<ArrayRef<unsigned>> Order = V.bestOrder();
5766  // TODO: Handle orders of size less than number of elements in the vector.
5767  if (Order && Order->size() == VL.size()) {
5768  // TODO: reorder tree nodes without tree rebuilding.
5769  SmallVector<Value *, 4> ReorderedOps(VL.size());
5770  llvm::transform(*Order, ReorderedOps.begin(),
5771  [VL](const unsigned Idx) { return VL[Idx]; });
5772  V.buildTree(ReorderedOps, ExternallyUsedValues, IgnoreList);
5773  }
5774  if (V.isTreeTinyAndNotFullyVectorizable())
5775  break;
5776 
5777  V.computeMinimumValueSizes();
5778 
5779  // Estimate cost.
5780  int TreeCost = V.getTreeCost();
5781  int ReductionCost = getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5782  int Cost = TreeCost + ReductionCost;
5783  if (Cost >= -SLPCostThreshold) {
5784  V.getORE()->emit([&]() {
5785  return OptimizationRemarkMissed(
5786  SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5787  << "Vectorizing horizontal reduction is possible"
5788  << "but not beneficial with cost "
5789  << ore::NV("Cost", Cost) << " and threshold "
5790  << ore::NV("Threshold", -SLPCostThreshold);
5791  });
5792  break;
5793  }
5794 
5795  LLVM_DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:"
5796  << Cost << ". (HorRdx)\n");
5797  V.getORE()->emit([&]() {
5798  return OptimizationRemark(
5799  SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5800  << "Vectorized horizontal reduction with cost "
5801  << ore::NV("Cost", Cost) << " and with tree size "
5802  << ore::NV("TreeSize", V.getTreeSize());
5803  });
5804 
5805  // Vectorize a tree.
5806  DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5807  Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5808 
5809  // Emit a reduction.
5810  Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
5811  Value *ReducedSubTree =
5812  emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5813  if (VectorizedTree) {
5814  Builder.SetCurrentDebugLocation(Loc);
5815  OperationData VectReductionData(ReductionData.getOpcode(),
5816  VectorizedTree, ReducedSubTree,
5817  ReductionData.getKind());
5818  VectorizedTree =
5819  VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5820  } else
5821  VectorizedTree = ReducedSubTree;
5822  i += ReduxWidth;
5823  ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5824  }
5825 
5826  if (VectorizedTree) {
5827  // Finish the reduction.
5828  for (; i < NumReducedVals; ++i) {
5829  auto *I = cast<Instruction>(ReducedVals[i]);
5830  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5831  OperationData VectReductionData(ReductionData.getOpcode(),
5832  VectorizedTree, I,
5833  ReductionData.getKind());
5834  VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5835  }
5836  for (auto &Pair : ExternallyUsedValues) {
5837  // Add each externally used value to the final reduction.
5838  for (auto *I : Pair.second) {
5839  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5840  OperationData VectReductionData(ReductionData.getOpcode(),
5841  VectorizedTree, Pair.first,
5842  ReductionData.getKind());
5843  VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5844  }
5845  }
5846  // Update users.
5847  ReductionRoot->replaceAllUsesWith(VectorizedTree);
5848  }
5849  return VectorizedTree != nullptr;
5850  }
5851 
5852  unsigned numReductionValues() const {
5853  return ReducedVals.size();
5854  }
5855 
5856 private:
5857  /// Calculate the cost of a reduction.
5858  int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5859  unsigned ReduxWidth) {
5860  Type *ScalarTy = FirstReducedVal->getType();
5861  Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5862 
5863  int PairwiseRdxCost;
5864  int SplittingRdxCost;
5865  switch (ReductionData.getKind()) {
5866  case RK_Arithmetic:
5867  PairwiseRdxCost =
5868  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5869  /*IsPairwiseForm=*/true);
5870  SplittingRdxCost =
5871  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5872  /*IsPairwiseForm=*/false);
5873  break;
5874  case RK_Min: