LLVM  7.0.0svn
SLPVectorizer.cpp
Go to the documentation of this file.
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11 // stores that can be put together into vector-stores. Next, it attempts to
12 // construct vectorizable tree using the use-def chains. If a profitable tree
13 // was found, the SLP vectorizer performs vectorization on the tree.
14 //
15 // The pass is inspired by the work described in the paper:
16 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17 //
18 //===----------------------------------------------------------------------===//
19 
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/DenseSet.h"
24 #include "llvm/ADT/MapVector.h"
25 #include "llvm/ADT/None.h"
26 #include "llvm/ADT/Optional.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SetVector.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/SmallSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/Statistic.h"
34 #include "llvm/ADT/iterator.h"
41 #include "llvm/Analysis/LoopInfo.h"
50 #include "llvm/IR/Attributes.h"
51 #include "llvm/IR/BasicBlock.h"
52 #include "llvm/IR/Constant.h"
53 #include "llvm/IR/Constants.h"
54 #include "llvm/IR/DataLayout.h"
55 #include "llvm/IR/DebugLoc.h"
56 #include "llvm/IR/DerivedTypes.h"
57 #include "llvm/IR/Dominators.h"
58 #include "llvm/IR/Function.h"
59 #include "llvm/IR/IRBuilder.h"
60 #include "llvm/IR/InstrTypes.h"
61 #include "llvm/IR/Instruction.h"
62 #include "llvm/IR/Instructions.h"
63 #include "llvm/IR/IntrinsicInst.h"
64 #include "llvm/IR/Intrinsics.h"
65 #include "llvm/IR/Module.h"
66 #include "llvm/IR/NoFolder.h"
67 #include "llvm/IR/Operator.h"
68 #include "llvm/IR/PassManager.h"
69 #include "llvm/IR/PatternMatch.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/Pass.h"
77 #include "llvm/Support/Casting.h"
79 #include "llvm/Support/Compiler.h"
81 #include "llvm/Support/Debug.h"
84 #include "llvm/Support/KnownBits.h"
89 #include <algorithm>
90 #include <cassert>
91 #include <cstdint>
92 #include <iterator>
93 #include <memory>
94 #include <set>
95 #include <string>
96 #include <tuple>
97 #include <utility>
98 #include <vector>
99 
100 using namespace llvm;
101 using namespace llvm::PatternMatch;
102 using namespace slpvectorizer;
103 
104 #define SV_NAME "slp-vectorizer"
105 #define DEBUG_TYPE "SLP"
106 
107 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
108 
109 static cl::opt<int>
110  SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111  cl::desc("Only vectorize if you gain more than this "
112  "number "));
113 
114 static cl::opt<bool>
115 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116  cl::desc("Attempt to vectorize horizontal reductions"));
117 
119  "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120  cl::desc(
121  "Attempt to vectorize horizontal reductions feeding into a store"));
122 
123 static cl::opt<int>
124 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125  cl::desc("Attempt to vectorize for this register size in bits"));
126 
127 /// Limits the size of scheduling regions in a block.
128 /// It avoid long compile times for _very_ large blocks where vector
129 /// instructions are spread over a wide range.
130 /// This limit is way higher than needed by real-world functions.
131 static cl::opt<int>
132 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133  cl::desc("Limit the size of the SLP scheduling region per block"));
134 
136  "slp-min-reg-size", cl::init(128), cl::Hidden,
137  cl::desc("Attempt to vectorize for this register size in bits"));
138 
140  "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141  cl::desc("Limit the recursion depth when building a vectorizable tree"));
142 
144  "slp-min-tree-size", cl::init(3), cl::Hidden,
145  cl::desc("Only vectorize small trees if they are fully vectorizable"));
146 
147 static cl::opt<bool>
148  ViewSLPTree("view-slp-tree", cl::Hidden,
149  cl::desc("Display the SLP trees with Graphviz"));
150 
151 // Limit the number of alias checks. The limit is chosen so that
152 // it has no negative effect on the llvm benchmarks.
153 static const unsigned AliasedCheckLimit = 10;
154 
155 // Another limit for the alias checks: The maximum distance between load/store
156 // instructions where alias checks are done.
157 // This limit is useful for very large basic blocks.
158 static const unsigned MaxMemDepDistance = 160;
159 
160 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161 /// regions to be handled.
162 static const int MinScheduleRegionSize = 16;
163 
164 /// \brief Predicate for the element types that the SLP vectorizer supports.
165 ///
166 /// The most important thing to filter here are types which are invalid in LLVM
167 /// vectors. We also filter target specific types which have absolutely no
168 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169 /// avoids spending time checking the cost model and realizing that they will
170 /// be inevitably scalarized.
171 static bool isValidElementType(Type *Ty) {
172  return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173  !Ty->isPPC_FP128Ty();
174 }
175 
176 /// \returns true if all of the instructions in \p VL are in the same block or
177 /// false otherwise.
179  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180  if (!I0)
181  return false;
182  BasicBlock *BB = I0->getParent();
183  for (int i = 1, e = VL.size(); i < e; i++) {
184  Instruction *I = dyn_cast<Instruction>(VL[i]);
185  if (!I)
186  return false;
187 
188  if (BB != I->getParent())
189  return false;
190  }
191  return true;
192 }
193 
194 /// \returns True if all of the values in \p VL are constants.
196  for (Value *i : VL)
197  if (!isa<Constant>(i))
198  return false;
199  return true;
200 }
201 
202 /// \returns True if all of the values in \p VL are identical.
203 static bool isSplat(ArrayRef<Value *> VL) {
204  for (unsigned i = 1, e = VL.size(); i < e; ++i)
205  if (VL[i] != VL[0])
206  return false;
207  return true;
208 }
209 
210 /// Checks if the vector of instructions can be represented as a shuffle, like:
211 /// %x0 = extractelement <4 x i8> %x, i32 0
212 /// %x3 = extractelement <4 x i8> %x, i32 3
213 /// %y1 = extractelement <4 x i8> %y, i32 1
214 /// %y2 = extractelement <4 x i8> %y, i32 2
215 /// %x0x0 = mul i8 %x0, %x0
216 /// %x3x3 = mul i8 %x3, %x3
217 /// %y1y1 = mul i8 %y1, %y1
218 /// %y2y2 = mul i8 %y2, %y2
219 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223 /// ret <4 x i8> %ins4
224 /// can be transformed into:
225 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226 /// i32 6>
227 /// %2 = mul <4 x i8> %1, %1
228 /// ret <4 x i8> %2
229 /// We convert this initially to something like:
230 /// %x0 = extractelement <4 x i8> %x, i32 0
231 /// %x3 = extractelement <4 x i8> %x, i32 3
232 /// %y1 = extractelement <4 x i8> %y, i32 1
233 /// %y2 = extractelement <4 x i8> %y, i32 2
234 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238 /// %5 = mul <4 x i8> %4, %4
239 /// %6 = extractelement <4 x i8> %5, i32 0
240 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241 /// %7 = extractelement <4 x i8> %5, i32 1
242 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243 /// %8 = extractelement <4 x i8> %5, i32 2
244 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245 /// %9 = extractelement <4 x i8> %5, i32 3
246 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247 /// ret <4 x i8> %ins4
248 /// InstCombiner transforms this into a shuffle and vector mul
251  auto *EI0 = cast<ExtractElementInst>(VL[0]);
252  unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253  Value *Vec1 = nullptr;
254  Value *Vec2 = nullptr;
255  enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256  ShuffleMode CommonShuffleMode = Unknown;
257  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258  auto *EI = cast<ExtractElementInst>(VL[I]);
259  auto *Vec = EI->getVectorOperand();
260  // All vector operands must have the same number of vector elements.
261  if (Vec->getType()->getVectorNumElements() != Size)
262  return None;
263  auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264  if (!Idx)
265  return None;
266  // Undefined behavior if Idx is negative or >= Size.
267  if (Idx->getValue().uge(Size))
268  continue;
269  unsigned IntIdx = Idx->getValue().getZExtValue();
270  // We can extractelement from undef vector.
271  if (isa<UndefValue>(Vec))
272  continue;
273  // For correct shuffling we have to have at most 2 different vector operands
274  // in all extractelement instructions.
275  if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276  return None;
277  if (CommonShuffleMode == Permute)
278  continue;
279  // If the extract index is not the same as the operation number, it is a
280  // permutation.
281  if (IntIdx != I) {
282  CommonShuffleMode = Permute;
283  continue;
284  }
285  // Check the shuffle mode for the current operation.
286  if (!Vec1)
287  Vec1 = Vec;
288  else if (Vec != Vec1)
289  Vec2 = Vec;
290  // Example: shufflevector A, B, <0,5,2,7>
291  // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292  // I is even and IntIdx for B == I - FirstAlternate shuffle.
293  // Example: shufflevector A, B, <4,1,6,3>
294  // I is even and IntIdx for A == I - SecondAlternate shuffle.
295  // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296  const bool IIsEven = I & 1;
297  const bool CurrVecIsA = Vec == Vec1;
298  const bool IIsOdd = !IIsEven;
299  const bool CurrVecIsB = !CurrVecIsA;
300  ShuffleMode CurrentShuffleMode =
301  ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302  : SecondAlternate;
303  // Common mode is not set or the same as the shuffle mode of the current
304  // operation - alternate.
305  if (CommonShuffleMode == Unknown)
306  CommonShuffleMode = CurrentShuffleMode;
307  // Common shuffle mode is not the same as the shuffle mode of the current
308  // operation - permutation.
309  if (CommonShuffleMode != CurrentShuffleMode)
310  CommonShuffleMode = Permute;
311  }
312  // If we're not crossing lanes in different vectors, consider it as blending.
313  if ((CommonShuffleMode == FirstAlternate ||
314  CommonShuffleMode == SecondAlternate) &&
315  Vec2)
317  // If Vec2 was never used, we have a permutation of a single vector, otherwise
318  // we have permutation of 2 vectors.
321 }
322 
323 ///\returns Opcode that can be clubbed with \p Op to create an alternate
324 /// sequence which can later be merged as a ShuffleVector instruction.
325 static unsigned getAltOpcode(unsigned Op) {
326  switch (Op) {
327  case Instruction::FAdd:
328  return Instruction::FSub;
329  case Instruction::FSub:
330  return Instruction::FAdd;
331  case Instruction::Add:
332  return Instruction::Sub;
333  case Instruction::Sub:
334  return Instruction::Add;
335  default:
336  return 0;
337  }
338 }
339 
340 static bool isOdd(unsigned Value) {
341  return Value & 1;
342 }
343 
344 static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345  unsigned CheckedOpcode) {
346  return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347 }
348 
349 /// Chooses the correct key for scheduling data. If \p Op has the same (or
350 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
351 /// OpValue.
352 static Value *isOneOf(Value *OpValue, Value *Op) {
353  auto *I = dyn_cast<Instruction>(Op);
354  if (!I)
355  return OpValue;
356  auto *OpInst = cast<Instruction>(OpValue);
357  unsigned OpInstOpcode = OpInst->getOpcode();
358  unsigned IOpcode = I->getOpcode();
359  if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
360  return Op;
361  return OpValue;
362 }
363 
364 namespace {
365 
366 /// Contains data for the instructions going to be vectorized.
367 struct RawInstructionsData {
368  /// Main Opcode of the instructions going to be vectorized.
369  unsigned Opcode = 0;
370 
371  /// The list of instructions have some instructions with alternate opcodes.
372  bool HasAltOpcodes = false;
373 };
374 
375 } // end anonymous namespace
376 
377 /// Checks the list of the vectorized instructions \p VL and returns info about
378 /// this list.
379 static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380  auto *I0 = dyn_cast<Instruction>(VL[0]);
381  if (!I0)
382  return {};
383  RawInstructionsData Res;
384  unsigned Opcode = I0->getOpcode();
385  // Walk through the list of the vectorized instructions
386  // in order to check its structure described by RawInstructionsData.
387  for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388  auto *I = dyn_cast<Instruction>(VL[Cnt]);
389  if (!I)
390  return {};
391  if (Opcode != I->getOpcode())
392  Res.HasAltOpcodes = true;
393  }
394  Res.Opcode = Opcode;
395  return Res;
396 }
397 
398 namespace {
399 
400 /// Main data required for vectorization of instructions.
401 struct InstructionsState {
402  /// The very first instruction in the list with the main opcode.
403  Value *OpValue = nullptr;
404 
405  /// The main opcode for the list of instructions.
406  unsigned Opcode = 0;
407 
408  /// Some of the instructions in the list have alternate opcodes.
409  bool IsAltShuffle = false;
410 
411  InstructionsState() = default;
412  InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413  : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
414 };
415 
416 } // end anonymous namespace
417 
418 /// \returns analysis of the Instructions in \p VL described in
419 /// InstructionsState, the Opcode that we suppose the whole list
420 /// could be vectorized even if its structure is diverse.
421 static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422  auto Res = getMainOpcode(VL);
423  unsigned Opcode = Res.Opcode;
424  if (!Res.HasAltOpcodes)
425  return InstructionsState(VL[0], Opcode, false);
426  auto *OpInst = cast<Instruction>(VL[0]);
427  unsigned AltOpcode = getAltOpcode(Opcode);
428  // Examine each element in the list instructions VL to determine
429  // if some operations there could be considered as an alternative
430  // (for example as subtraction relates to addition operation).
431  for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432  auto *I = cast<Instruction>(VL[Cnt]);
433  unsigned InstOpcode = I->getOpcode();
434  if ((Res.HasAltOpcodes &&
435  InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436  (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437  return InstructionsState(OpInst, 0, false);
438  }
439  }
440  return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
441 }
442 
443 /// \returns true if all of the values in \p VL have the same type or false
444 /// otherwise.
446  Type *Ty = VL[0]->getType();
447  for (int i = 1, e = VL.size(); i < e; i++)
448  if (VL[i]->getType() != Ty)
449  return false;
450 
451  return true;
452 }
453 
454 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
455 static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
456  assert(Opcode == Instruction::ExtractElement ||
457  Opcode == Instruction::ExtractValue);
458  if (Opcode == Instruction::ExtractElement) {
460  return CI && CI->getZExtValue() == Idx;
461  } else {
462  ExtractValueInst *EI = cast<ExtractValueInst>(E);
463  return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
464  }
465 }
466 
467 /// \returns True if in-tree use also needs extract. This refers to
468 /// possible scalar operand in vectorized instruction.
470  TargetLibraryInfo *TLI) {
471  unsigned Opcode = UserInst->getOpcode();
472  switch (Opcode) {
473  case Instruction::Load: {
474  LoadInst *LI = cast<LoadInst>(UserInst);
475  return (LI->getPointerOperand() == Scalar);
476  }
477  case Instruction::Store: {
478  StoreInst *SI = cast<StoreInst>(UserInst);
479  return (SI->getPointerOperand() == Scalar);
480  }
481  case Instruction::Call: {
482  CallInst *CI = cast<CallInst>(UserInst);
484  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
485  return (CI->getArgOperand(1) == Scalar);
486  }
488  }
489  default:
490  return false;
491  }
492 }
493 
494 /// \returns the AA location that is being access by the instruction.
496  if (StoreInst *SI = dyn_cast<StoreInst>(I))
497  return MemoryLocation::get(SI);
498  if (LoadInst *LI = dyn_cast<LoadInst>(I))
499  return MemoryLocation::get(LI);
500  return MemoryLocation();
501 }
502 
503 /// \returns True if the instruction is not a volatile or atomic load/store.
504 static bool isSimple(Instruction *I) {
505  if (LoadInst *LI = dyn_cast<LoadInst>(I))
506  return LI->isSimple();
507  if (StoreInst *SI = dyn_cast<StoreInst>(I))
508  return SI->isSimple();
509  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
510  return !MI->isVolatile();
511  return true;
512 }
513 
514 namespace llvm {
515 
516 namespace slpvectorizer {
517 
518 /// Bottom Up SLP Vectorizer.
519 class BoUpSLP {
520 public:
527 
531  const DataLayout *DL, OptimizationRemarkEmitter *ORE)
532  : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
533  DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
534  CodeMetrics::collectEphemeralValues(F, AC, EphValues);
535  // Use the vector register size specified by the target unless overridden
536  // by a command-line option.
537  // TODO: It would be better to limit the vectorization factor based on
538  // data type rather than just register size. For example, x86 AVX has
539  // 256-bit registers, but it does not support integer operations
540  // at that width (that requires AVX2).
541  if (MaxVectorRegSizeOption.getNumOccurrences())
542  MaxVecRegSize = MaxVectorRegSizeOption;
543  else
544  MaxVecRegSize = TTI->getRegisterBitWidth(true);
545 
546  if (MinVectorRegSizeOption.getNumOccurrences())
547  MinVecRegSize = MinVectorRegSizeOption;
548  else
549  MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
550  }
551 
552  /// \brief Vectorize the tree that starts with the elements in \p VL.
553  /// Returns the vectorized root.
554  Value *vectorizeTree();
555 
556  /// Vectorize the tree but with the list of externally used values \p
557  /// ExternallyUsedValues. Values in this MapVector can be replaced but the
558  /// generated extractvalue instructions.
559  Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
560 
561  /// \returns the cost incurred by unwanted spills and fills, caused by
562  /// holding live values over call sites.
563  int getSpillCost();
564 
565  /// \returns the vectorization cost of the subtree that starts at \p VL.
566  /// A negative number means that this is profitable.
567  int getTreeCost();
568 
569  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
570  /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
571  void buildTree(ArrayRef<Value *> Roots,
572  ArrayRef<Value *> UserIgnoreLst = None);
573 
574  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575  /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
576  /// into account (anf updating it, if required) list of externally used
577  /// values stored in \p ExternallyUsedValues.
578  void buildTree(ArrayRef<Value *> Roots,
579  ExtraValueToDebugLocsMap &ExternallyUsedValues,
580  ArrayRef<Value *> UserIgnoreLst = None);
581 
582  /// Clear the internal data structures that are created by 'buildTree'.
583  void deleteTree() {
584  VectorizableTree.clear();
585  ScalarToTreeEntry.clear();
586  MustGather.clear();
587  ExternalUses.clear();
588  NumOpsWantToKeepOrder.clear();
589  for (auto &Iter : BlocksSchedules) {
590  BlockScheduling *BS = Iter.second.get();
591  BS->clear();
592  }
593  MinBWs.clear();
594  }
595 
596  unsigned getTreeSize() const { return VectorizableTree.size(); }
597 
598  /// \brief Perform LICM and CSE on the newly generated gather sequences.
599  void optimizeGatherSequence();
600 
601  /// \returns true if it is beneficial to reverse the vector order.
602  bool shouldReorder() const {
603  return std::accumulate(
604  NumOpsWantToKeepOrder.begin(), NumOpsWantToKeepOrder.end(), 0,
605  [](int Val1,
606  const decltype(NumOpsWantToKeepOrder)::value_type &Val2) {
607  return Val1 + (Val2.second < 0 ? 1 : -1);
608  }) > 0;
609  }
610 
611  /// \return The vector element size in bits to use when vectorizing the
612  /// expression tree ending at \p V. If V is a store, the size is the width of
613  /// the stored value. Otherwise, the size is the width of the largest loaded
614  /// value reaching V. This method is used by the vectorizer to calculate
615  /// vectorization factors.
616  unsigned getVectorElementSize(Value *V);
617 
618  /// Compute the minimum type sizes required to represent the entries in a
619  /// vectorizable tree.
621 
622  // \returns maximum vector register size as set by TTI or overridden by cl::opt.
623  unsigned getMaxVecRegSize() const {
624  return MaxVecRegSize;
625  }
626 
627  // \returns minimum vector register size as set by cl::opt.
628  unsigned getMinVecRegSize() const {
629  return MinVecRegSize;
630  }
631 
632  /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
633  ///
634  /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
635  unsigned canMapToVector(Type *T, const DataLayout &DL) const;
636 
637  /// \returns True if the VectorizableTree is both tiny and not fully
638  /// vectorizable. We do not vectorize such trees.
639  bool isTreeTinyAndNotFullyVectorizable();
640 
642 
643 private:
644  struct TreeEntry;
645 
646  /// Checks if all users of \p I are the part of the vectorization tree.
647  bool areAllUsersVectorized(Instruction *I) const;
648 
649  /// \returns the cost of the vectorizable entry.
650  int getEntryCost(TreeEntry *E);
651 
652  /// This is the recursive part of buildTree.
653  void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
654 
655  /// \returns True if the ExtractElement/ExtractValue instructions in VL can
656  /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
657  bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const;
658 
659  /// Vectorize a single entry in the tree.
660  Value *vectorizeTree(TreeEntry *E);
661 
662  /// Vectorize a single entry in the tree, starting in \p VL.
663  Value *vectorizeTree(ArrayRef<Value *> VL);
664 
665  /// \returns the pointer to the vectorized value if \p VL is already
666  /// vectorized, or NULL. They may happen in cycles.
667  Value *alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const;
668 
669  /// \returns the scalarization cost for this type. Scalarization in this
670  /// context means the creation of vectors from a group of scalars.
671  int getGatherCost(Type *Ty);
672 
673  /// \returns the scalarization cost for this list of values. Assuming that
674  /// this subtree gets vectorized, we may need to extract the values from the
675  /// roots. This method calculates the cost of extracting the values.
676  int getGatherCost(ArrayRef<Value *> VL);
677 
678  /// \brief Set the Builder insert point to one after the last instruction in
679  /// the bundle
680  void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
681 
682  /// \returns a vector from a collection of scalars in \p VL.
683  Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
684 
685  /// \returns whether the VectorizableTree is fully vectorizable and will
686  /// be beneficial even the tree height is tiny.
687  bool isFullyVectorizableTinyTree();
688 
689  /// \reorder commutative operands in alt shuffle if they result in
690  /// vectorized code.
691  void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
694 
695  /// \reorder commutative operands to get better probability of
696  /// generating vectorized code.
697  void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
699  SmallVectorImpl<Value *> &Right);
700  struct TreeEntry {
701  TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
702 
703  /// \returns true if the scalars in VL are equal to this entry.
704  bool isSame(ArrayRef<Value *> VL) const {
705  assert(VL.size() == Scalars.size() && "Invalid size");
706  return std::equal(VL.begin(), VL.end(), Scalars.begin());
707  }
708 
709  /// A vector of scalars.
710  ValueList Scalars;
711 
712  /// The Scalars are vectorized into this value. It is initialized to Null.
713  Value *VectorizedValue = nullptr;
714 
715  /// Do we need to gather this sequence ?
716  bool NeedToGather = false;
717 
718  /// Points back to the VectorizableTree.
719  ///
720  /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
721  /// to be a pointer and needs to be able to initialize the child iterator.
722  /// Thus we need a reference back to the container to translate the indices
723  /// to entries.
724  std::vector<TreeEntry> &Container;
725 
726  /// The TreeEntry index containing the user of this entry. We can actually
727  /// have multiple users so the data structure is not truly a tree.
728  SmallVector<int, 1> UserTreeIndices;
729  };
730 
731  /// Create a new VectorizableTree entry.
732  TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
733  int &UserTreeIdx) {
734  VectorizableTree.emplace_back(VectorizableTree);
735  int idx = VectorizableTree.size() - 1;
736  TreeEntry *Last = &VectorizableTree[idx];
737  Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
738  Last->NeedToGather = !Vectorized;
739  if (Vectorized) {
740  for (int i = 0, e = VL.size(); i != e; ++i) {
741  assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
742  ScalarToTreeEntry[VL[i]] = idx;
743  }
744  } else {
745  MustGather.insert(VL.begin(), VL.end());
746  }
747 
748  if (UserTreeIdx >= 0)
749  Last->UserTreeIndices.push_back(UserTreeIdx);
750  UserTreeIdx = idx;
751  return Last;
752  }
753 
754  /// -- Vectorization State --
755  /// Holds all of the tree entries.
756  std::vector<TreeEntry> VectorizableTree;
757 
758  TreeEntry *getTreeEntry(Value *V) {
759  auto I = ScalarToTreeEntry.find(V);
760  if (I != ScalarToTreeEntry.end())
761  return &VectorizableTree[I->second];
762  return nullptr;
763  }
764 
765  const TreeEntry *getTreeEntry(Value *V) const {
766  auto I = ScalarToTreeEntry.find(V);
767  if (I != ScalarToTreeEntry.end())
768  return &VectorizableTree[I->second];
769  return nullptr;
770  }
771 
772  /// Maps a specific scalar to its tree entry.
773  SmallDenseMap<Value*, int> ScalarToTreeEntry;
774 
775  /// A list of scalars that we found that we need to keep as scalars.
776  ValueSet MustGather;
777 
778  /// This POD struct describes one external user in the vectorized tree.
779  struct ExternalUser {
780  ExternalUser(Value *S, llvm::User *U, int L)
781  : Scalar(S), User(U), Lane(L) {}
782 
783  // Which scalar in our function.
784  Value *Scalar;
785 
786  // Which user that uses the scalar.
787  llvm::User *User;
788 
789  // Which lane does the scalar belong to.
790  int Lane;
791  };
793 
794  /// Checks if two instructions may access the same memory.
795  ///
796  /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
797  /// is invariant in the calling loop.
798  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
799  Instruction *Inst2) {
800  // First check if the result is already in the cache.
801  AliasCacheKey key = std::make_pair(Inst1, Inst2);
802  Optional<bool> &result = AliasCache[key];
803  if (result.hasValue()) {
804  return result.getValue();
805  }
806  MemoryLocation Loc2 = getLocation(Inst2, AA);
807  bool aliased = true;
808  if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
809  // Do the alias check.
810  aliased = AA->alias(Loc1, Loc2);
811  }
812  // Store the result in the cache.
813  result = aliased;
814  return aliased;
815  }
816 
817  using AliasCacheKey = std::pair<Instruction *, Instruction *>;
818 
819  /// Cache for alias results.
820  /// TODO: consider moving this to the AliasAnalysis itself.
822 
823  /// Removes an instruction from its block and eventually deletes it.
824  /// It's like Instruction::eraseFromParent() except that the actual deletion
825  /// is delayed until BoUpSLP is destructed.
826  /// This is required to ensure that there are no incorrect collisions in the
827  /// AliasCache, which can happen if a new instruction is allocated at the
828  /// same address as a previously deleted instruction.
829  void eraseInstruction(Instruction *I) {
830  I->removeFromParent();
831  I->dropAllReferences();
832  DeletedInstructions.emplace_back(I);
833  }
834 
835  /// Temporary store for deleted instructions. Instructions will be deleted
836  /// eventually when the BoUpSLP is destructed.
837  SmallVector<unique_value, 8> DeletedInstructions;
838 
839  /// A list of values that need to extracted out of the tree.
840  /// This list holds pairs of (Internal Scalar : External User). External User
841  /// can be nullptr, it means that this Internal Scalar will be used later,
842  /// after vectorization.
843  UserList ExternalUses;
844 
845  /// Values used only by @llvm.assume calls.
847 
848  /// Holds all of the instructions that we gathered.
849  SetVector<Instruction *> GatherSeq;
850 
851  /// A list of blocks that we are going to CSE.
852  SetVector<BasicBlock *> CSEBlocks;
853 
854  /// Contains all scheduling relevant data for an instruction.
855  /// A ScheduleData either represents a single instruction or a member of an
856  /// instruction bundle (= a group of instructions which is combined into a
857  /// vector instruction).
858  struct ScheduleData {
859  // The initial value for the dependency counters. It means that the
860  // dependencies are not calculated yet.
861  enum { InvalidDeps = -1 };
862 
863  ScheduleData() = default;
864 
865  void init(int BlockSchedulingRegionID, Value *OpVal) {
866  FirstInBundle = this;
867  NextInBundle = nullptr;
868  NextLoadStore = nullptr;
869  IsScheduled = false;
870  SchedulingRegionID = BlockSchedulingRegionID;
871  UnscheduledDepsInBundle = UnscheduledDeps;
872  clearDependencies();
873  OpValue = OpVal;
874  }
875 
876  /// Returns true if the dependency information has been calculated.
877  bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
878 
879  /// Returns true for single instructions and for bundle representatives
880  /// (= the head of a bundle).
881  bool isSchedulingEntity() const { return FirstInBundle == this; }
882 
883  /// Returns true if it represents an instruction bundle and not only a
884  /// single instruction.
885  bool isPartOfBundle() const {
886  return NextInBundle != nullptr || FirstInBundle != this;
887  }
888 
889  /// Returns true if it is ready for scheduling, i.e. it has no more
890  /// unscheduled depending instructions/bundles.
891  bool isReady() const {
892  assert(isSchedulingEntity() &&
893  "can't consider non-scheduling entity for ready list");
894  return UnscheduledDepsInBundle == 0 && !IsScheduled;
895  }
896 
897  /// Modifies the number of unscheduled dependencies, also updating it for
898  /// the whole bundle.
899  int incrementUnscheduledDeps(int Incr) {
900  UnscheduledDeps += Incr;
901  return FirstInBundle->UnscheduledDepsInBundle += Incr;
902  }
903 
904  /// Sets the number of unscheduled dependencies to the number of
905  /// dependencies.
906  void resetUnscheduledDeps() {
907  incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
908  }
909 
910  /// Clears all dependency information.
911  void clearDependencies() {
912  Dependencies = InvalidDeps;
913  resetUnscheduledDeps();
914  MemoryDependencies.clear();
915  }
916 
917  void dump(raw_ostream &os) const {
918  if (!isSchedulingEntity()) {
919  os << "/ " << *Inst;
920  } else if (NextInBundle) {
921  os << '[' << *Inst;
922  ScheduleData *SD = NextInBundle;
923  while (SD) {
924  os << ';' << *SD->Inst;
925  SD = SD->NextInBundle;
926  }
927  os << ']';
928  } else {
929  os << *Inst;
930  }
931  }
932 
933  Instruction *Inst = nullptr;
934 
935  /// Points to the head in an instruction bundle (and always to this for
936  /// single instructions).
937  ScheduleData *FirstInBundle = nullptr;
938 
939  /// Single linked list of all instructions in a bundle. Null if it is a
940  /// single instruction.
941  ScheduleData *NextInBundle = nullptr;
942 
943  /// Single linked list of all memory instructions (e.g. load, store, call)
944  /// in the block - until the end of the scheduling region.
945  ScheduleData *NextLoadStore = nullptr;
946 
947  /// The dependent memory instructions.
948  /// This list is derived on demand in calculateDependencies().
949  SmallVector<ScheduleData *, 4> MemoryDependencies;
950 
951  /// This ScheduleData is in the current scheduling region if this matches
952  /// the current SchedulingRegionID of BlockScheduling.
953  int SchedulingRegionID = 0;
954 
955  /// Used for getting a "good" final ordering of instructions.
956  int SchedulingPriority = 0;
957 
958  /// The number of dependencies. Constitutes of the number of users of the
959  /// instruction plus the number of dependent memory instructions (if any).
960  /// This value is calculated on demand.
961  /// If InvalidDeps, the number of dependencies is not calculated yet.
962  int Dependencies = InvalidDeps;
963 
964  /// The number of dependencies minus the number of dependencies of scheduled
965  /// instructions. As soon as this is zero, the instruction/bundle gets ready
966  /// for scheduling.
967  /// Note that this is negative as long as Dependencies is not calculated.
968  int UnscheduledDeps = InvalidDeps;
969 
970  /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
971  /// single instructions.
972  int UnscheduledDepsInBundle = InvalidDeps;
973 
974  /// True if this instruction is scheduled (or considered as scheduled in the
975  /// dry-run).
976  bool IsScheduled = false;
977 
978  /// Opcode of the current instruction in the schedule data.
979  Value *OpValue = nullptr;
980  };
981 
982 #ifndef NDEBUG
983  friend inline raw_ostream &operator<<(raw_ostream &os,
984  const BoUpSLP::ScheduleData &SD) {
985  SD.dump(os);
986  return os;
987  }
988 #endif
989 
990  friend struct GraphTraits<BoUpSLP *>;
991  friend struct DOTGraphTraits<BoUpSLP *>;
992 
993  /// Contains all scheduling data for a basic block.
994  struct BlockScheduling {
995  BlockScheduling(BasicBlock *BB)
996  : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
997 
998  void clear() {
999  ReadyInsts.clear();
1000  ScheduleStart = nullptr;
1001  ScheduleEnd = nullptr;
1002  FirstLoadStoreInRegion = nullptr;
1003  LastLoadStoreInRegion = nullptr;
1004 
1005  // Reduce the maximum schedule region size by the size of the
1006  // previous scheduling run.
1007  ScheduleRegionSizeLimit -= ScheduleRegionSize;
1008  if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1009  ScheduleRegionSizeLimit = MinScheduleRegionSize;
1010  ScheduleRegionSize = 0;
1011 
1012  // Make a new scheduling region, i.e. all existing ScheduleData is not
1013  // in the new region yet.
1014  ++SchedulingRegionID;
1015  }
1016 
1017  ScheduleData *getScheduleData(Value *V) {
1018  ScheduleData *SD = ScheduleDataMap[V];
1019  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1020  return SD;
1021  return nullptr;
1022  }
1023 
1024  ScheduleData *getScheduleData(Value *V, Value *Key) {
1025  if (V == Key)
1026  return getScheduleData(V);
1027  auto I = ExtraScheduleDataMap.find(V);
1028  if (I != ExtraScheduleDataMap.end()) {
1029  ScheduleData *SD = I->second[Key];
1030  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1031  return SD;
1032  }
1033  return nullptr;
1034  }
1035 
1036  bool isInSchedulingRegion(ScheduleData *SD) {
1037  return SD->SchedulingRegionID == SchedulingRegionID;
1038  }
1039 
1040  /// Marks an instruction as scheduled and puts all dependent ready
1041  /// instructions into the ready-list.
1042  template <typename ReadyListType>
1043  void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1044  SD->IsScheduled = true;
1045  DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1046 
1047  ScheduleData *BundleMember = SD;
1048  while (BundleMember) {
1049  if (BundleMember->Inst != BundleMember->OpValue) {
1050  BundleMember = BundleMember->NextInBundle;
1051  continue;
1052  }
1053  // Handle the def-use chain dependencies.
1054  for (Use &U : BundleMember->Inst->operands()) {
1055  auto *I = dyn_cast<Instruction>(U.get());
1056  if (!I)
1057  continue;
1058  doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1059  if (OpDef && OpDef->hasValidDependencies() &&
1060  OpDef->incrementUnscheduledDeps(-1) == 0) {
1061  // There are no more unscheduled dependencies after
1062  // decrementing, so we can put the dependent instruction
1063  // into the ready list.
1064  ScheduleData *DepBundle = OpDef->FirstInBundle;
1065  assert(!DepBundle->IsScheduled &&
1066  "already scheduled bundle gets ready");
1067  ReadyList.insert(DepBundle);
1068  DEBUG(dbgs()
1069  << "SLP: gets ready (def): " << *DepBundle << "\n");
1070  }
1071  });
1072  }
1073  // Handle the memory dependencies.
1074  for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1075  if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1076  // There are no more unscheduled dependencies after decrementing,
1077  // so we can put the dependent instruction into the ready list.
1078  ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1079  assert(!DepBundle->IsScheduled &&
1080  "already scheduled bundle gets ready");
1081  ReadyList.insert(DepBundle);
1082  DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle
1083  << "\n");
1084  }
1085  }
1086  BundleMember = BundleMember->NextInBundle;
1087  }
1088  }
1089 
1090  void doForAllOpcodes(Value *V,
1091  function_ref<void(ScheduleData *SD)> Action) {
1092  if (ScheduleData *SD = getScheduleData(V))
1093  Action(SD);
1094  auto I = ExtraScheduleDataMap.find(V);
1095  if (I != ExtraScheduleDataMap.end())
1096  for (auto &P : I->second)
1097  if (P.second->SchedulingRegionID == SchedulingRegionID)
1098  Action(P.second);
1099  }
1100 
1101  /// Put all instructions into the ReadyList which are ready for scheduling.
1102  template <typename ReadyListType>
1103  void initialFillReadyList(ReadyListType &ReadyList) {
1104  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1105  doForAllOpcodes(I, [&](ScheduleData *SD) {
1106  if (SD->isSchedulingEntity() && SD->isReady()) {
1107  ReadyList.insert(SD);
1108  DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n");
1109  }
1110  });
1111  }
1112  }
1113 
1114  /// Checks if a bundle of instructions can be scheduled, i.e. has no
1115  /// cyclic dependencies. This is only a dry-run, no instructions are
1116  /// actually moved at this stage.
1117  bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1118 
1119  /// Un-bundles a group of instructions.
1120  void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1121 
1122  /// Allocates schedule data chunk.
1123  ScheduleData *allocateScheduleDataChunks();
1124 
1125  /// Extends the scheduling region so that V is inside the region.
1126  /// \returns true if the region size is within the limit.
1127  bool extendSchedulingRegion(Value *V, Value *OpValue);
1128 
1129  /// Initialize the ScheduleData structures for new instructions in the
1130  /// scheduling region.
1131  void initScheduleData(Instruction *FromI, Instruction *ToI,
1132  ScheduleData *PrevLoadStore,
1133  ScheduleData *NextLoadStore);
1134 
1135  /// Updates the dependency information of a bundle and of all instructions/
1136  /// bundles which depend on the original bundle.
1137  void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1138  BoUpSLP *SLP);
1139 
1140  /// Sets all instruction in the scheduling region to un-scheduled.
1141  void resetSchedule();
1142 
1143  BasicBlock *BB;
1144 
1145  /// Simple memory allocation for ScheduleData.
1146  std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1147 
1148  /// The size of a ScheduleData array in ScheduleDataChunks.
1149  int ChunkSize;
1150 
1151  /// The allocator position in the current chunk, which is the last entry
1152  /// of ScheduleDataChunks.
1153  int ChunkPos;
1154 
1155  /// Attaches ScheduleData to Instruction.
1156  /// Note that the mapping survives during all vectorization iterations, i.e.
1157  /// ScheduleData structures are recycled.
1158  DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1159 
1160  /// Attaches ScheduleData to Instruction with the leading key.
1162  ExtraScheduleDataMap;
1163 
1164  struct ReadyList : SmallVector<ScheduleData *, 8> {
1165  void insert(ScheduleData *SD) { push_back(SD); }
1166  };
1167 
1168  /// The ready-list for scheduling (only used for the dry-run).
1169  ReadyList ReadyInsts;
1170 
1171  /// The first instruction of the scheduling region.
1172  Instruction *ScheduleStart = nullptr;
1173 
1174  /// The first instruction _after_ the scheduling region.
1175  Instruction *ScheduleEnd = nullptr;
1176 
1177  /// The first memory accessing instruction in the scheduling region
1178  /// (can be null).
1179  ScheduleData *FirstLoadStoreInRegion = nullptr;
1180 
1181  /// The last memory accessing instruction in the scheduling region
1182  /// (can be null).
1183  ScheduleData *LastLoadStoreInRegion = nullptr;
1184 
1185  /// The current size of the scheduling region.
1186  int ScheduleRegionSize = 0;
1187 
1188  /// The maximum size allowed for the scheduling region.
1189  int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1190 
1191  /// The ID of the scheduling region. For a new vectorization iteration this
1192  /// is incremented which "removes" all ScheduleData from the region.
1193  // Make sure that the initial SchedulingRegionID is greater than the
1194  // initial SchedulingRegionID in ScheduleData (which is 0).
1195  int SchedulingRegionID = 1;
1196  };
1197 
1198  /// Attaches the BlockScheduling structures to basic blocks.
1200 
1201  /// Performs the "real" scheduling. Done before vectorization is actually
1202  /// performed in a basic block.
1203  void scheduleBlock(BlockScheduling *BS);
1204 
1205  /// List of users to ignore during scheduling and that don't need extracting.
1206  ArrayRef<Value *> UserIgnoreList;
1207 
1208  /// Number of operation bundles that contain consecutive operations - number
1209  /// of operation bundles that contain consecutive operations in reversed
1210  /// order.
1211  DenseMap<unsigned, int> NumOpsWantToKeepOrder;
1212 
1213  // Analysis and block reference.
1214  Function *F;
1215  ScalarEvolution *SE;
1216  TargetTransformInfo *TTI;
1217  TargetLibraryInfo *TLI;
1218  AliasAnalysis *AA;
1219  LoopInfo *LI;
1220  DominatorTree *DT;
1221  AssumptionCache *AC;
1222  DemandedBits *DB;
1223  const DataLayout *DL;
1225 
1226  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1227  unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1228 
1229  /// Instruction builder to construct the vectorized tree.
1230  IRBuilder<> Builder;
1231 
1232  /// A map of scalar integer values to the smallest bit width with which they
1233  /// can legally be represented. The values map to (width, signed) pairs,
1234  /// where "width" indicates the minimum bit width and "signed" is True if the
1235  /// value must be signed-extended, rather than zero-extended, back to its
1236  /// original width.
1238 };
1239 
1240 } // end namespace slpvectorizer
1241 
1242 template <> struct GraphTraits<BoUpSLP *> {
1243  using TreeEntry = BoUpSLP::TreeEntry;
1244 
1245  /// NodeRef has to be a pointer per the GraphWriter.
1246  using NodeRef = TreeEntry *;
1247 
1248  /// \brief Add the VectorizableTree to the index iterator to be able to return
1249  /// TreeEntry pointers.
1250  struct ChildIteratorType
1251  : public iterator_adaptor_base<ChildIteratorType,
1252  SmallVector<int, 1>::iterator> {
1253  std::vector<TreeEntry> &VectorizableTree;
1254 
1256  std::vector<TreeEntry> &VT)
1257  : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1258 
1259  NodeRef operator*() { return &VectorizableTree[*I]; }
1260  };
1261 
1262  static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1263 
1264  static ChildIteratorType child_begin(NodeRef N) {
1265  return {N->UserTreeIndices.begin(), N->Container};
1266  }
1267 
1268  static ChildIteratorType child_end(NodeRef N) {
1269  return {N->UserTreeIndices.end(), N->Container};
1270  }
1271 
1272  /// For the node iterator we just need to turn the TreeEntry iterator into a
1273  /// TreeEntry* iterator so that it dereferences to NodeRef.
1275 
1276  static nodes_iterator nodes_begin(BoUpSLP *R) {
1277  return nodes_iterator(R->VectorizableTree.begin());
1278  }
1279 
1280  static nodes_iterator nodes_end(BoUpSLP *R) {
1281  return nodes_iterator(R->VectorizableTree.end());
1282  }
1283 
1284  static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1285 };
1286 
1287 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1288  using TreeEntry = BoUpSLP::TreeEntry;
1289 
1291 
1292  std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1293  std::string Str;
1294  raw_string_ostream OS(Str);
1295  if (isSplat(Entry->Scalars)) {
1296  OS << "<splat> " << *Entry->Scalars[0];
1297  return Str;
1298  }
1299  for (auto V : Entry->Scalars) {
1300  OS << *V;
1301  if (std::any_of(
1302  R->ExternalUses.begin(), R->ExternalUses.end(),
1303  [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1304  OS << " <extract>";
1305  OS << "\n";
1306  }
1307  return Str;
1308  }
1309 
1310  static std::string getNodeAttributes(const TreeEntry *Entry,
1311  const BoUpSLP *) {
1312  if (Entry->NeedToGather)
1313  return "color=red";
1314  return "";
1315  }
1316 };
1317 
1318 } // end namespace llvm
1319 
1320 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1321  ArrayRef<Value *> UserIgnoreLst) {
1322  ExtraValueToDebugLocsMap ExternallyUsedValues;
1323  buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1324 }
1325 
1326 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1327  ExtraValueToDebugLocsMap &ExternallyUsedValues,
1328  ArrayRef<Value *> UserIgnoreLst) {
1329  deleteTree();
1330  UserIgnoreList = UserIgnoreLst;
1331  if (!allSameType(Roots))
1332  return;
1333  buildTree_rec(Roots, 0, -1);
1334 
1335  // Collect the values that we need to extract from the tree.
1336  for (TreeEntry &EIdx : VectorizableTree) {
1337  TreeEntry *Entry = &EIdx;
1338 
1339  // No need to handle users of gathered values.
1340  if (Entry->NeedToGather)
1341  continue;
1342 
1343  // For each lane:
1344  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1345  Value *Scalar = Entry->Scalars[Lane];
1346 
1347  // Check if the scalar is externally used as an extra arg.
1348  auto ExtI = ExternallyUsedValues.find(Scalar);
1349  if (ExtI != ExternallyUsedValues.end()) {
1350  DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<
1351  Lane << " from " << *Scalar << ".\n");
1352  ExternalUses.emplace_back(Scalar, nullptr, Lane);
1353  }
1354  for (User *U : Scalar->users()) {
1355  DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1356 
1357  Instruction *UserInst = dyn_cast<Instruction>(U);
1358  if (!UserInst)
1359  continue;
1360 
1361  // Skip in-tree scalars that become vectors
1362  if (TreeEntry *UseEntry = getTreeEntry(U)) {
1363  Value *UseScalar = UseEntry->Scalars[0];
1364  // Some in-tree scalars will remain as scalar in vectorized
1365  // instructions. If that is the case, the one in Lane 0 will
1366  // be used.
1367  if (UseScalar != U ||
1368  !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1369  DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1370  << ".\n");
1371  assert(!UseEntry->NeedToGather && "Bad state");
1372  continue;
1373  }
1374  }
1375 
1376  // Ignore users in the user ignore list.
1377  if (is_contained(UserIgnoreList, UserInst))
1378  continue;
1379 
1380  DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<
1381  Lane << " from " << *Scalar << ".\n");
1382  ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1383  }
1384  }
1385  }
1386 }
1387 
1388 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1389  int UserTreeIdx) {
1390  assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1391 
1392  InstructionsState S = getSameOpcode(VL);
1393  if (Depth == RecursionMaxDepth) {
1394  DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1395  newTreeEntry(VL, false, UserTreeIdx);
1396  return;
1397  }
1398 
1399  // Don't handle vectors.
1400  if (S.OpValue->getType()->isVectorTy()) {
1401  DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1402  newTreeEntry(VL, false, UserTreeIdx);
1403  return;
1404  }
1405 
1406  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1407  if (SI->getValueOperand()->getType()->isVectorTy()) {
1408  DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1409  newTreeEntry(VL, false, UserTreeIdx);
1410  return;
1411  }
1412 
1413  // If all of the operands are identical or constant we have a simple solution.
1414  if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1415  DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1416  newTreeEntry(VL, false, UserTreeIdx);
1417  return;
1418  }
1419 
1420  // We now know that this is a vector of instructions of the same type from
1421  // the same block.
1422 
1423  // Don't vectorize ephemeral values.
1424  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1425  if (EphValues.count(VL[i])) {
1426  DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1427  ") is ephemeral.\n");
1428  newTreeEntry(VL, false, UserTreeIdx);
1429  return;
1430  }
1431  }
1432 
1433  // Check if this is a duplicate of another entry.
1434  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1435  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1436  DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n");
1437  if (E->Scalars[i] != VL[i]) {
1438  DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1439  newTreeEntry(VL, false, UserTreeIdx);
1440  return;
1441  }
1442  }
1443  // Record the reuse of the tree node. FIXME, currently this is only used to
1444  // properly draw the graph rather than for the actual vectorization.
1445  E->UserTreeIndices.push_back(UserTreeIdx);
1446  DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n");
1447  return;
1448  }
1449 
1450  // Check that none of the instructions in the bundle are already in the tree.
1451  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1452  auto *I = dyn_cast<Instruction>(VL[i]);
1453  if (!I)
1454  continue;
1455  if (getTreeEntry(I)) {
1456  DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1457  ") is already in tree.\n");
1458  newTreeEntry(VL, false, UserTreeIdx);
1459  return;
1460  }
1461  }
1462 
1463  // If any of the scalars is marked as a value that needs to stay scalar, then
1464  // we need to gather the scalars.
1465  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1466  if (MustGather.count(VL[i])) {
1467  DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1468  newTreeEntry(VL, false, UserTreeIdx);
1469  return;
1470  }
1471  }
1472 
1473  // Check that all of the users of the scalars that we want to vectorize are
1474  // schedulable.
1475  auto *VL0 = cast<Instruction>(S.OpValue);
1476  BasicBlock *BB = VL0->getParent();
1477 
1478  if (!DT->isReachableFromEntry(BB)) {
1479  // Don't go into unreachable blocks. They may contain instructions with
1480  // dependency cycles which confuse the final scheduling.
1481  DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1482  newTreeEntry(VL, false, UserTreeIdx);
1483  return;
1484  }
1485 
1486  // Check that every instruction appears once in this bundle.
1487  for (unsigned i = 0, e = VL.size(); i < e; ++i)
1488  for (unsigned j = i + 1; j < e; ++j)
1489  if (VL[i] == VL[j]) {
1490  DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1491  newTreeEntry(VL, false, UserTreeIdx);
1492  return;
1493  }
1494 
1495  auto &BSRef = BlocksSchedules[BB];
1496  if (!BSRef)
1497  BSRef = llvm::make_unique<BlockScheduling>(BB);
1498 
1499  BlockScheduling &BS = *BSRef.get();
1500 
1501  if (!BS.tryScheduleBundle(VL, this, S.OpValue)) {
1502  DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1503  assert((!BS.getScheduleData(VL0) ||
1504  !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1505  "tryScheduleBundle should cancelScheduling on failure");
1506  newTreeEntry(VL, false, UserTreeIdx);
1507  return;
1508  }
1509  DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1510 
1511  unsigned ShuffleOrOp = S.IsAltShuffle ?
1512  (unsigned) Instruction::ShuffleVector : S.Opcode;
1513  switch (ShuffleOrOp) {
1514  case Instruction::PHI: {
1515  PHINode *PH = dyn_cast<PHINode>(VL0);
1516 
1517  // Check for terminator values (e.g. invoke).
1518  for (unsigned j = 0; j < VL.size(); ++j)
1519  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1521  cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1522  if (Term) {
1523  DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n");
1524  BS.cancelScheduling(VL, VL0);
1525  newTreeEntry(VL, false, UserTreeIdx);
1526  return;
1527  }
1528  }
1529 
1530  newTreeEntry(VL, true, UserTreeIdx);
1531  DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1532 
1533  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1534  ValueList Operands;
1535  // Prepare the operand vector.
1536  for (Value *j : VL)
1537  Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1538  PH->getIncomingBlock(i)));
1539 
1540  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1541  }
1542  return;
1543  }
1544  case Instruction::ExtractValue:
1545  case Instruction::ExtractElement: {
1546  bool Reuse = canReuseExtract(VL, VL0);
1547  if (Reuse) {
1548  DEBUG(dbgs() << "SLP: Reusing extract sequence.\n");
1549  ++NumOpsWantToKeepOrder[S.Opcode];
1550  } else {
1551  SmallVector<Value *, 4> ReverseVL(VL.rbegin(), VL.rend());
1552  if (canReuseExtract(ReverseVL, VL0))
1553  --NumOpsWantToKeepOrder[S.Opcode];
1554  BS.cancelScheduling(VL, VL0);
1555  }
1556  newTreeEntry(VL, Reuse, UserTreeIdx);
1557  return;
1558  }
1559  case Instruction::Load: {
1560  // Check that a vectorized load would load the same memory as a scalar
1561  // load. For example, we don't want to vectorize loads that are smaller
1562  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1563  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1564  // from such a struct, we read/write packed bits disagreeing with the
1565  // unvectorized version.
1566  Type *ScalarTy = VL0->getType();
1567 
1568  if (DL->getTypeSizeInBits(ScalarTy) !=
1569  DL->getTypeAllocSizeInBits(ScalarTy)) {
1570  BS.cancelScheduling(VL, VL0);
1571  newTreeEntry(VL, false, UserTreeIdx);
1572  DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1573  return;
1574  }
1575 
1576  // Make sure all loads in the bundle are simple - we can't vectorize
1577  // atomic or volatile loads.
1578  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1579  LoadInst *L = cast<LoadInst>(VL[i]);
1580  if (!L->isSimple()) {
1581  BS.cancelScheduling(VL, VL0);
1582  newTreeEntry(VL, false, UserTreeIdx);
1583  DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1584  return;
1585  }
1586  }
1587 
1588  // Check if the loads are consecutive, reversed, or neither.
1589  // TODO: What we really want is to sort the loads, but for now, check
1590  // the two likely directions.
1591  bool Consecutive = true;
1592  bool ReverseConsecutive = true;
1593  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1594  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1595  Consecutive = false;
1596  break;
1597  } else {
1598  ReverseConsecutive = false;
1599  }
1600  }
1601 
1602  if (Consecutive) {
1603  ++NumOpsWantToKeepOrder[S.Opcode];
1604  newTreeEntry(VL, true, UserTreeIdx);
1605  DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1606  return;
1607  }
1608 
1609  // If none of the load pairs were consecutive when checked in order,
1610  // check the reverse order.
1611  if (ReverseConsecutive)
1612  for (unsigned i = VL.size() - 1; i > 0; --i)
1613  if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1614  ReverseConsecutive = false;
1615  break;
1616  }
1617 
1618  BS.cancelScheduling(VL, VL0);
1619  newTreeEntry(VL, false, UserTreeIdx);
1620 
1621  if (ReverseConsecutive) {
1622  --NumOpsWantToKeepOrder[S.Opcode];
1623  DEBUG(dbgs() << "SLP: Gathering reversed loads.\n");
1624  } else {
1625  DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1626  }
1627  return;
1628  }
1629  case Instruction::ZExt:
1630  case Instruction::SExt:
1631  case Instruction::FPToUI:
1632  case Instruction::FPToSI:
1633  case Instruction::FPExt:
1634  case Instruction::PtrToInt:
1635  case Instruction::IntToPtr:
1636  case Instruction::SIToFP:
1637  case Instruction::UIToFP:
1638  case Instruction::Trunc:
1639  case Instruction::FPTrunc:
1640  case Instruction::BitCast: {
1641  Type *SrcTy = VL0->getOperand(0)->getType();
1642  for (unsigned i = 0; i < VL.size(); ++i) {
1643  Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1644  if (Ty != SrcTy || !isValidElementType(Ty)) {
1645  BS.cancelScheduling(VL, VL0);
1646  newTreeEntry(VL, false, UserTreeIdx);
1647  DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n");
1648  return;
1649  }
1650  }
1651  newTreeEntry(VL, true, UserTreeIdx);
1652  DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1653 
1654  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1655  ValueList Operands;
1656  // Prepare the operand vector.
1657  for (Value *j : VL)
1658  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1659 
1660  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1661  }
1662  return;
1663  }
1664  case Instruction::ICmp:
1665  case Instruction::FCmp: {
1666  // Check that all of the compares have the same predicate.
1667  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1668  Type *ComparedTy = VL0->getOperand(0)->getType();
1669  for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1670  CmpInst *Cmp = cast<CmpInst>(VL[i]);
1671  if (Cmp->getPredicate() != P0 ||
1672  Cmp->getOperand(0)->getType() != ComparedTy) {
1673  BS.cancelScheduling(VL, VL0);
1674  newTreeEntry(VL, false, UserTreeIdx);
1675  DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
1676  return;
1677  }
1678  }
1679 
1680  newTreeEntry(VL, true, UserTreeIdx);
1681  DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1682 
1683  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1684  ValueList Operands;
1685  // Prepare the operand vector.
1686  for (Value *j : VL)
1687  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1688 
1689  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1690  }
1691  return;
1692  }
1693  case Instruction::Select:
1694  case Instruction::Add:
1695  case Instruction::FAdd:
1696  case Instruction::Sub:
1697  case Instruction::FSub:
1698  case Instruction::Mul:
1699  case Instruction::FMul:
1700  case Instruction::UDiv:
1701  case Instruction::SDiv:
1702  case Instruction::FDiv:
1703  case Instruction::URem:
1704  case Instruction::SRem:
1705  case Instruction::FRem:
1706  case Instruction::Shl:
1707  case Instruction::LShr:
1708  case Instruction::AShr:
1709  case Instruction::And:
1710  case Instruction::Or:
1711  case Instruction::Xor:
1712  newTreeEntry(VL, true, UserTreeIdx);
1713  DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1714 
1715  // Sort operands of the instructions so that each side is more likely to
1716  // have the same opcode.
1717  if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1718  ValueList Left, Right;
1719  reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1720  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1721  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1722  return;
1723  }
1724 
1725  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1726  ValueList Operands;
1727  // Prepare the operand vector.
1728  for (Value *j : VL)
1729  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1730 
1731  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1732  }
1733  return;
1734 
1735  case Instruction::GetElementPtr: {
1736  // We don't combine GEPs with complicated (nested) indexing.
1737  for (unsigned j = 0; j < VL.size(); ++j) {
1738  if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1739  DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1740  BS.cancelScheduling(VL, VL0);
1741  newTreeEntry(VL, false, UserTreeIdx);
1742  return;
1743  }
1744  }
1745 
1746  // We can't combine several GEPs into one vector if they operate on
1747  // different types.
1748  Type *Ty0 = VL0->getOperand(0)->getType();
1749  for (unsigned j = 0; j < VL.size(); ++j) {
1750  Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1751  if (Ty0 != CurTy) {
1752  DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
1753  BS.cancelScheduling(VL, VL0);
1754  newTreeEntry(VL, false, UserTreeIdx);
1755  return;
1756  }
1757  }
1758 
1759  // We don't combine GEPs with non-constant indexes.
1760  for (unsigned j = 0; j < VL.size(); ++j) {
1761  auto Op = cast<Instruction>(VL[j])->getOperand(1);
1762  if (!isa<ConstantInt>(Op)) {
1763  DEBUG(
1764  dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1765  BS.cancelScheduling(VL, VL0);
1766  newTreeEntry(VL, false, UserTreeIdx);
1767  return;
1768  }
1769  }
1770 
1771  newTreeEntry(VL, true, UserTreeIdx);
1772  DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1773  for (unsigned i = 0, e = 2; i < e; ++i) {
1774  ValueList Operands;
1775  // Prepare the operand vector.
1776  for (Value *j : VL)
1777  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1778 
1779  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1780  }
1781  return;
1782  }
1783  case Instruction::Store: {
1784  // Check if the stores are consecutive or of we need to swizzle them.
1785  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1786  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1787  BS.cancelScheduling(VL, VL0);
1788  newTreeEntry(VL, false, UserTreeIdx);
1789  DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1790  return;
1791  }
1792 
1793  newTreeEntry(VL, true, UserTreeIdx);
1794  DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1795 
1796  ValueList Operands;
1797  for (Value *j : VL)
1798  Operands.push_back(cast<Instruction>(j)->getOperand(0));
1799 
1800  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1801  return;
1802  }
1803  case Instruction::Call: {
1804  // Check if the calls are all to the same vectorizable intrinsic.
1805  CallInst *CI = cast<CallInst>(VL0);
1806  // Check if this is an Intrinsic call or something that can be
1807  // represented by an intrinsic call
1809  if (!isTriviallyVectorizable(ID)) {
1810  BS.cancelScheduling(VL, VL0);
1811  newTreeEntry(VL, false, UserTreeIdx);
1812  DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1813  return;
1814  }
1815  Function *Int = CI->getCalledFunction();
1816  Value *A1I = nullptr;
1817  if (hasVectorInstrinsicScalarOpd(ID, 1))
1818  A1I = CI->getArgOperand(1);
1819  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1820  CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1821  if (!CI2 || CI2->getCalledFunction() != Int ||
1822  getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1823  !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1824  BS.cancelScheduling(VL, VL0);
1825  newTreeEntry(VL, false, UserTreeIdx);
1826  DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1827  << "\n");
1828  return;
1829  }
1830  // ctlz,cttz and powi are special intrinsics whose second argument
1831  // should be same in order for them to be vectorized.
1832  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1833  Value *A1J = CI2->getArgOperand(1);
1834  if (A1I != A1J) {
1835  BS.cancelScheduling(VL, VL0);
1836  newTreeEntry(VL, false, UserTreeIdx);
1837  DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1838  << " argument "<< A1I<<"!=" << A1J
1839  << "\n");
1840  return;
1841  }
1842  }
1843  // Verify that the bundle operands are identical between the two calls.
1844  if (CI->hasOperandBundles() &&
1846  CI->op_begin() + CI->getBundleOperandsEndIndex(),
1847  CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1848  BS.cancelScheduling(VL, VL0);
1849  newTreeEntry(VL, false, UserTreeIdx);
1850  DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="
1851  << *VL[i] << '\n');
1852  return;
1853  }
1854  }
1855 
1856  newTreeEntry(VL, true, UserTreeIdx);
1857  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1858  ValueList Operands;
1859  // Prepare the operand vector.
1860  for (Value *j : VL) {
1861  CallInst *CI2 = dyn_cast<CallInst>(j);
1862  Operands.push_back(CI2->getArgOperand(i));
1863  }
1864  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1865  }
1866  return;
1867  }
1868  case Instruction::ShuffleVector:
1869  // If this is not an alternate sequence of opcode like add-sub
1870  // then do not vectorize this instruction.
1871  if (!S.IsAltShuffle) {
1872  BS.cancelScheduling(VL, VL0);
1873  newTreeEntry(VL, false, UserTreeIdx);
1874  DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1875  return;
1876  }
1877  newTreeEntry(VL, true, UserTreeIdx);
1878  DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1879 
1880  // Reorder operands if reordering would enable vectorization.
1881  if (isa<BinaryOperator>(VL0)) {
1882  ValueList Left, Right;
1883  reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1884  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1885  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1886  return;
1887  }
1888 
1889  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1890  ValueList Operands;
1891  // Prepare the operand vector.
1892  for (Value *j : VL)
1893  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1894 
1895  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1896  }
1897  return;
1898 
1899  default:
1900  BS.cancelScheduling(VL, VL0);
1901  newTreeEntry(VL, false, UserTreeIdx);
1902  DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1903  return;
1904  }
1905 }
1906 
1907 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1908  unsigned N;
1909  Type *EltTy;
1910  auto *ST = dyn_cast<StructType>(T);
1911  if (ST) {
1912  N = ST->getNumElements();
1913  EltTy = *ST->element_begin();
1914  } else {
1915  N = cast<ArrayType>(T)->getNumElements();
1916  EltTy = cast<ArrayType>(T)->getElementType();
1917  }
1918  if (!isValidElementType(EltTy))
1919  return 0;
1920  uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1921  if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1922  return 0;
1923  if (ST) {
1924  // Check that struct is homogeneous.
1925  for (const auto *Ty : ST->elements())
1926  if (Ty != EltTy)
1927  return 0;
1928  }
1929  return N;
1930 }
1931 
1932 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const {
1933  Instruction *E0 = cast<Instruction>(OpValue);
1934  assert(E0->getOpcode() == Instruction::ExtractElement ||
1935  E0->getOpcode() == Instruction::ExtractValue);
1936  assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode");
1937  // Check if all of the extracts come from the same vector and from the
1938  // correct offset.
1939  Value *Vec = E0->getOperand(0);
1940 
1941  // We have to extract from a vector/aggregate with the same number of elements.
1942  unsigned NElts;
1943  if (E0->getOpcode() == Instruction::ExtractValue) {
1944  const DataLayout &DL = E0->getModule()->getDataLayout();
1945  NElts = canMapToVector(Vec->getType(), DL);
1946  if (!NElts)
1947  return false;
1948  // Check if load can be rewritten as load of vector.
1949  LoadInst *LI = dyn_cast<LoadInst>(Vec);
1950  if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1951  return false;
1952  } else {
1953  NElts = Vec->getType()->getVectorNumElements();
1954  }
1955 
1956  if (NElts != VL.size())
1957  return false;
1958 
1959  // Check that all of the indices extract from the correct offset.
1960  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
1961  Instruction *Inst = cast<Instruction>(VL[I]);
1962  if (!matchExtractIndex(Inst, I, Inst->getOpcode()))
1963  return false;
1964  if (Inst->getOperand(0) != Vec)
1965  return false;
1966  }
1967 
1968  return true;
1969 }
1970 
1971 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
1972  return I->hasOneUse() ||
1973  std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
1974  return ScalarToTreeEntry.count(U) > 0;
1975  });
1976 }
1977 
1978 int BoUpSLP::getEntryCost(TreeEntry *E) {
1979  ArrayRef<Value*> VL = E->Scalars;
1980 
1981  Type *ScalarTy = VL[0]->getType();
1982  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1983  ScalarTy = SI->getValueOperand()->getType();
1984  else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
1985  ScalarTy = CI->getOperand(0)->getType();
1986  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1987 
1988  // If we have computed a smaller type for the expression, update VecTy so
1989  // that the costs will be accurate.
1990  if (MinBWs.count(VL[0]))
1991  VecTy = VectorType::get(
1992  IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1993 
1994  if (E->NeedToGather) {
1995  if (allConstant(VL))
1996  return 0;
1997  if (isSplat(VL)) {
1998  return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1999  }
2000  if (getSameOpcode(VL).Opcode == Instruction::ExtractElement) {
2002  if (ShuffleKind.hasValue()) {
2003  int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
2004  for (auto *V : VL) {
2005  // If all users of instruction are going to be vectorized and this
2006  // instruction itself is not going to be vectorized, consider this
2007  // instruction as dead and remove its cost from the final cost of the
2008  // vectorized tree.
2009  if (areAllUsersVectorized(cast<Instruction>(V)) &&
2010  !ScalarToTreeEntry.count(V)) {
2011  auto *IO = cast<ConstantInt>(
2012  cast<ExtractElementInst>(V)->getIndexOperand());
2013  Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2014  IO->getZExtValue());
2015  }
2016  }
2017  return Cost;
2018  }
2019  }
2020  return getGatherCost(E->Scalars);
2021  }
2022  InstructionsState S = getSameOpcode(VL);
2023  assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2024  Instruction *VL0 = cast<Instruction>(S.OpValue);
2025  unsigned ShuffleOrOp = S.IsAltShuffle ?
2026  (unsigned) Instruction::ShuffleVector : S.Opcode;
2027  switch (ShuffleOrOp) {
2028  case Instruction::PHI:
2029  return 0;
2030 
2031  case Instruction::ExtractValue:
2032  case Instruction::ExtractElement:
2033  if (canReuseExtract(VL, S.OpValue)) {
2034  int DeadCost = 0;
2035  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2036  Instruction *E = cast<Instruction>(VL[i]);
2037  // If all users are going to be vectorized, instruction can be
2038  // considered as dead.
2039  // The same, if have only one user, it will be vectorized for sure.
2040  if (areAllUsersVectorized(E))
2041  // Take credit for instruction that will become dead.
2042  DeadCost +=
2043  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2044  }
2045  return -DeadCost;
2046  }
2047  return getGatherCost(VecTy);
2048 
2049  case Instruction::ZExt:
2050  case Instruction::SExt:
2051  case Instruction::FPToUI:
2052  case Instruction::FPToSI:
2053  case Instruction::FPExt:
2054  case Instruction::PtrToInt:
2055  case Instruction::IntToPtr:
2056  case Instruction::SIToFP:
2057  case Instruction::UIToFP:
2058  case Instruction::Trunc:
2059  case Instruction::FPTrunc:
2060  case Instruction::BitCast: {
2061  Type *SrcTy = VL0->getOperand(0)->getType();
2062 
2063  // Calculate the cost of this instruction.
2064  int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2065  VL0->getType(), SrcTy, VL0);
2066 
2067  VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2068  int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2069  return VecCost - ScalarCost;
2070  }
2071  case Instruction::FCmp:
2072  case Instruction::ICmp:
2073  case Instruction::Select: {
2074  // Calculate the cost of this instruction.
2075  VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2076  int ScalarCost = VecTy->getNumElements() *
2077  TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2078  int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2079  return VecCost - ScalarCost;
2080  }
2081  case Instruction::Add:
2082  case Instruction::FAdd:
2083  case Instruction::Sub:
2084  case Instruction::FSub:
2085  case Instruction::Mul:
2086  case Instruction::FMul:
2087  case Instruction::UDiv:
2088  case Instruction::SDiv:
2089  case Instruction::FDiv:
2090  case Instruction::URem:
2091  case Instruction::SRem:
2092  case Instruction::FRem:
2093  case Instruction::Shl:
2094  case Instruction::LShr:
2095  case Instruction::AShr:
2096  case Instruction::And:
2097  case Instruction::Or:
2098  case Instruction::Xor: {
2099  // Certain instructions can be cheaper to vectorize if they have a
2100  // constant second vector operand.
2109 
2110  // If all operands are exactly the same ConstantInt then set the
2111  // operand kind to OK_UniformConstantValue.
2112  // If instead not all operands are constants, then set the operand kind
2113  // to OK_AnyValue. If all operands are constants but not the same,
2114  // then set the operand kind to OK_NonUniformConstantValue.
2115  ConstantInt *CInt = nullptr;
2116  for (unsigned i = 0; i < VL.size(); ++i) {
2117  const Instruction *I = cast<Instruction>(VL[i]);
2118  if (!isa<ConstantInt>(I->getOperand(1))) {
2120  break;
2121  }
2122  if (i == 0) {
2123  CInt = cast<ConstantInt>(I->getOperand(1));
2124  continue;
2125  }
2127  CInt != cast<ConstantInt>(I->getOperand(1)))
2129  }
2130  // FIXME: Currently cost of model modification for division by power of
2131  // 2 is handled for X86 and AArch64. Add support for other targets.
2132  if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2133  CInt->getValue().isPowerOf2())
2135 
2137  int ScalarCost =
2138  VecTy->getNumElements() *
2139  TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2140  Op2VP, Operands);
2141  int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2142  Op1VP, Op2VP, Operands);
2143  return VecCost - ScalarCost;
2144  }
2145  case Instruction::GetElementPtr: {
2150 
2151  int ScalarCost =
2152  VecTy->getNumElements() *
2153  TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2154  int VecCost =
2155  TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2156 
2157  return VecCost - ScalarCost;
2158  }
2159  case Instruction::Load: {
2160  // Cost of wide load - cost of scalar loads.
2161  unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2162  int ScalarLdCost = VecTy->getNumElements() *
2163  TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2164  int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2165  VecTy, alignment, 0, VL0);
2166  return VecLdCost - ScalarLdCost;
2167  }
2168  case Instruction::Store: {
2169  // We know that we can merge the stores. Calculate the cost.
2170  unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2171  int ScalarStCost = VecTy->getNumElements() *
2172  TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2173  int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2174  VecTy, alignment, 0, VL0);
2175  return VecStCost - ScalarStCost;
2176  }
2177  case Instruction::Call: {
2178  CallInst *CI = cast<CallInst>(VL0);
2180 
2181  // Calculate the cost of the scalar and vector calls.
2182  SmallVector<Type*, 4> ScalarTys;
2183  for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2184  ScalarTys.push_back(CI->getArgOperand(op)->getType());
2185 
2186  FastMathFlags FMF;
2187  if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2188  FMF = FPMO->getFastMathFlags();
2189 
2190  int ScalarCallCost = VecTy->getNumElements() *
2191  TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2192 
2194  int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2195  VecTy->getNumElements());
2196 
2197  DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCost
2198  << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2199  << " for " << *CI << "\n");
2200 
2201  return VecCallCost - ScalarCallCost;
2202  }
2203  case Instruction::ShuffleVector: {
2208  int ScalarCost = 0;
2209  int VecCost = 0;
2210  for (Value *i : VL) {
2211  Instruction *I = cast<Instruction>(i);
2212  if (!I)
2213  break;
2214  ScalarCost +=
2215  TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2216  }
2217  // VecCost is equal to sum of the cost of creating 2 vectors
2218  // and the cost of creating shuffle.
2219  Instruction *I0 = cast<Instruction>(VL[0]);
2220  VecCost =
2221  TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2222  Instruction *I1 = cast<Instruction>(VL[1]);
2223  VecCost +=
2224  TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2225  VecCost +=
2226  TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2227  return VecCost - ScalarCost;
2228  }
2229  default:
2230  llvm_unreachable("Unknown instruction");
2231  }
2232 }
2233 
2234 bool BoUpSLP::isFullyVectorizableTinyTree() {
2235  DEBUG(dbgs() << "SLP: Check whether the tree with height " <<
2236  VectorizableTree.size() << " is fully vectorizable .\n");
2237 
2238  // We only handle trees of heights 1 and 2.
2239  if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2240  return true;
2241 
2242  if (VectorizableTree.size() != 2)
2243  return false;
2244 
2245  // Handle splat and all-constants stores.
2246  if (!VectorizableTree[0].NeedToGather &&
2247  (allConstant(VectorizableTree[1].Scalars) ||
2248  isSplat(VectorizableTree[1].Scalars)))
2249  return true;
2250 
2251  // Gathering cost would be too much for tiny trees.
2252  if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2253  return false;
2254 
2255  return true;
2256 }
2257 
2258 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2259  // We can vectorize the tree if its size is greater than or equal to the
2260  // minimum size specified by the MinTreeSize command line option.
2261  if (VectorizableTree.size() >= MinTreeSize)
2262  return false;
2263 
2264  // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2265  // can vectorize it if we can prove it fully vectorizable.
2266  if (isFullyVectorizableTinyTree())
2267  return false;
2268 
2269  assert(VectorizableTree.empty()
2270  ? ExternalUses.empty()
2271  : true && "We shouldn't have any external users");
2272 
2273  // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2274  // vectorizable.
2275  return true;
2276 }
2277 
2278 int BoUpSLP::getSpillCost() {
2279  // Walk from the bottom of the tree to the top, tracking which values are
2280  // live. When we see a call instruction that is not part of our tree,
2281  // query TTI to see if there is a cost to keeping values live over it
2282  // (for example, if spills and fills are required).
2283  unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2284  int Cost = 0;
2285 
2286  SmallPtrSet<Instruction*, 4> LiveValues;
2287  Instruction *PrevInst = nullptr;
2288 
2289  for (const auto &N : VectorizableTree) {
2290  Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2291  if (!Inst)
2292  continue;
2293 
2294  if (!PrevInst) {
2295  PrevInst = Inst;
2296  continue;
2297  }
2298 
2299  // Update LiveValues.
2300  LiveValues.erase(PrevInst);
2301  for (auto &J : PrevInst->operands()) {
2302  if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2303  LiveValues.insert(cast<Instruction>(&*J));
2304  }
2305 
2306  DEBUG(
2307  dbgs() << "SLP: #LV: " << LiveValues.size();
2308  for (auto *X : LiveValues)
2309  dbgs() << " " << X->getName();
2310  dbgs() << ", Looking at ";
2311  Inst->dump();
2312  );
2313 
2314  // Now find the sequence of instructions between PrevInst and Inst.
2316  PrevInstIt =
2317  PrevInst->getIterator().getReverse();
2318  while (InstIt != PrevInstIt) {
2319  if (PrevInstIt == PrevInst->getParent()->rend()) {
2320  PrevInstIt = Inst->getParent()->rbegin();
2321  continue;
2322  }
2323 
2324  if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2326  for (auto *II : LiveValues)
2327  V.push_back(VectorType::get(II->getType(), BundleWidth));
2328  Cost += TTI->getCostOfKeepingLiveOverCall(V);
2329  }
2330 
2331  ++PrevInstIt;
2332  }
2333 
2334  PrevInst = Inst;
2335  }
2336 
2337  return Cost;
2338 }
2339 
2340 int BoUpSLP::getTreeCost() {
2341  int Cost = 0;
2342  DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<
2343  VectorizableTree.size() << ".\n");
2344 
2345  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2346 
2347  for (TreeEntry &TE : VectorizableTree) {
2348  int C = getEntryCost(&TE);
2349  DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "
2350  << *TE.Scalars[0] << ".\n");
2351  Cost += C;
2352  }
2353 
2354  SmallSet<Value *, 16> ExtractCostCalculated;
2355  int ExtractCost = 0;
2356  for (ExternalUser &EU : ExternalUses) {
2357  // We only add extract cost once for the same scalar.
2358  if (!ExtractCostCalculated.insert(EU.Scalar).second)
2359  continue;
2360 
2361  // Uses by ephemeral values are free (because the ephemeral value will be
2362  // removed prior to code generation, and so the extraction will be
2363  // removed as well).
2364  if (EphValues.count(EU.User))
2365  continue;
2366 
2367  // If we plan to rewrite the tree in a smaller type, we will need to sign
2368  // extend the extracted value back to the original type. Here, we account
2369  // for the extract and the added cost of the sign extend if needed.
2370  auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2371  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2372  if (MinBWs.count(ScalarRoot)) {
2373  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2374  auto Extend =
2375  MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2376  VecTy = VectorType::get(MinTy, BundleWidth);
2377  ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2378  VecTy, EU.Lane);
2379  } else {
2380  ExtractCost +=
2381  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2382  }
2383  }
2384 
2385  int SpillCost = getSpillCost();
2386  Cost += SpillCost + ExtractCost;
2387 
2388  std::string Str;
2389  {
2390  raw_string_ostream OS(Str);
2391  OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2392  << "SLP: Extract Cost = " << ExtractCost << ".\n"
2393  << "SLP: Total Cost = " << Cost << ".\n";
2394  }
2395  DEBUG(dbgs() << Str);
2396 
2397  if (ViewSLPTree)
2398  ViewGraph(this, "SLP" + F->getName(), false, Str);
2399 
2400  return Cost;
2401 }
2402 
2403 int BoUpSLP::getGatherCost(Type *Ty) {
2404  int Cost = 0;
2405  for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2406  Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2407  return Cost;
2408 }
2409 
2410 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2411  // Find the type of the operands in VL.
2412  Type *ScalarTy = VL[0]->getType();
2413  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2414  ScalarTy = SI->getValueOperand()->getType();
2415  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2416  // Find the cost of inserting/extracting values from the vector.
2417  return getGatherCost(VecTy);
2418 }
2419 
2420 // Reorder commutative operations in alternate shuffle if the resulting vectors
2421 // are consecutive loads. This would allow us to vectorize the tree.
2422 // If we have something like-
2423 // load a[0] - load b[0]
2424 // load b[1] + load a[1]
2425 // load a[2] - load b[2]
2426 // load a[3] + load b[3]
2427 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2428 // code.
2429 void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2432  // Push left and right operands of binary operation into Left and Right
2433  unsigned AltOpcode = getAltOpcode(Opcode);
2434  (void)AltOpcode;
2435  for (Value *V : VL) {
2436  auto *I = cast<Instruction>(V);
2437  assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&
2438  "Incorrect instruction in vector");
2439  Left.push_back(I->getOperand(0));
2440  Right.push_back(I->getOperand(1));
2441  }
2442 
2443  // Reorder if we have a commutative operation and consecutive access
2444  // are on either side of the alternate instructions.
2445  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2446  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2447  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2448  Instruction *VL1 = cast<Instruction>(VL[j]);
2449  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2450  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2451  std::swap(Left[j], Right[j]);
2452  continue;
2453  } else if (VL2->isCommutative() &&
2454  isConsecutiveAccess(L, L1, *DL, *SE)) {
2455  std::swap(Left[j + 1], Right[j + 1]);
2456  continue;
2457  }
2458  // else unchanged
2459  }
2460  }
2461  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2462  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2463  Instruction *VL1 = cast<Instruction>(VL[j]);
2464  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2465  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2466  std::swap(Left[j], Right[j]);
2467  continue;
2468  } else if (VL2->isCommutative() &&
2469  isConsecutiveAccess(L, L1, *DL, *SE)) {
2470  std::swap(Left[j + 1], Right[j + 1]);
2471  continue;
2472  }
2473  // else unchanged
2474  }
2475  }
2476  }
2477 }
2478 
2479 // Return true if I should be commuted before adding it's left and right
2480 // operands to the arrays Left and Right.
2481 //
2482 // The vectorizer is trying to either have all elements one side being
2483 // instruction with the same opcode to enable further vectorization, or having
2484 // a splat to lower the vectorizing cost.
2486  int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2487  ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2488  bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2489  VLeft = I.getOperand(0);
2490  VRight = I.getOperand(1);
2491  // If we have "SplatRight", try to see if commuting is needed to preserve it.
2492  if (SplatRight) {
2493  if (VRight == Right[i - 1])
2494  // Preserve SplatRight
2495  return false;
2496  if (VLeft == Right[i - 1]) {
2497  // Commuting would preserve SplatRight, but we don't want to break
2498  // SplatLeft either, i.e. preserve the original order if possible.
2499  // (FIXME: why do we care?)
2500  if (SplatLeft && VLeft == Left[i - 1])
2501  return false;
2502  return true;
2503  }
2504  }
2505  // Symmetrically handle Right side.
2506  if (SplatLeft) {
2507  if (VLeft == Left[i - 1])
2508  // Preserve SplatLeft
2509  return false;
2510  if (VRight == Left[i - 1])
2511  return true;
2512  }
2513 
2514  Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2515  Instruction *IRight = dyn_cast<Instruction>(VRight);
2516 
2517  // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2518  // it and not the right, in this case we want to commute.
2519  if (AllSameOpcodeRight) {
2520  unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2521  if (IRight && RightPrevOpcode == IRight->getOpcode())
2522  // Do not commute, a match on the right preserves AllSameOpcodeRight
2523  return false;
2524  if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2525  // We have a match and may want to commute, but first check if there is
2526  // not also a match on the existing operands on the Left to preserve
2527  // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2528  // (FIXME: why do we care?)
2529  if (AllSameOpcodeLeft && ILeft &&
2530  cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2531  return false;
2532  return true;
2533  }
2534  }
2535  // Symmetrically handle Left side.
2536  if (AllSameOpcodeLeft) {
2537  unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2538  if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2539  return false;
2540  if (IRight && LeftPrevOpcode == IRight->getOpcode())
2541  return true;
2542  }
2543  return false;
2544 }
2545 
2546 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2547  ArrayRef<Value *> VL,
2549  SmallVectorImpl<Value *> &Right) {
2550  if (!VL.empty()) {
2551  // Peel the first iteration out of the loop since there's nothing
2552  // interesting to do anyway and it simplifies the checks in the loop.
2553  auto *I = cast<Instruction>(VL[0]);
2554  Value *VLeft = I->getOperand(0);
2555  Value *VRight = I->getOperand(1);
2556  if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2557  // Favor having instruction to the right. FIXME: why?
2558  std::swap(VLeft, VRight);
2559  Left.push_back(VLeft);
2560  Right.push_back(VRight);
2561  }
2562 
2563  // Keep track if we have instructions with all the same opcode on one side.
2564  bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2565  bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2566  // Keep track if we have one side with all the same value (broadcast).
2567  bool SplatLeft = true;
2568  bool SplatRight = true;
2569 
2570  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2571  Instruction *I = cast<Instruction>(VL[i]);
2572  assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2573  (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2574  "Can only process commutative instruction");
2575  // Commute to favor either a splat or maximizing having the same opcodes on
2576  // one side.
2577  Value *VLeft;
2578  Value *VRight;
2579  if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2580  AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2581  VRight)) {
2582  Left.push_back(VRight);
2583  Right.push_back(VLeft);
2584  } else {
2585  Left.push_back(VLeft);
2586  Right.push_back(VRight);
2587  }
2588  // Update Splat* and AllSameOpcode* after the insertion.
2589  SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2590  SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2591  AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2592  (cast<Instruction>(Left[i - 1])->getOpcode() ==
2593  cast<Instruction>(Left[i])->getOpcode());
2594  AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2595  (cast<Instruction>(Right[i - 1])->getOpcode() ==
2596  cast<Instruction>(Right[i])->getOpcode());
2597  }
2598 
2599  // If one operand end up being broadcast, return this operand order.
2600  if (SplatRight || SplatLeft)
2601  return;
2602 
2603  // Finally check if we can get longer vectorizable chain by reordering
2604  // without breaking the good operand order detected above.
2605  // E.g. If we have something like-
2606  // load a[0] load b[0]
2607  // load b[1] load a[1]
2608  // load a[2] load b[2]
2609  // load a[3] load b[3]
2610  // Reordering the second load b[1] load a[1] would allow us to vectorize
2611  // this code and we still retain AllSameOpcode property.
2612  // FIXME: This load reordering might break AllSameOpcode in some rare cases
2613  // such as-
2614  // add a[0],c[0] load b[0]
2615  // add a[1],c[2] load b[1]
2616  // b[2] load b[2]
2617  // add a[3],c[3] load b[3]
2618  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2619  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2620  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2621  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2622  std::swap(Left[j + 1], Right[j + 1]);
2623  continue;
2624  }
2625  }
2626  }
2627  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2628  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2629  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2630  std::swap(Left[j + 1], Right[j + 1]);
2631  continue;
2632  }
2633  }
2634  }
2635  // else unchanged
2636  }
2637 }
2638 
2639 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2640  // Get the basic block this bundle is in. All instructions in the bundle
2641  // should be in this block.
2642  auto *Front = cast<Instruction>(OpValue);
2643  auto *BB = Front->getParent();
2644  const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2645  const unsigned AltOpcode = getAltOpcode(Opcode);
2646  assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2647  return !sameOpcodeOrAlt(Opcode, AltOpcode,
2648  cast<Instruction>(V)->getOpcode()) ||
2649  cast<Instruction>(V)->getParent() == BB;
2650  }));
2651 
2652  // The last instruction in the bundle in program order.
2653  Instruction *LastInst = nullptr;
2654 
2655  // Find the last instruction. The common case should be that BB has been
2656  // scheduled, and the last instruction is VL.back(). So we start with
2657  // VL.back() and iterate over schedule data until we reach the end of the
2658  // bundle. The end of the bundle is marked by null ScheduleData.
2659  if (BlocksSchedules.count(BB)) {
2660  auto *Bundle =
2661  BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2662  if (Bundle && Bundle->isPartOfBundle())
2663  for (; Bundle; Bundle = Bundle->NextInBundle)
2664  if (Bundle->OpValue == Bundle->Inst)
2665  LastInst = Bundle->Inst;
2666  }
2667 
2668  // LastInst can still be null at this point if there's either not an entry
2669  // for BB in BlocksSchedules or there's no ScheduleData available for
2670  // VL.back(). This can be the case if buildTree_rec aborts for various
2671  // reasons (e.g., the maximum recursion depth is reached, the maximum region
2672  // size is reached, etc.). ScheduleData is initialized in the scheduling
2673  // "dry-run".
2674  //
2675  // If this happens, we can still find the last instruction by brute force. We
2676  // iterate forwards from Front (inclusive) until we either see all
2677  // instructions in the bundle or reach the end of the block. If Front is the
2678  // last instruction in program order, LastInst will be set to Front, and we
2679  // will visit all the remaining instructions in the block.
2680  //
2681  // One of the reasons we exit early from buildTree_rec is to place an upper
2682  // bound on compile-time. Thus, taking an additional compile-time hit here is
2683  // not ideal. However, this should be exceedingly rare since it requires that
2684  // we both exit early from buildTree_rec and that the bundle be out-of-order
2685  // (causing us to iterate all the way to the end of the block).
2686  if (!LastInst) {
2687  SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2688  for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2689  if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2690  LastInst = &I;
2691  if (Bundle.empty())
2692  break;
2693  }
2694  }
2695 
2696  // Set the insertion point after the last instruction in the bundle. Set the
2697  // debug location to Front.
2698  Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2699  Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2700 }
2701 
2702 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2703  Value *Vec = UndefValue::get(Ty);
2704  // Generate the 'InsertElement' instruction.
2705  for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2706  Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2707  if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2708  GatherSeq.insert(Insrt);
2709  CSEBlocks.insert(Insrt->getParent());
2710 
2711  // Add to our 'need-to-extract' list.
2712  if (TreeEntry *E = getTreeEntry(VL[i])) {
2713  // Find which lane we need to extract.
2714  int FoundLane = -1;
2715  for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2716  // Is this the lane of the scalar that we are looking for ?
2717  if (E->Scalars[Lane] == VL[i]) {
2718  FoundLane = Lane;
2719  break;
2720  }
2721  }
2722  assert(FoundLane >= 0 && "Could not find the correct lane");
2723  ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2724  }
2725  }
2726  }
2727 
2728  return Vec;
2729 }
2730 
2731 Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const {
2732  if (const TreeEntry *En = getTreeEntry(OpValue)) {
2733  if (En->isSame(VL) && En->VectorizedValue)
2734  return En->VectorizedValue;
2735  }
2736  return nullptr;
2737 }
2738 
2739 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2740  InstructionsState S = getSameOpcode(VL);
2741  if (S.Opcode) {
2742  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2743  if (E->isSame(VL))
2744  return vectorizeTree(E);
2745  }
2746  }
2747 
2748  Type *ScalarTy = S.OpValue->getType();
2749  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2750  ScalarTy = SI->getValueOperand()->getType();
2751  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2752 
2753  return Gather(VL, VecTy);
2754 }
2755 
2756 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2757  IRBuilder<>::InsertPointGuard Guard(Builder);
2758 
2759  if (E->VectorizedValue) {
2760  DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
2761  return E->VectorizedValue;
2762  }
2763 
2764  InstructionsState S = getSameOpcode(E->Scalars);
2765  Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2766  Type *ScalarTy = VL0->getType();
2767  if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2768  ScalarTy = SI->getValueOperand()->getType();
2769  VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2770 
2771  if (E->NeedToGather) {
2772  setInsertPointAfterBundle(E->Scalars, VL0);
2773  auto *V = Gather(E->Scalars, VecTy);
2774  E->VectorizedValue = V;
2775  return V;
2776  }
2777 
2778  unsigned ShuffleOrOp = S.IsAltShuffle ?
2779  (unsigned) Instruction::ShuffleVector : S.Opcode;
2780  switch (ShuffleOrOp) {
2781  case Instruction::PHI: {
2782  PHINode *PH = dyn_cast<PHINode>(VL0);
2783  Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2784  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2785  PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2786  E->VectorizedValue = NewPhi;
2787 
2788  // PHINodes may have multiple entries from the same block. We want to
2789  // visit every block once.
2790  SmallSet<BasicBlock*, 4> VisitedBBs;
2791 
2792  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2793  ValueList Operands;
2794  BasicBlock *IBB = PH->getIncomingBlock(i);
2795 
2796  if (!VisitedBBs.insert(IBB).second) {
2797  NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2798  continue;
2799  }
2800 
2801  // Prepare the operand vector.
2802  for (Value *V : E->Scalars)
2803  Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2804 
2805  Builder.SetInsertPoint(IBB->getTerminator());
2806  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2807  Value *Vec = vectorizeTree(Operands);
2808  NewPhi->addIncoming(Vec, IBB);
2809  }
2810 
2811  assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
2812  "Invalid number of incoming values");
2813  return NewPhi;
2814  }
2815 
2816  case Instruction::ExtractElement: {
2817  if (canReuseExtract(E->Scalars, VL0)) {
2818  Value *V = VL0->getOperand(0);
2819  E->VectorizedValue = V;
2820  return V;
2821  }
2822  setInsertPointAfterBundle(E->Scalars, VL0);
2823  auto *V = Gather(E->Scalars, VecTy);
2824  E->VectorizedValue = V;
2825  return V;
2826  }
2827  case Instruction::ExtractValue: {
2828  if (canReuseExtract(E->Scalars, VL0)) {
2829  LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2830  Builder.SetInsertPoint(LI);
2831  PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2832  Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2833  LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2834  E->VectorizedValue = V;
2835  return propagateMetadata(V, E->Scalars);
2836  }
2837  setInsertPointAfterBundle(E->Scalars, VL0);
2838  auto *V = Gather(E->Scalars, VecTy);
2839  E->VectorizedValue = V;
2840  return V;
2841  }
2842  case Instruction::ZExt:
2843  case Instruction::SExt:
2844  case Instruction::FPToUI:
2845  case Instruction::FPToSI:
2846  case Instruction::FPExt:
2847  case Instruction::PtrToInt:
2848  case Instruction::IntToPtr:
2849  case Instruction::SIToFP:
2850  case Instruction::UIToFP:
2851  case Instruction::Trunc:
2852  case Instruction::FPTrunc:
2853  case Instruction::BitCast: {
2854  ValueList INVL;
2855  for (Value *V : E->Scalars)
2856  INVL.push_back(cast<Instruction>(V)->getOperand(0));
2857 
2858  setInsertPointAfterBundle(E->Scalars, VL0);
2859 
2860  Value *InVec = vectorizeTree(INVL);
2861 
2862  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2863  return V;
2864 
2865  CastInst *CI = dyn_cast<CastInst>(VL0);
2866  Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2867  E->VectorizedValue = V;
2868  ++NumVectorInstructions;
2869  return V;
2870  }
2871  case Instruction::FCmp:
2872  case Instruction::ICmp: {
2873  ValueList LHSV, RHSV;
2874  for (Value *V : E->Scalars) {
2875  LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2876  RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2877  }
2878 
2879  setInsertPointAfterBundle(E->Scalars, VL0);
2880 
2881  Value *L = vectorizeTree(LHSV);
2882  Value *R = vectorizeTree(RHSV);
2883 
2884  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2885  return V;
2886 
2887  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2888  Value *V;
2889  if (S.Opcode == Instruction::FCmp)
2890  V = Builder.CreateFCmp(P0, L, R);
2891  else
2892  V = Builder.CreateICmp(P0, L, R);
2893 
2894  E->VectorizedValue = V;
2895  propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2896  ++NumVectorInstructions;
2897  return V;
2898  }
2899  case Instruction::Select: {
2900  ValueList TrueVec, FalseVec, CondVec;
2901  for (Value *V : E->Scalars) {
2902  CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2903  TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2904  FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2905  }
2906 
2907  setInsertPointAfterBundle(E->Scalars, VL0);
2908 
2909  Value *Cond = vectorizeTree(CondVec);
2910  Value *True = vectorizeTree(TrueVec);
2911  Value *False = vectorizeTree(FalseVec);
2912 
2913  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2914  return V;
2915 
2916  Value *V = Builder.CreateSelect(Cond, True, False);
2917  E->VectorizedValue = V;
2918  ++NumVectorInstructions;
2919  return V;
2920  }
2921  case Instruction::Add:
2922  case Instruction::FAdd:
2923  case Instruction::Sub:
2924  case Instruction::FSub:
2925  case Instruction::Mul:
2926  case Instruction::FMul:
2927  case Instruction::UDiv:
2928  case Instruction::SDiv:
2929  case Instruction::FDiv:
2930  case Instruction::URem:
2931  case Instruction::SRem:
2932  case Instruction::FRem:
2933  case Instruction::Shl:
2934  case Instruction::LShr:
2935  case Instruction::AShr:
2936  case Instruction::And:
2937  case Instruction::Or:
2938  case Instruction::Xor: {
2939  ValueList LHSVL, RHSVL;
2940  if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2941  reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
2942  RHSVL);
2943  else
2944  for (Value *V : E->Scalars) {
2945  auto *I = cast<Instruction>(V);
2946  LHSVL.push_back(I->getOperand(0));
2947  RHSVL.push_back(I->getOperand(1));
2948  }
2949 
2950  setInsertPointAfterBundle(E->Scalars, VL0);
2951 
2952  Value *LHS = vectorizeTree(LHSVL);
2953  Value *RHS = vectorizeTree(RHSVL);
2954 
2955  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2956  return V;
2957 
2958  Value *V = Builder.CreateBinOp(
2959  static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
2960  E->VectorizedValue = V;
2961  propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2962  ++NumVectorInstructions;
2963 
2964  if (Instruction *I = dyn_cast<Instruction>(V))
2965  return propagateMetadata(I, E->Scalars);
2966 
2967  return V;
2968  }
2969  case Instruction::Load: {
2970  // Loads are inserted at the head of the tree because we don't want to
2971  // sink them all the way down past store instructions.
2972  setInsertPointAfterBundle(E->Scalars, VL0);
2973 
2974  LoadInst *LI = cast<LoadInst>(VL0);
2975  Type *ScalarLoadTy = LI->getType();
2976  unsigned AS = LI->getPointerAddressSpace();
2977 
2978  Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2979  VecTy->getPointerTo(AS));
2980 
2981  // The pointer operand uses an in-tree scalar so we add the new BitCast to
2982  // ExternalUses list to make sure that an extract will be generated in the
2983  // future.
2984  Value *PO = LI->getPointerOperand();
2985  if (getTreeEntry(PO))
2986  ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
2987 
2988  unsigned Alignment = LI->getAlignment();
2989  LI = Builder.CreateLoad(VecPtr);
2990  if (!Alignment) {
2991  Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2992  }
2993  LI->setAlignment(Alignment);
2994  E->VectorizedValue = LI;
2995  ++NumVectorInstructions;
2996  return propagateMetadata(LI, E->Scalars);
2997  }
2998  case Instruction::Store: {
2999  StoreInst *SI = cast<StoreInst>(VL0);
3000  unsigned Alignment = SI->getAlignment();
3001  unsigned AS = SI->getPointerAddressSpace();
3002 
3003  ValueList ScalarStoreValues;
3004  for (Value *V : E->Scalars)
3005  ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3006 
3007  setInsertPointAfterBundle(E->Scalars, VL0);
3008 
3009  Value *VecValue = vectorizeTree(ScalarStoreValues);
3010  Value *ScalarPtr = SI->getPointerOperand();
3011  Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3012  StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3013 
3014  // The pointer operand uses an in-tree scalar, so add the new BitCast to
3015  // ExternalUses to make sure that an extract will be generated in the
3016  // future.
3017  if (getTreeEntry(ScalarPtr))
3018  ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3019 
3020  if (!Alignment)
3021  Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3022 
3023  S->setAlignment(Alignment);
3024  E->VectorizedValue = S;
3025  ++NumVectorInstructions;
3026  return propagateMetadata(S, E->Scalars);
3027  }
3028  case Instruction::GetElementPtr: {
3029  setInsertPointAfterBundle(E->Scalars, VL0);
3030 
3031  ValueList Op0VL;
3032  for (Value *V : E->Scalars)
3033  Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3034 
3035  Value *Op0 = vectorizeTree(Op0VL);
3036 
3037  std::vector<Value *> OpVecs;
3038  for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3039  ++j) {
3040  ValueList OpVL;
3041  for (Value *V : E->Scalars)
3042  OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3043 
3044  Value *OpVec = vectorizeTree(OpVL);
3045  OpVecs.push_back(OpVec);
3046  }
3047 
3048  Value *V = Builder.CreateGEP(
3049  cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3050  E->VectorizedValue = V;
3051  ++NumVectorInstructions;
3052 
3053  if (Instruction *I = dyn_cast<Instruction>(V))
3054  return propagateMetadata(I, E->Scalars);
3055 
3056  return V;
3057  }
3058  case Instruction::Call: {
3059  CallInst *CI = cast<CallInst>(VL0);
3060  setInsertPointAfterBundle(E->Scalars, VL0);
3061  Function *FI;
3063  Value *ScalarArg = nullptr;
3064  if (CI && (FI = CI->getCalledFunction())) {
3065  IID = FI->getIntrinsicID();
3066  }
3067  std::vector<Value *> OpVecs;
3068  for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3069  ValueList OpVL;
3070  // ctlz,cttz and powi are special intrinsics whose second argument is
3071  // a scalar. This argument should not be vectorized.
3072  if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3073  CallInst *CEI = cast<CallInst>(VL0);
3074  ScalarArg = CEI->getArgOperand(j);
3075  OpVecs.push_back(CEI->getArgOperand(j));
3076  continue;
3077  }
3078  for (Value *V : E->Scalars) {
3079  CallInst *CEI = cast<CallInst>(V);
3080  OpVL.push_back(CEI->getArgOperand(j));
3081  }
3082 
3083  Value *OpVec = vectorizeTree(OpVL);
3084  DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3085  OpVecs.push_back(OpVec);
3086  }
3087 
3088  Module *M = F->getParent();
3090  Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3091  Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3093  CI->getOperandBundlesAsDefs(OpBundles);
3094  Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3095 
3096  // The scalar argument uses an in-tree scalar so we add the new vectorized
3097  // call to ExternalUses list to make sure that an extract will be
3098  // generated in the future.
3099  if (ScalarArg && getTreeEntry(ScalarArg))
3100  ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3101 
3102  E->VectorizedValue = V;
3103  propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3104  ++NumVectorInstructions;
3105  return V;
3106  }
3107  case Instruction::ShuffleVector: {
3108  ValueList LHSVL, RHSVL;
3109  assert(Instruction::isBinaryOp(S.Opcode) &&
3110  "Invalid Shuffle Vector Operand");
3111  reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3112  setInsertPointAfterBundle(E->Scalars, VL0);
3113 
3114  Value *LHS = vectorizeTree(LHSVL);
3115  Value *RHS = vectorizeTree(RHSVL);
3116 
3117  if (Value *V = alreadyVectorized(E->Scalars, VL0))
3118  return V;
3119 
3120  // Create a vector of LHS op1 RHS
3121  Value *V0 = Builder.CreateBinOp(
3122  static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3123 
3124  unsigned AltOpcode = getAltOpcode(S.Opcode);
3125  // Create a vector of LHS op2 RHS
3126  Value *V1 = Builder.CreateBinOp(
3127  static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3128 
3129  // Create shuffle to take alternate operations from the vector.
3130  // Also, gather up odd and even scalar ops to propagate IR flags to
3131  // each vector operation.
3132  ValueList OddScalars, EvenScalars;
3133  unsigned e = E->Scalars.size();
3135  for (unsigned i = 0; i < e; ++i) {
3136  if (isOdd(i)) {
3137  Mask[i] = Builder.getInt32(e + i);
3138  OddScalars.push_back(E->Scalars[i]);
3139  } else {
3140  Mask[i] = Builder.getInt32(i);
3141  EvenScalars.push_back(E->Scalars[i]);
3142  }
3143  }
3144 
3145  Value *ShuffleMask = ConstantVector::get(Mask);
3146  propagateIRFlags(V0, EvenScalars);
3147  propagateIRFlags(V1, OddScalars);
3148 
3149  Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3150  E->VectorizedValue = V;
3151  ++NumVectorInstructions;
3152  if (Instruction *I = dyn_cast<Instruction>(V))
3153  return propagateMetadata(I, E->Scalars);
3154 
3155  return V;
3156  }
3157  default:
3158  llvm_unreachable("unknown inst");
3159  }
3160  return nullptr;
3161 }
3162 
3163 Value *BoUpSLP::vectorizeTree() {
3164  ExtraValueToDebugLocsMap ExternallyUsedValues;
3165  return vectorizeTree(ExternallyUsedValues);
3166 }
3167 
3168 Value *
3169 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3170  // All blocks must be scheduled before any instructions are inserted.
3171  for (auto &BSIter : BlocksSchedules) {
3172  scheduleBlock(BSIter.second.get());
3173  }
3174 
3175  Builder.SetInsertPoint(&F->getEntryBlock().front());
3176  auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3177 
3178  // If the vectorized tree can be rewritten in a smaller type, we truncate the
3179  // vectorized root. InstCombine will then rewrite the entire expression. We
3180  // sign extend the extracted values below.
3181  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3182  if (MinBWs.count(ScalarRoot)) {
3183  if (auto *I = dyn_cast<Instruction>(VectorRoot))
3184  Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3185  auto BundleWidth = VectorizableTree[0].Scalars.size();
3186  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3187  auto *VecTy = VectorType::get(MinTy, BundleWidth);
3188  auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3189  VectorizableTree[0].VectorizedValue = Trunc;
3190  }
3191 
3192  DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n");
3193 
3194  // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3195  // specified by ScalarType.
3196  auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3197  if (!MinBWs.count(ScalarRoot))
3198  return Ex;
3199  if (MinBWs[ScalarRoot].second)
3200  return Builder.CreateSExt(Ex, ScalarType);
3201  return Builder.CreateZExt(Ex, ScalarType);
3202  };
3203 
3204  // Extract all of the elements with the external uses.
3205  for (const auto &ExternalUse : ExternalUses) {
3206  Value *Scalar = ExternalUse.Scalar;
3207  llvm::User *User = ExternalUse.User;
3208 
3209  // Skip users that we already RAUW. This happens when one instruction
3210  // has multiple uses of the same value.
3211  if (User && !is_contained(Scalar->users(), User))
3212  continue;
3213  TreeEntry *E = getTreeEntry(Scalar);
3214  assert(E && "Invalid scalar");
3215  assert(!E->NeedToGather && "Extracting from a gather list");
3216 
3217  Value *Vec = E->VectorizedValue;
3218  assert(Vec && "Can't find vectorizable value");
3219 
3220  Value *Lane = Builder.getInt32(ExternalUse.Lane);
3221  // If User == nullptr, the Scalar is used as extra arg. Generate
3222  // ExtractElement instruction and update the record for this scalar in
3223  // ExternallyUsedValues.
3224  if (!User) {
3225  assert(ExternallyUsedValues.count(Scalar) &&
3226  "Scalar with nullptr as an external user must be registered in "
3227  "ExternallyUsedValues map");
3228  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3229  Builder.SetInsertPoint(VecI->getParent(),
3230  std::next(VecI->getIterator()));
3231  } else {
3232  Builder.SetInsertPoint(&F->getEntryBlock().front());
3233  }
3234  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3235  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3236  CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3237  auto &Locs = ExternallyUsedValues[Scalar];
3238  ExternallyUsedValues.insert({Ex, Locs});
3239  ExternallyUsedValues.erase(Scalar);
3240  continue;
3241  }
3242 
3243  // Generate extracts for out-of-tree users.
3244  // Find the insertion point for the extractelement lane.
3245  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3246  if (PHINode *PH = dyn_cast<PHINode>(User)) {
3247  for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3248  if (PH->getIncomingValue(i) == Scalar) {
3249  TerminatorInst *IncomingTerminator =
3250  PH->getIncomingBlock(i)->getTerminator();
3251  if (isa<CatchSwitchInst>(IncomingTerminator)) {
3252  Builder.SetInsertPoint(VecI->getParent(),
3253  std::next(VecI->getIterator()));
3254  } else {
3255  Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3256  }
3257  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3258  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3259  CSEBlocks.insert(PH->getIncomingBlock(i));
3260  PH->setOperand(i, Ex);
3261  }
3262  }
3263  } else {
3264  Builder.SetInsertPoint(cast<Instruction>(User));
3265  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3266  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3267  CSEBlocks.insert(cast<Instruction>(User)->getParent());
3268  User->replaceUsesOfWith(Scalar, Ex);
3269  }
3270  } else {
3271  Builder.SetInsertPoint(&F->getEntryBlock().front());
3272  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3273  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3274  CSEBlocks.insert(&F->getEntryBlock());
3275  User->replaceUsesOfWith(Scalar, Ex);
3276  }
3277 
3278  DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3279  }
3280 
3281  // For each vectorized value:
3282  for (TreeEntry &EIdx : VectorizableTree) {
3283  TreeEntry *Entry = &EIdx;
3284 
3285  // No need to handle users of gathered values.
3286  if (Entry->NeedToGather)
3287  continue;
3288 
3289  assert(Entry->VectorizedValue && "Can't find vectorizable value");
3290 
3291  // For each lane:
3292  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3293  Value *Scalar = Entry->Scalars[Lane];
3294 
3295  Type *Ty = Scalar->getType();
3296  if (!Ty->isVoidTy()) {
3297 #ifndef NDEBUG
3298  for (User *U : Scalar->users()) {
3299  DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3300 
3301  // It is legal to replace users in the ignorelist by undef.
3302  assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3303  "Replacing out-of-tree value with undef");
3304  }
3305 #endif
3306  Value *Undef = UndefValue::get(Ty);
3307  Scalar->replaceAllUsesWith(Undef);
3308  }
3309  DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3310  eraseInstruction(cast<Instruction>(Scalar));
3311  }
3312  }
3313 
3314  Builder.ClearInsertionPoint();
3315 
3316  return VectorizableTree[0].VectorizedValue;
3317 }
3318 
3319 void BoUpSLP::optimizeGatherSequence() {
3320  DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3321  << " gather sequences instructions.\n");
3322  // LICM InsertElementInst sequences.
3323  for (Instruction *it : GatherSeq) {
3325 
3326  if (!Insert)
3327  continue;
3328 
3329  // Check if this block is inside a loop.
3330  Loop *L = LI->getLoopFor(Insert->getParent());
3331  if (!L)
3332  continue;
3333 
3334  // Check if it has a preheader.
3335  BasicBlock *PreHeader = L->getLoopPreheader();
3336  if (!PreHeader)
3337  continue;
3338 
3339  // If the vector or the element that we insert into it are
3340  // instructions that are defined in this basic block then we can't
3341  // hoist this instruction.
3342  Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
3343  Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
3344  if (CurrVec && L->contains(CurrVec))
3345  continue;
3346  if (NewElem && L->contains(NewElem))
3347  continue;
3348 
3349  // We can hoist this instruction. Move it to the pre-header.
3350  Insert->moveBefore(PreHeader->getTerminator());
3351  }
3352 
3353  // Make a list of all reachable blocks in our CSE queue.
3355  CSEWorkList.reserve(CSEBlocks.size());
3356  for (BasicBlock *BB : CSEBlocks)
3357  if (DomTreeNode *N = DT->getNode(BB)) {
3358  assert(DT->isReachableFromEntry(N));
3359  CSEWorkList.push_back(N);
3360  }
3361 
3362  // Sort blocks by domination. This ensures we visit a block after all blocks
3363  // dominating it are visited.
3364  std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3365  [this](const DomTreeNode *A, const DomTreeNode *B) {
3366  return DT->properlyDominates(A, B);
3367  });
3368 
3369  // Perform O(N^2) search over the gather sequences and merge identical
3370  // instructions. TODO: We can further optimize this scan if we split the
3371  // instructions into different buckets based on the insert lane.
3373  for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3374  assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
3375  "Worklist not sorted properly!");
3376  BasicBlock *BB = (*I)->getBlock();
3377  // For all instructions in blocks containing gather sequences:
3378  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3379  Instruction *In = &*it++;
3380  if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3381  continue;
3382 
3383  // Check if we can replace this instruction with any of the
3384  // visited instructions.
3385  for (Instruction *v : Visited) {
3386  if (In->isIdenticalTo(v) &&
3387  DT->dominates(v->getParent(), In->getParent())) {
3388  In->replaceAllUsesWith(v);
3389  eraseInstruction(In);
3390  In = nullptr;
3391  break;
3392  }
3393  }
3394  if (In) {
3395  assert(!is_contained(Visited, In));
3396  Visited.push_back(In);
3397  }
3398  }
3399  }
3400  CSEBlocks.clear();
3401  GatherSeq.clear();
3402 }
3403 
3404 // Groups the instructions to a bundle (which is then a single scheduling entity)
3405 // and schedules instructions until the bundle gets ready.
3406 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3407  BoUpSLP *SLP, Value *OpValue) {
3408  if (isa<PHINode>(OpValue))
3409  return true;
3410 
3411  // Initialize the instruction bundle.
3412  Instruction *OldScheduleEnd = ScheduleEnd;
3413  ScheduleData *PrevInBundle = nullptr;
3414  ScheduleData *Bundle = nullptr;
3415  bool ReSchedule = false;
3416  DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n");
3417 
3418  // Make sure that the scheduling region contains all
3419  // instructions of the bundle.
3420  for (Value *V : VL) {
3421  if (!extendSchedulingRegion(V, OpValue))
3422  return false;
3423  }
3424 
3425  for (Value *V : VL) {
3426  ScheduleData *BundleMember = getScheduleData(V);
3427  assert(BundleMember &&
3428  "no ScheduleData for bundle member (maybe not in same basic block)");
3429  if (BundleMember->IsScheduled) {
3430  // A bundle member was scheduled as single instruction before and now
3431  // needs to be scheduled as part of the bundle. We just get rid of the
3432  // existing schedule.
3433  DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3434  << " was already scheduled\n");
3435  ReSchedule = true;
3436  }
3437  assert(BundleMember->isSchedulingEntity() &&
3438  "bundle member already part of other bundle");
3439  if (PrevInBundle) {
3440  PrevInBundle->NextInBundle = BundleMember;
3441  } else {
3442  Bundle = BundleMember;
3443  }
3444  BundleMember->UnscheduledDepsInBundle = 0;
3445  Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3446 
3447  // Group the instructions to a bundle.
3448  BundleMember->FirstInBundle = Bundle;
3449  PrevInBundle = BundleMember;
3450  }
3451  if (ScheduleEnd != OldScheduleEnd) {
3452  // The scheduling region got new instructions at the lower end (or it is a
3453  // new region for the first bundle). This makes it necessary to
3454  // recalculate all dependencies.
3455  // It is seldom that this needs to be done a second time after adding the
3456  // initial bundle to the region.
3457  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3458  doForAllOpcodes(I, [](ScheduleData *SD) {
3459  SD->clearDependencies();
3460  });
3461  }
3462  ReSchedule = true;
3463  }
3464  if (ReSchedule) {
3465  resetSchedule();
3466  initialFillReadyList(ReadyInsts);
3467  }
3468 
3469  DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3470  << BB->getName() << "\n");
3471 
3472  calculateDependencies(Bundle, true, SLP);
3473 
3474  // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3475  // means that there are no cyclic dependencies and we can schedule it.
3476  // Note that's important that we don't "schedule" the bundle yet (see
3477  // cancelScheduling).
3478  while (!Bundle->isReady() && !ReadyInsts.empty()) {
3479 
3480  ScheduleData *pickedSD = ReadyInsts.back();
3481  ReadyInsts.pop_back();
3482 
3483  if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3484  schedule(pickedSD, ReadyInsts);
3485  }
3486  }
3487  if (!Bundle->isReady()) {
3488  cancelScheduling(VL, OpValue);
3489  return false;
3490  }
3491  return true;
3492 }
3493 
3494 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3495  Value *OpValue) {
3496  if (isa<PHINode>(OpValue))
3497  return;
3498 
3499  ScheduleData *Bundle = getScheduleData(OpValue);
3500  DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3501  assert(!Bundle->IsScheduled &&
3502  "Can't cancel bundle which is already scheduled");
3503  assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3504  "tried to unbundle something which is not a bundle");
3505 
3506  // Un-bundle: make single instructions out of the bundle.
3507  ScheduleData *BundleMember = Bundle;
3508  while (BundleMember) {
3509  assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3510  BundleMember->FirstInBundle = BundleMember;
3511  ScheduleData *Next = BundleMember->NextInBundle;
3512  BundleMember->NextInBundle = nullptr;
3513  BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3514  if (BundleMember->UnscheduledDepsInBundle == 0) {
3515  ReadyInsts.insert(BundleMember);
3516  }
3517  BundleMember = Next;
3518  }
3519 }
3520 
3521 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3522  // Allocate a new ScheduleData for the instruction.
3523  if (ChunkPos >= ChunkSize) {
3524  ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3525  ChunkPos = 0;
3526  }
3527  return &(ScheduleDataChunks.back()[ChunkPos++]);
3528 }
3529 
3530 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3531  Value *OpValue) {
3532  if (getScheduleData(V, isOneOf(OpValue, V)))
3533  return true;
3534  Instruction *I = dyn_cast<Instruction>(V);
3535  assert(I && "bundle member must be an instruction");
3536  assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3537  auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3538  ScheduleData *ISD = getScheduleData(I);
3539  if (!ISD)
3540  return false;
3541  assert(isInSchedulingRegion(ISD) &&
3542  "ScheduleData not in scheduling region");
3543  ScheduleData *SD = allocateScheduleDataChunks();
3544  SD->Inst = I;
3545  SD->init(SchedulingRegionID, OpValue);
3546  ExtraScheduleDataMap[I][OpValue] = SD;
3547  return true;
3548  };
3549  if (CheckSheduleForI(I))
3550  return true;
3551  if (!ScheduleStart) {
3552  // It's the first instruction in the new region.
3553  initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3554  ScheduleStart = I;
3555  ScheduleEnd = I->getNextNode();
3556  if (isOneOf(OpValue, I) != I)
3557  CheckSheduleForI(I);
3558  assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3559  DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3560  return true;
3561  }
3562  // Search up and down at the same time, because we don't know if the new
3563  // instruction is above or below the existing scheduling region.
3565  ++ScheduleStart->getIterator().getReverse();
3566  BasicBlock::reverse_iterator UpperEnd = BB->rend();
3567  BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3568  BasicBlock::iterator LowerEnd = BB->end();
3569  while (true) {
3570  if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3571  DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3572  return false;
3573  }
3574 
3575  if (UpIter != UpperEnd) {
3576  if (&*UpIter == I) {
3577  initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3578  ScheduleStart = I;
3579  if (isOneOf(OpValue, I) != I)
3580  CheckSheduleForI(I);
3581  DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n");
3582  return true;
3583  }
3584  UpIter++;
3585  }
3586  if (DownIter != LowerEnd) {
3587  if (&*DownIter == I) {
3588  initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3589  nullptr);
3590  ScheduleEnd = I->getNextNode();
3591  if (isOneOf(OpValue, I) != I)
3592  CheckSheduleForI(I);
3593  assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3594  DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
3595  return true;
3596  }
3597  DownIter++;
3598  }
3599  assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
3600  "instruction not found in block");
3601  }
3602  return true;
3603 }
3604 
3605 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3606  Instruction *ToI,
3607  ScheduleData *PrevLoadStore,
3608  ScheduleData *NextLoadStore) {
3609  ScheduleData *CurrentLoadStore = PrevLoadStore;
3610  for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3611  ScheduleData *SD = ScheduleDataMap[I];
3612  if (!SD) {
3613  SD = allocateScheduleDataChunks();
3614  ScheduleDataMap[I] = SD;
3615  SD->Inst = I;
3616  }
3617  assert(!isInSchedulingRegion(SD) &&
3618  "new ScheduleData already in scheduling region");
3619  SD->init(SchedulingRegionID, I);
3620 
3621  if (I->mayReadOrWriteMemory() &&
3622  (!isa<IntrinsicInst>(I) ||
3623  cast<IntrinsicInst>(I)->getIntrinsicID() != Intrinsic::sideeffect)) {
3624  // Update the linked list of memory accessing instructions.
3625  if (CurrentLoadStore) {
3626  CurrentLoadStore->NextLoadStore = SD;
3627  } else {
3628  FirstLoadStoreInRegion = SD;
3629  }
3630  CurrentLoadStore = SD;
3631  }
3632  }
3633  if (NextLoadStore) {
3634  if (CurrentLoadStore)
3635  CurrentLoadStore->NextLoadStore = NextLoadStore;
3636  } else {
3637  LastLoadStoreInRegion = CurrentLoadStore;
3638  }
3639 }
3640 
3641 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3642  bool InsertInReadyList,
3643  BoUpSLP *SLP) {
3644  assert(SD->isSchedulingEntity());
3645 
3647  WorkList.push_back(SD);
3648 
3649  while (!WorkList.empty()) {
3650  ScheduleData *SD = WorkList.back();
3651  WorkList.pop_back();
3652 
3653  ScheduleData *BundleMember = SD;
3654  while (BundleMember) {
3655  assert(isInSchedulingRegion(BundleMember));
3656  if (!BundleMember->hasValidDependencies()) {
3657 
3658  DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n");
3659  BundleMember->Dependencies = 0;
3660  BundleMember->resetUnscheduledDeps();
3661 
3662  // Handle def-use chain dependencies.
3663  if (BundleMember->OpValue != BundleMember->Inst) {
3664  ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
3665  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3666  BundleMember->Dependencies++;
3667  ScheduleData *DestBundle = UseSD->FirstInBundle;
3668  if (!DestBundle->IsScheduled)
3669  BundleMember->incrementUnscheduledDeps(1);
3670  if (!DestBundle->hasValidDependencies())
3671  WorkList.push_back(DestBundle);
3672  }
3673  } else {
3674  for (User *U : BundleMember->Inst->users()) {
3675  if (isa<Instruction>(U)) {
3676  ScheduleData *UseSD = getScheduleData(U);
3677  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3678  BundleMember->Dependencies++;
3679  ScheduleData *DestBundle = UseSD->FirstInBundle;
3680  if (!DestBundle->IsScheduled)
3681  BundleMember->incrementUnscheduledDeps(1);
3682  if (!DestBundle->hasValidDependencies())
3683  WorkList.push_back(DestBundle);
3684  }
3685  } else {
3686  // I'm not sure if this can ever happen. But we need to be safe.
3687  // This lets the instruction/bundle never be scheduled and
3688  // eventually disable vectorization.
3689  BundleMember->Dependencies++;
3690  BundleMember->incrementUnscheduledDeps(1);
3691  }
3692  }
3693  }
3694 
3695  // Handle the memory dependencies.
3696  ScheduleData *DepDest = BundleMember->NextLoadStore;
3697  if (DepDest) {
3698  Instruction *SrcInst = BundleMember->Inst;
3699  MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3700  bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3701  unsigned numAliased = 0;
3702  unsigned DistToSrc = 1;
3703 
3704  while (DepDest) {
3705  assert(isInSchedulingRegion(DepDest));
3706 
3707  // We have two limits to reduce the complexity:
3708  // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3709  // SLP->isAliased (which is the expensive part in this loop).
3710  // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3711  // the whole loop (even if the loop is fast, it's quadratic).
3712  // It's important for the loop break condition (see below) to
3713  // check this limit even between two read-only instructions.
3714  if (DistToSrc >= MaxMemDepDistance ||
3715  ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3716  (numAliased >= AliasedCheckLimit ||
3717  SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3718 
3719  // We increment the counter only if the locations are aliased
3720  // (instead of counting all alias checks). This gives a better
3721  // balance between reduced runtime and accurate dependencies.
3722  numAliased++;
3723 
3724  DepDest->MemoryDependencies.push_back(BundleMember);
3725  BundleMember->Dependencies++;
3726  ScheduleData *DestBundle = DepDest->FirstInBundle;
3727  if (!DestBundle->IsScheduled) {
3728  BundleMember->incrementUnscheduledDeps(1);
3729  }
3730  if (!DestBundle->hasValidDependencies()) {
3731  WorkList.push_back(DestBundle);
3732  }
3733  }
3734  DepDest = DepDest->NextLoadStore;
3735 
3736  // Example, explaining the loop break condition: Let's assume our
3737  // starting instruction is i0 and MaxMemDepDistance = 3.
3738  //
3739  // +--------v--v--v
3740  // i0,i1,i2,i3,i4,i5,i6,i7,i8
3741  // +--------^--^--^
3742  //
3743  // MaxMemDepDistance let us stop alias-checking at i3 and we add
3744  // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3745  // Previously we already added dependencies from i3 to i6,i7,i8
3746  // (because of MaxMemDepDistance). As we added a dependency from
3747  // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3748  // and we can abort this loop at i6.
3749  if (DistToSrc >= 2 * MaxMemDepDistance)
3750  break;
3751  DistToSrc++;
3752  }
3753  }
3754  }
3755  BundleMember = BundleMember->NextInBundle;
3756  }
3757  if (InsertInReadyList && SD->isReady()) {
3758  ReadyInsts.push_back(SD);
3759  DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n");
3760  }
3761  }
3762 }
3763 
3764 void BoUpSLP::BlockScheduling::resetSchedule() {
3765  assert(ScheduleStart &&
3766  "tried to reset schedule on block which has not been scheduled");
3767  for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3768  doForAllOpcodes(I, [&](ScheduleData *SD) {
3769  assert(isInSchedulingRegion(SD) &&
3770  "ScheduleData not in scheduling region");
3771  SD->IsScheduled = false;
3772  SD->resetUnscheduledDeps();
3773  });
3774  }
3775  ReadyInsts.clear();
3776 }
3777 
3778 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3779  if (!BS->ScheduleStart)
3780  return;
3781 
3782  DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
3783 
3784  BS->resetSchedule();
3785 
3786  // For the real scheduling we use a more sophisticated ready-list: it is
3787  // sorted by the original instruction location. This lets the final schedule
3788  // be as close as possible to the original instruction order.
3789  struct ScheduleDataCompare {
3790  bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3791  return SD2->SchedulingPriority < SD1->SchedulingPriority;
3792  }
3793  };
3794  std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3795 
3796  // Ensure that all dependency data is updated and fill the ready-list with
3797  // initial instructions.
3798  int Idx = 0;
3799  int NumToSchedule = 0;
3800  for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3801  I = I->getNextNode()) {
3802  BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
3803  assert(SD->isPartOfBundle() ==
3804  (getTreeEntry(SD->Inst) != nullptr) &&
3805  "scheduler and vectorizer bundle mismatch");
3806  SD->FirstInBundle->SchedulingPriority = Idx++;
3807  if (SD->isSchedulingEntity()) {
3808  BS->calculateDependencies(SD, false, this);
3809  NumToSchedule++;
3810  }
3811  });
3812  }
3813  BS->initialFillReadyList(ReadyInsts);
3814 
3815  Instruction *LastScheduledInst = BS->ScheduleEnd;
3816 
3817  // Do the "real" scheduling.
3818  while (!ReadyInsts.empty()) {
3819  ScheduleData *picked = *ReadyInsts.begin();
3820  ReadyInsts.erase(ReadyInsts.begin());
3821 
3822  // Move the scheduled instruction(s) to their dedicated places, if not
3823  // there yet.
3824  ScheduleData *BundleMember = picked;
3825  while (BundleMember) {
3826  Instruction *pickedInst = BundleMember->Inst;
3827  if (LastScheduledInst->getNextNode() != pickedInst) {
3828  BS->BB->getInstList().remove(pickedInst);
3829  BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3830  pickedInst);
3831  }
3832  LastScheduledInst = pickedInst;
3833  BundleMember = BundleMember->NextInBundle;
3834  }
3835 
3836  BS->schedule(picked, ReadyInsts);
3837  NumToSchedule--;
3838  }
3839  assert(NumToSchedule == 0 && "could not schedule all instructions");
3840 
3841  // Avoid duplicate scheduling of the block.
3842  BS->ScheduleStart = nullptr;
3843 }
3844 
3845 unsigned BoUpSLP::getVectorElementSize(Value *V) {
3846  // If V is a store, just return the width of the stored value without
3847  // traversing the expression tree. This is the common case.
3848  if (auto *Store = dyn_cast<StoreInst>(V))
3849  return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3850 
3851  // If V is not a store, we can traverse the expression tree to find loads
3852  // that feed it. The type of the loaded value may indicate a more suitable
3853  // width than V's type. We want to base the vector element size on the width
3854  // of memory operations where possible.
3857  if (auto *I = dyn_cast<Instruction>(V))
3858  Worklist.push_back(I);
3859 
3860  // Traverse the expression tree in bottom-up order looking for loads. If we
3861  // encounter an instruciton we don't yet handle, we give up.
3862  auto MaxWidth = 0u;
3863  auto FoundUnknownInst = false;
3864  while (!Worklist.empty() && !FoundUnknownInst) {
3865  auto *I = Worklist.pop_back_val();
3866  Visited.insert(I);
3867 
3868  // We should only be looking at scalar instructions here. If the current
3869  // instruction has a vector type, give up.
3870  auto *Ty = I->getType();
3871  if (isa<VectorType>(Ty))
3872  FoundUnknownInst = true;
3873 
3874  // If the current instruction is a load, update MaxWidth to reflect the
3875  // width of the loaded value.
3876  else if (isa<LoadInst>(I))
3877  MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3878 
3879  // Otherwise, we need to visit the operands of the instruction. We only
3880  // handle the interesting cases from buildTree here. If an operand is an
3881  // instruction we haven't yet visited, we add it to the worklist.
3882  else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3883  isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3884  for (Use &U : I->operands())
3885  if (auto *J = dyn_cast<Instruction>(U.get()))
3886  if (!Visited.count(J))
3887  Worklist.push_back(J);
3888  }
3889 
3890  // If we don't yet handle the instruction, give up.
3891  else
3892  FoundUnknownInst = true;
3893  }
3894 
3895  // If we didn't encounter a memory access in the expression tree, or if we
3896  // gave up for some reason, just return the width of V.
3897  if (!MaxWidth || FoundUnknownInst)
3898  return DL->getTypeSizeInBits(V->getType());
3899 
3900  // Otherwise, return the maximum width we found.
3901  return MaxWidth;
3902 }
3903 
3904 // Determine if a value V in a vectorizable expression Expr can be demoted to a
3905 // smaller type with a truncation. We collect the values that will be demoted
3906 // in ToDemote and additional roots that require investigating in Roots.
3908  SmallVectorImpl<Value *> &ToDemote,
3909  SmallVectorImpl<Value *> &Roots) {
3910  // We can always demote constants.
3911  if (isa<Constant>(V)) {
3912  ToDemote.push_back(V);
3913  return true;
3914  }
3915 
3916  // If the value is not an instruction in the expression with only one use, it
3917  // cannot be demoted.
3918  auto *I = dyn_cast<Instruction>(V);
3919  if (!I || !I->hasOneUse() || !Expr.count(I))
3920  return false;
3921 
3922  switch (I->getOpcode()) {
3923 
3924  // We can always demote truncations and extensions. Since truncations can
3925  // seed additional demotion, we save the truncated value.
3926  case Instruction::Trunc:
3927  Roots.push_back(I->getOperand(0));
3928  break;
3929  case Instruction::ZExt:
3930  case Instruction::SExt:
3931  break;
3932 
3933  // We can demote certain binary operations if we can demote both of their
3934  // operands.
3935  case Instruction::Add:
3936  case Instruction::Sub:
3937  case Instruction::Mul:
3938  case Instruction::And:
3939  case Instruction::Or:
3940  case Instruction::Xor:
3941  if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3942  !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3943  return false;
3944  break;
3945 
3946  // We can demote selects if we can demote their true and false values.
3947  case Instruction::Select: {
3948  SelectInst *SI = cast<SelectInst>(I);
3949  if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3950  !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3951  return false;
3952  break;
3953  }
3954 
3955  // We can demote phis if we can demote all their incoming operands. Note that
3956  // we don't need to worry about cycles since we ensure single use above.
3957  case Instruction::PHI: {
3958  PHINode *PN = cast<PHINode>(I);
3959  for (Value *IncValue : PN->incoming_values())
3960  if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3961  return false;
3962  break;
3963  }
3964 
3965  // Otherwise, conservatively give up.
3966  default:
3967  return false;
3968  }
3969 
3970  // Record the value that we can demote.
3971  ToDemote.push_back(V);
3972  return true;
3973 }
3974 
3976  // If there are no external uses, the expression tree must be rooted by a
3977  // store. We can't demote in-memory values, so there is nothing to do here.
3978  if (ExternalUses.empty())
3979  return;
3980 
3981  // We only attempt to truncate integer expressions.
3982  auto &TreeRoot = VectorizableTree[0].Scalars;
3983  auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3984  if (!TreeRootIT)
3985  return;
3986 
3987  // If the expression is not rooted by a store, these roots should have
3988  // external uses. We will rely on InstCombine to rewrite the expression in
3989  // the narrower type. However, InstCombine only rewrites single-use values.
3990  // This means that if a tree entry other than a root is used externally, it
3991  // must have multiple uses and InstCombine will not rewrite it. The code
3992  // below ensures that only the roots are used externally.
3993  SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3994  for (auto &EU : ExternalUses)
3995  if (!Expr.erase(EU.Scalar))
3996  return;
3997  if (!Expr.empty())
3998  return;
3999 
4000  // Collect the scalar values of the vectorizable expression. We will use this
4001  // context to determine which values can be demoted. If we see a truncation,
4002  // we mark it as seeding another demotion.
4003  for (auto &Entry : VectorizableTree)
4004  Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
4005 
4006  // Ensure the roots of the vectorizable tree don't form a cycle. They must
4007  // have a single external user that is not in the vectorizable tree.
4008  for (auto *Root : TreeRoot)
4009  if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4010  return;
4011 
4012  // Conservatively determine if we can actually truncate the roots of the
4013  // expression. Collect the values that can be demoted in ToDemote and
4014  // additional roots that require investigating in Roots.
4015  SmallVector<Value *, 32> ToDemote;
4017  for (auto *Root : TreeRoot)
4018  if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4019  return;
4020 
4021  // The maximum bit width required to represent all the values that can be
4022  // demoted without loss of precision. It would be safe to truncate the roots
4023  // of the expression to this width.
4024  auto MaxBitWidth = 8u;
4025 
4026  // We first check if all the bits of the roots are demanded. If they're not,
4027  // we can truncate the roots to this narrower type.
4028  for (auto *Root : TreeRoot) {
4029  auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4030  MaxBitWidth = std::max<unsigned>(
4031  Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4032  }
4033 
4034  // True if the roots can be zero-extended back to their original type, rather
4035  // than sign-extended. We know that if the leading bits are not demanded, we
4036  // can safely zero-extend. So we initialize IsKnownPositive to True.
4037  bool IsKnownPositive = true;
4038 
4039  // If all the bits of the roots are demanded, we can try a little harder to
4040  // compute a narrower type. This can happen, for example, if the roots are
4041  // getelementptr indices. InstCombine promotes these indices to the pointer
4042  // width. Thus, all their bits are technically demanded even though the
4043  // address computation might be vectorized in a smaller type.
4044  //
4045  // We start by looking at each entry that can be demoted. We compute the
4046  // maximum bit width required to store the scalar by using ValueTracking to
4047  // compute the number of high-order bits we can truncate.
4048  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
4049  MaxBitWidth = 8u;
4050 
4051  // Determine if the sign bit of all the roots is known to be zero. If not,
4052  // IsKnownPositive is set to False.
4053  IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4054  KnownBits Known = computeKnownBits(R, *DL);
4055  return Known.isNonNegative();
4056  });
4057 
4058  // Determine the maximum number of bits required to store the scalar
4059  // values.
4060  for (auto *Scalar : ToDemote) {
4061  auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4062  auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4063  MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4064  }
4065 
4066  // If we can't prove that the sign bit is zero, we must add one to the
4067  // maximum bit width to account for the unknown sign bit. This preserves
4068  // the existing sign bit so we can safely sign-extend the root back to the
4069  // original type. Otherwise, if we know the sign bit is zero, we will
4070  // zero-extend the root instead.
4071  //
4072  // FIXME: This is somewhat suboptimal, as there will be cases where adding
4073  // one to the maximum bit width will yield a larger-than-necessary
4074  // type. In general, we need to add an extra bit only if we can't
4075  // prove that the upper bit of the original type is equal to the
4076  // upper bit of the proposed smaller type. If these two bits are the
4077  // same (either zero or one) we know that sign-extending from the
4078  // smaller type will result in the same value. Here, since we can't
4079  // yet prove this, we are just making the proposed smaller type
4080  // larger to ensure correctness.
4081  if (!IsKnownPositive)
4082  ++MaxBitWidth;
4083  }
4084 
4085  // Round MaxBitWidth up to the next power-of-two.
4086  if (!isPowerOf2_64(MaxBitWidth))
4087  MaxBitWidth = NextPowerOf2(MaxBitWidth);
4088 
4089  // If the maximum bit width we compute is less than the with of the roots'
4090  // type, we can proceed with the narrowing. Otherwise, do nothing.
4091  if (MaxBitWidth >= TreeRootIT->getBitWidth())
4092  return;
4093 
4094  // If we can truncate the root, we must collect additional values that might
4095  // be demoted as a result. That is, those seeded by truncations we will
4096  // modify.
4097  while (!Roots.empty())
4098  collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4099 
4100  // Finally, map the values we can demote to the maximum bit with we computed.
4101  for (auto *Scalar : ToDemote)
4102  MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4103 }
4104 
4105 namespace {
4106 
4107 /// The SLPVectorizer Pass.
4108 struct SLPVectorizer : public FunctionPass {
4109  SLPVectorizerPass Impl;
4110 
4111  /// Pass identification, replacement for typeid
4112  static char ID;
4113 
4114  explicit SLPVectorizer() : FunctionPass(ID) {
4116  }
4117 
4118  bool doInitialization(Module &M) override {
4119  return false;
4120  }
4121 
4122  bool runOnFunction(Function &F) override {
4123  if (skipFunction(F))
4124  return false;
4125 
4126  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4127  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4128  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4129  auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4130  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4131  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4132  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4133  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4134  auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4135  auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4136 
4137  return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4138  }
4139 
4140  void getAnalysisUsage(AnalysisUsage &AU) const override {
4154  AU.setPreservesCFG();
4155  }
4156 };
4157 
4158 } // end anonymous namespace
4159 
4161  auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4162  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4163  auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4164  auto *AA = &AM.getResult<AAManager>(F);
4165  auto *LI = &AM.getResult<LoopAnalysis>(F);
4166  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4167  auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4168  auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4169  auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4170 
4171  bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4172  if (!Changed)
4173  return PreservedAnalyses::all();
4174 
4175  PreservedAnalyses PA;
4176  PA.preserveSet<CFGAnalyses>();
4177  PA.preserve<AAManager>();
4178  PA.preserve<GlobalsAA>();
4179  return PA;
4180 }
4181 
4183  TargetTransformInfo *TTI_,
4184  TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4185  LoopInfo *LI_, DominatorTree *DT_,
4186  AssumptionCache *AC_, DemandedBits *DB_,
4187  OptimizationRemarkEmitter *ORE_) {
4188  SE = SE_;
4189  TTI = TTI_;
4190  TLI = TLI_;
4191  AA = AA_;
4192  LI = LI_;
4193  DT = DT_;
4194  AC = AC_;
4195  DB = DB_;
4196  DL = &F.getParent()->getDataLayout();
4197 
4198  Stores.clear();
4199  GEPs.clear();
4200  bool Changed = false;
4201 
4202  // If the target claims to have no vector registers don't attempt
4203  // vectorization.
4204  if (!TTI->getNumberOfRegisters(true))
4205  return false;
4206 
4207  // Don't vectorize when the attribute NoImplicitFloat is used.
4208  if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4209  return false;
4210 
4211  DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4212 
4213  // Use the bottom up slp vectorizer to construct chains that start with
4214  // store instructions.
4215  BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4216 
4217  // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4218  // delete instructions.
4219 
4220  // Scan the blocks in the function in post order.
4221  for (auto BB : post_order(&F.getEntryBlock())) {
4222  collectSeedInstructions(BB);
4223 
4224  // Vectorize trees that end at stores.
4225  if (!Stores.empty()) {
4226  DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4227  << " underlying objects.\n");
4228  Changed |= vectorizeStoreChains(R);
4229  }
4230 
4231  // Vectorize trees that end at reductions.
4232  Changed |= vectorizeChainsInBlock(BB, R);
4233 
4234  // Vectorize the index computations of getelementptr instructions. This
4235  // is primarily intended to catch gather-like idioms ending at
4236  // non-consecutive loads.
4237  if (!GEPs.empty()) {
4238  DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4239  << " underlying objects.\n");
4240  Changed |= vectorizeGEPIndices(BB, R);
4241  }
4242  }
4243 
4244  if (Changed) {
4245  R.optimizeGatherSequence();
4246  DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4247  DEBUG(verifyFunction(F));
4248  }
4249  return Changed;
4250 }
4251 
4252 /// \brief Check that the Values in the slice in VL array are still existent in
4253 /// the WeakTrackingVH array.
4254 /// Vectorization of part of the VL array may cause later values in the VL array
4255 /// to become invalid. We track when this has happened in the WeakTrackingVH
4256 /// array.
4258  ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4259  unsigned SliceSize) {
4260  VL = VL.slice(SliceBegin, SliceSize);
4261  VH = VH.slice(SliceBegin, SliceSize);
4262  return !std::equal(VL.begin(), VL.end(), VH.begin());
4263 }
4264 
4265 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4266  unsigned VecRegSize) {
4267  unsigned ChainLen = Chain.size();
4268  DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4269  << "\n");
4270  unsigned Sz = R.getVectorElementSize(Chain[0]);
4271  unsigned VF = VecRegSize / Sz;
4272 
4273  if (!isPowerOf2_32(Sz) || VF < 2)
4274  return false;
4275 
4276  // Keep track of values that were deleted by vectorizing in the loop below.
4277  SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4278 
4279  bool Changed = false;
4280  // Look for profitable vectorizable trees at all offsets, starting at zero.
4281  for (unsigned i = 0, e = ChainLen; i < e; ++i) {
4282  if (i + VF > e)
4283  break;
4284 
4285  // Check that a previous iteration of this loop did not delete the Value.
4286  if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4287  continue;
4288 
4289  DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4290  << "\n");
4291  ArrayRef<Value *> Operands = Chain.slice(i, VF);
4292 
4293  R.buildTree(Operands);
4294  if (R.isTreeTinyAndNotFullyVectorizable())
4295  continue;
4296 
4297  R.computeMinimumValueSizes();
4298 
4299  int Cost = R.getTreeCost();
4300 
4301  DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
4302  if (Cost < -SLPCostThreshold) {
4303  DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4304 
4305  using namespace ore;
4306 
4307  R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4308  cast<StoreInst>(Chain[i]))
4309  << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4310  << " and with tree size "
4311  << NV("TreeSize", R.getTreeSize()));
4312 
4313  R.vectorizeTree();
4314 
4315  // Move to the next bundle.
4316  i += VF - 1;
4317  Changed = true;
4318  }
4319  }
4320 
4321  return Changed;
4322 }
4323 
4324 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4325  BoUpSLP &R) {
4326  SetVector<StoreInst *> Heads;
4328  SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4329 
4330  // We may run into multiple chains that merge into a single chain. We mark the
4331  // stores that we vectorized so that we don't visit the same store twice.
4332  BoUpSLP::ValueSet VectorizedStores;
4333  bool Changed = false;
4334 
4335  // Do a quadratic search on all of the given stores in reverse order and find
4336  // all of the pairs of stores that follow each other.
4337  SmallVector<unsigned, 16> IndexQueue;
4338  unsigned E = Stores.size();
4339  IndexQueue.resize(E - 1);
4340  for (unsigned I = E; I > 0; --I) {
4341  unsigned Idx = I - 1;
4342  // If a store has multiple consecutive store candidates, search Stores
4343  // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
4344  // This is because usually pairing with immediate succeeding or preceding
4345  // candidate create the best chance to find slp vectorization opportunity.
4346  unsigned Offset = 1;
4347  unsigned Cnt = 0;
4348  for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
4349  if (Idx >= Offset) {
4350  IndexQueue[Cnt] = Idx - Offset;
4351  ++Cnt;
4352  }
4353  if (Idx + Offset < E) {
4354  IndexQueue[Cnt] = Idx + Offset;
4355  ++Cnt;
4356  }
4357  }
4358 
4359  for (auto K : IndexQueue) {
4360  if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
4361  Tails.insert(Stores[Idx]);
4362  Heads.insert(Stores[K]);
4363  ConsecutiveChain[Stores[K]] = Stores[Idx];
4364  break;
4365  }
4366  }
4367  }
4368 
4369  // For stores that start but don't end a link in the chain:
4370  for (auto *SI : llvm::reverse(Heads)) {
4371  if (Tails.count(SI))
4372  continue;
4373 
4374  // We found a store instr that starts a chain. Now follow the chain and try
4375  // to vectorize it.
4376  BoUpSLP::ValueList Operands;
4377  StoreInst *I = SI;
4378  // Collect the chain into a list.
4379  while ((Tails.count(I) || Heads.count(I)) && !VectorizedStores.count(I)) {
4380  Operands.push_back(I);
4381  // Move to the next value in the chain.
4382  I = ConsecutiveChain[I];
4383  }
4384 
4385  // FIXME: Is division-by-2 the correct step? Should we assert that the
4386  // register size is a power-of-2?
4387  for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4388  Size /= 2) {
4389  if (vectorizeStoreChain(Operands, R, Size)) {
4390  // Mark the vectorized stores so that we don't vectorize them again.
4391  VectorizedStores.insert(Operands.begin(), Operands.end());
4392  Changed = true;
4393  break;
4394  }
4395  }
4396  }
4397 
4398  return Changed;
4399 }
4400 
4401 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4402  // Initialize the collections. We will make a single pass over the block.
4403  Stores.clear();
4404  GEPs.clear();
4405 
4406  // Visit the store and getelementptr instructions in BB and organize them in
4407  // Stores and GEPs according to the underlying objects of their pointer
4408  // operands.
4409  for (Instruction &I : *BB) {
4410  // Ignore store instructions that are volatile or have a pointer operand
4411  // that doesn't point to a scalar type.
4412  if (auto *SI = dyn_cast<StoreInst>(&I)) {
4413  if (!SI->isSimple())
4414  continue;
4415  if (!isValidElementType(SI->getValueOperand()->getType()))
4416  continue;
4417  Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4418  }
4419 
4420  // Ignore getelementptr instructions that have more than one index, a
4421  // constant index, or a pointer operand that doesn't point to a scalar
4422  // type.
4423  else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4424  auto Idx = GEP->idx_begin()->get();
4425  if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4426  continue;
4427  if (!isValidElementType(Idx->getType()))
4428  continue;
4429  if (GEP->getType()->isVectorTy())
4430  continue;
4431  GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4432  }
4433  }
4434 }
4435 
4436 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4437  if (!A || !B)
4438  return false;
4439  Value *VL[] = { A, B };
4440  return tryToVectorizeList(VL, R, true);
4441 }
4442 
4443 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4444  bool AllowReorder) {
4445  if (VL.size() < 2)
4446  return false;
4447 
4448  DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()
4449  << ".\n");
4450 
4451  // Check that all of the parts are scalar instructions of the same type.
4452  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4453  if (!I0)
4454  return false;
4455 
4456  unsigned Opcode0 = I0->getOpcode();
4457 
4458  unsigned Sz = R.getVectorElementSize(I0);
4459  unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4460  unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4461  if (MaxVF < 2) {
4462  R.getORE()->emit([&]() {
4463  return OptimizationRemarkMissed(
4464  SV_NAME, "SmallVF", I0)
4465  << "Cannot SLP vectorize list: vectorization factor "
4466  << "less than 2 is not supported";
4467  });
4468  return false;
4469  }
4470 
4471  for (Value *V : VL) {
4472  Type *Ty = V->getType();
4473  if (!isValidElementType(Ty)) {
4474  // NOTE: the following will give user internal llvm type name, which may not be useful
4475  R.getORE()->emit([&]() {
4476  std::string type_str;
4477  llvm::raw_string_ostream rso(type_str);
4478  Ty->print(rso);
4479  return OptimizationRemarkMissed(
4480  SV_NAME, "UnsupportedType", I0)
4481  << "Cannot SLP vectorize list: type "
4482  << rso.str() + " is unsupported by vectorizer";
4483  });
4484  return false;
4485  }
4486  Instruction *Inst = dyn_cast<Instruction>(V);
4487 
4488  if (!Inst)
4489  return false;
4490  if (Inst->getOpcode() != Opcode0) {
4491  R.getORE()->emit([&]() {
4492  return OptimizationRemarkMissed(
4493  SV_NAME, "InequableTypes", I0)
4494  << "Cannot SLP vectorize list: not all of the "
4495  << "parts of scalar instructions are of the same type: "
4496  << ore::NV("Instruction1Opcode", I0) << " and "
4497  << ore::NV("Instruction2Opcode", Inst);
4498  });
4499  return false;
4500  }
4501  }
4502 
4503  bool Changed = false;
4504  bool CandidateFound = false;
4505  int MinCost = SLPCostThreshold;
4506 
4507  // Keep track of values that were deleted by vectorizing in the loop below.
4508  SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4509 
4510  unsigned NextInst = 0, MaxInst = VL.size();
4511  for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4512  VF /= 2) {
4513  // No actual vectorization should happen, if number of parts is the same as
4514  // provided vectorization factor (i.e. the scalar type is used for vector
4515  // code during codegen).
4516  auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4517  if (TTI->getNumberOfParts(VecTy) == VF)
4518  continue;
4519  for (unsigned I = NextInst; I < MaxInst; ++I) {
4520  unsigned OpsWidth = 0;
4521 
4522  if (I + VF > MaxInst)
4523  OpsWidth = MaxInst - I;
4524  else
4525  OpsWidth = VF;
4526 
4527  if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4528  break;
4529 
4530  // Check that a previous iteration of this loop did not delete the Value.
4531  if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4532  continue;
4533 
4534  DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4535  << "\n");
4536  ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4537 
4538  R.buildTree(Ops);
4539  // TODO: check if we can allow reordering for more cases.
4540  if (AllowReorder && R.shouldReorder()) {
4541  // Conceptually, there is nothing actually preventing us from trying to
4542  // reorder a larger list. In fact, we do exactly this when vectorizing
4543  // reductions. However, at this point, we only expect to get here when
4544  // there are exactly two operations.
4545  assert(Ops.size() == 2);
4546  Value *ReorderedOps[] = {Ops[1], Ops[0]};
4547  R.buildTree(ReorderedOps, None);
4548  }
4549  if (R.isTreeTinyAndNotFullyVectorizable())
4550  continue;
4551 
4552  R.computeMinimumValueSizes();
4553  int Cost = R.getTreeCost();
4554  CandidateFound = true;
4555  MinCost = std::min(MinCost, Cost);
4556 
4557  if (Cost < -SLPCostThreshold) {
4558  DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4559  R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4560  cast<Instruction>(Ops[0]))
4561  << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4562  << " and with tree size "
4563  << ore::NV("TreeSize", R.getTreeSize()));
4564 
4565  R.vectorizeTree();
4566  // Move to the next bundle.
4567  I += VF - 1;
4568  NextInst = I + 1;
4569  Changed = true;
4570  }
4571  }
4572  }
4573 
4574  if (!Changed && CandidateFound) {
4575  R.getORE()->emit([&]() {
4576  return OptimizationRemarkMissed(
4577  SV_NAME, "NotBeneficial", I0)
4578  << "List vectorization was possible but not beneficial with cost "
4579  << ore::NV("Cost", MinCost) << " >= "
4580  << ore::NV("Treshold", -SLPCostThreshold);
4581  });
4582  } else if (!Changed) {
4583  R.getORE()->emit([&]() {
4584  return OptimizationRemarkMissed(
4585  SV_NAME, "NotPossible", I0)
4586  << "Cannot SLP vectorize list: vectorization was impossible"
4587  << " with available vectorization factors";
4588  });
4589  }
4590  return Changed;
4591 }
4592 
4593 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4594  if (!I)
4595  return false;
4596 
4597  if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4598  return false;
4599 
4600  Value *P = I->getParent();
4601 
4602  // Vectorize in current basic block only.
4603  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4604  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4605  if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4606  return false;
4607 
4608  // Try to vectorize V.
4609  if (tryToVectorizePair(Op0, Op1, R))
4610  return true;
4611 
4612  auto *A = dyn_cast<BinaryOperator>(Op0);
4613  auto *B = dyn_cast<BinaryOperator>(Op1);
4614  // Try to skip B.
4615  if (B && B->hasOneUse()) {
4616  auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4617  auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4618  if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4619  return true;
4620  if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4621  return true;
4622  }
4623 
4624  // Try to skip A.
4625  if (A && A->hasOneUse()) {
4626  auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4627  auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4628  if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4629  return true;
4630  if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4631  return true;
4632  }
4633  return false;
4634 }
4635 
4636 /// \brief Generate a shuffle mask to be used in a reduction tree.
4637 ///
4638 /// \param VecLen The length of the vector to be reduced.
4639 /// \param NumEltsToRdx The number of elements that should be reduced in the
4640 /// vector.
4641 /// \param IsPairwise Whether the reduction is a pairwise or splitting
4642 /// reduction. A pairwise reduction will generate a mask of
4643 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
4644 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4645 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
4646 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4647  bool IsPairwise, bool IsLeft,
4648  IRBuilder<> &Builder) {
4649  assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
4650 
4651  SmallVector<Constant *, 32> ShuffleMask(
4652  VecLen, UndefValue::get(Builder.getInt32Ty()));
4653 
4654  if (IsPairwise)
4655  // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4656  for (unsigned i = 0; i != NumEltsToRdx; ++i)
4657  ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4658  else
4659  // Move the upper half of the vector to the lower half.
4660  for (unsigned i = 0; i != NumEltsToRdx; ++i)
4661  ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4662 
4663  return ConstantVector::get(ShuffleMask);
4664 }
4665 
4666 namespace {
4667 
4668 /// Model horizontal reductions.
4669 ///
4670 /// A horizontal reduction is a tree of reduction operations (currently add and
4671 /// fadd) that has operations that can be put into a vector as its leaf.
4672 /// For example, this tree:
4673 ///
4674 /// mul mul mul mul
4675 /// \ / \ /
4676 /// + +
4677 /// \ /
4678 /// +
4679 /// This tree has "mul" as its reduced values and "+" as its reduction
4680 /// operations. A reduction might be feeding into a store or a binary operation
4681 /// feeding a phi.
4682 /// ...
4683 /// \ /
4684 /// +
4685 /// |
4686 /// phi +=
4687 ///
4688 /// Or:
4689 /// ...
4690 /// \ /
4691 /// +
4692 /// |
4693 /// *p =
4694 ///
4695 class HorizontalReduction {
4696  using ReductionOpsType = SmallVector<Value *, 16>;
4697  using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
4698  ReductionOpsListType ReductionOps;
4699  SmallVector<Value *, 32> ReducedVals;
4700  // Use map vector to make stable output.
4702 
4703  /// Kind of the reduction data.
4704  enum ReductionKind {
4705  RK_None, /// Not a reduction.
4706  RK_Arithmetic, /// Binary reduction data.
4707  RK_Min, /// Minimum reduction data.
4708  RK_UMin, /// Unsigned minimum reduction data.
4709  RK_Max, /// Maximum reduction data.
4710  RK_UMax, /// Unsigned maximum reduction data.
4711  };
4712 
4713  /// Contains info about operation, like its opcode, left and right operands.
4714  class OperationData {
4715  /// Opcode of the instruction.
4716  unsigned Opcode = 0;
4717 
4718  /// Left operand of the reduction operation.
4719  Value *LHS = nullptr;
4720 
4721  /// Right operand of the reduction operation.
4722  Value *RHS = nullptr;
4723 
4724  /// Kind of the reduction operation.
4725  ReductionKind Kind = RK_None;
4726 
4727  /// True if float point min/max reduction has no NaNs.
4728  bool NoNaN = false;
4729 
4730  /// Checks if the reduction operation can be vectorized.
4731  bool isVectorizable() const {
4732  return LHS && RHS &&
4733  // We currently only support adds && min/max reductions.
4734  ((Kind == RK_Arithmetic &&
4735  (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
4736  ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
4737  (Kind == RK_Min || Kind == RK_Max)) ||
4738  (Opcode == Instruction::ICmp &&
4739  (Kind == RK_UMin || Kind == RK_UMax)));
4740  }
4741 
4742  /// Creates reduction operation with the current opcode.
4743  Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
4744  assert(isVectorizable() &&
4745  "Expected add|fadd or min/max reduction operation.");
4746  Value *Cmp;
4747  switch (Kind) {
4748  case RK_Arithmetic:
4749  return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
4750  Name);
4751  case RK_Min:
4752  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
4753  : Builder.CreateFCmpOLT(LHS, RHS);
4754  break;
4755  case RK_Max:
4756  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
4757  : Builder.CreateFCmpOGT(LHS, RHS);
4758  break;
4759  case RK_UMin:
4760  assert(Opcode == Instruction::ICmp && "Expected integer types.");
4761  Cmp = Builder.CreateICmpULT(LHS, RHS);
4762  break;
4763  case RK_UMax:
4764  assert(Opcode == Instruction::ICmp && "Expected integer types.");
4765  Cmp = Builder.CreateICmpUGT(LHS, RHS);
4766  break;
4767  case RK_None:
4768  llvm_unreachable("Unknown reduction operation.");
4769  }
4770  return Builder.CreateSelect(Cmp, LHS, RHS, Name);
4771  }
4772 
4773  public:
4774  explicit OperationData() = default;
4775 
4776  /// Construction for reduced values. They are identified by opcode only and
4777  /// don't have associated LHS/RHS values.
4778  explicit OperationData(Value *V) {
4779  if (auto *I = dyn_cast<Instruction>(V))
4780  Opcode = I->getOpcode();
4781  }
4782 
4783  /// Constructor for reduction operations with opcode and its left and
4784  /// right operands.
4785  OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
4786  bool NoNaN = false)
4787  : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
4788  assert(Kind != RK_None && "One of the reduction operations is expected.");
4789  }
4790 
4791  explicit operator bool() const { return Opcode; }
4792 
4793  /// Get the index of the first operand.
4794  unsigned getFirstOperandIndex() const {
4795  assert(!!*this && "The opcode is not set.");
4796  switch (Kind) {
4797  case RK_Min:
4798  case RK_UMin:
4799  case RK_Max:
4800  case RK_UMax:
4801  return 1;
4802  case RK_Arithmetic:
4803  case RK_None:
4804  break;
4805  }
4806  return 0;
4807  }
4808 
4809  /// Total number of operands in the reduction operation.
4810  unsigned getNumberOfOperands() const {
4811  assert(Kind != RK_None && !!*this && LHS && RHS &&
4812  "Expected reduction operation.");
4813  switch (Kind) {
4814  case RK_Arithmetic:
4815  return 2;
4816  case RK_Min:
4817  case RK_UMin:
4818  case RK_Max:
4819  case RK_UMax:
4820  return 3;
4821  case RK_None:
4822  break;
4823  }
4824  llvm_unreachable("Reduction kind is not set");
4825  }
4826 
4827  /// Checks if the operation has the same parent as \p P.
4828  bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
4829  assert(Kind != RK_None && !!*this && LHS && RHS &&
4830  "Expected reduction operation.");
4831  if (!IsRedOp)
4832  return I->getParent() == P;
4833  switch (Kind) {
4834  case RK_Arithmetic:
4835  // Arithmetic reduction operation must be used once only.
4836  return I->getParent() == P;
4837  case RK_Min:
4838  case RK_UMin:
4839  case RK_Max:
4840  case RK_UMax: {
4841  // SelectInst must be used twice while the condition op must have single
4842  // use only.
4843  auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
4844  return I->getParent() == P && Cmp && Cmp->getParent() == P;
4845  }
4846  case RK_None:
4847  break;
4848  }
4849  llvm_unreachable("Reduction kind is not set");
4850  }
4851  /// Expected number of uses for reduction operations/reduced values.
4852  bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
4853  assert(Kind != RK_None && !!*this && LHS && RHS &&
4854  "Expected reduction operation.");
4855  switch (Kind) {
4856  case RK_Arithmetic:
4857  return I->hasOneUse();
4858  case RK_Min:
4859  case RK_UMin:
4860  case RK_Max:
4861  case RK_UMax:
4862  return I->hasNUses(2) &&
4863  (!IsReductionOp ||
4864  cast<SelectInst>(I)->getCondition()->hasOneUse());
4865  case RK_None:
4866  break;
4867  }
4868  llvm_unreachable("Reduction kind is not set");
4869  }
4870 
4871  /// Initializes the list of reduction operations.
4872  void initReductionOps(ReductionOpsListType &ReductionOps) {
4873  assert(Kind != RK_None && !!*this && LHS && RHS &&
4874  "Expected reduction operation.");
4875  switch (Kind) {
4876  case RK_Arithmetic:
4877  ReductionOps.assign(1, ReductionOpsType());
4878  break;
4879  case RK_Min:
4880  case RK_UMin:
4881  case RK_Max:
4882  case RK_UMax:
4883  ReductionOps.assign(2, ReductionOpsType());
4884  break;
4885  case RK_None:
4886  llvm_unreachable("Reduction kind is not set");
4887  }
4888  }
4889  /// Add all reduction operations for the reduction instruction \p I.
4890  void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
4891  assert(Kind != RK_None && !!*this && LHS && RHS &&
4892  "Expected reduction operation.");
4893  switch (Kind) {
4894  case RK_Arithmetic:
4895  ReductionOps[0].emplace_back(I);
4896  break;
4897  case RK_Min:
4898  case RK_UMin:
4899  case RK_Max:
4900  case RK_UMax:
4901  ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
4902  ReductionOps[1].emplace_back(I);
4903  break;
4904  case RK_None:
4905  llvm_unreachable("Reduction kind is not set");
4906  }
4907  }
4908 
4909  /// Checks if instruction is associative and can be vectorized.
4910  bool isAssociative(Instruction *I) const {
4911  assert(Kind != RK_None && *this && LHS && RHS &&
4912  "Expected reduction operation.");
4913  switch (Kind) {
4914  case RK_Arithmetic:
4915  return I->isAssociative();
4916  case RK_Min:
4917  case RK_Max:
4918  return Opcode == Instruction::ICmp ||
4919  cast<Instruction>(I->getOperand(0))->isFast();
4920  case RK_UMin:
4921  case RK_UMax:
4922  assert(Opcode == Instruction::ICmp &&
4923  "Only integer compare operation is expected.");
4924  return true;
4925  case RK_None:
4926  break;
4927  }
4928  llvm_unreachable("Reduction kind is not set");
4929  }
4930 
4931  /// Checks if the reduction operation can be vectorized.
4932  bool isVectorizable(Instruction *I) const {
4933  return isVectorizable() && isAssociative(I);
4934  }
4935 
4936  /// Checks if two operation data are both a reduction op or both a reduced
4937  /// value.
4938  bool operator==(const OperationData &OD) {
4939  assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
4940  "One of the comparing operations is incorrect.");
4941  return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
4942  }
4943  bool operator!=(const OperationData &OD) { return !(*this == OD); }
4944  void clear() {
4945  Opcode = 0;
4946  LHS = nullptr;
4947  RHS = nullptr;
4948  Kind = RK_None;
4949  NoNaN = false;
4950  }
4951 
4952  /// Get the opcode of the reduction operation.
4953  unsigned getOpcode() const {
4954  assert(isVectorizable() && "Expected vectorizable operation.");
4955  return Opcode;
4956  }
4957 
4958  /// Get kind of reduction data.
4959  ReductionKind getKind() const { return Kind; }
4960  Value *getLHS() const { return LHS; }
4961  Value *getRHS() const { return RHS; }
4962  Type *getConditionType() const {
4963  switch (Kind) {
4964  case RK_Arithmetic:
4965  return nullptr;
4966  case RK_Min:
4967  case RK_Max:
4968  case RK_UMin:
4969  case RK_UMax:
4970  return CmpInst::makeCmpResultType(LHS->getType());
4971  case RK_None:
4972  break;
4973  }
4974  llvm_unreachable("Reduction kind is not set");
4975  }
4976 
4977  /// Creates reduction operation with the current opcode with the IR flags
4978  /// from \p ReductionOps.
4979  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
4980  const ReductionOpsListType &ReductionOps) const {
4981  assert(isVectorizable() &&
4982  "Expected add|fadd or min/max reduction operation.");
4983  auto *Op = createOp(Builder, Name);
4984  switch (Kind) {
4985  case RK_Arithmetic:
4986  propagateIRFlags(Op, ReductionOps[0]);
4987  return Op;
4988  case RK_Min:
4989  case RK_Max:
4990  case RK_UMin:
4991  case RK_UMax:
4992  if (auto *SI = dyn_cast<SelectInst>(Op))
4993  propagateIRFlags(SI->getCondition(), ReductionOps[0]);
4994  propagateIRFlags(Op, ReductionOps[1]);
4995  return Op;
4996  case RK_None:
4997  break;
4998  }
4999  llvm_unreachable("Unknown reduction operation.");
5000  }
5001  /// Creates reduction operation with the current opcode with the IR flags
5002  /// from \p I.
5003  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
5004  Instruction *I) const {
5005  assert(isVectorizable() &&
5006  "Expected add|fadd or min/max reduction operation.");
5007  auto *Op = createOp(Builder, Name);
5008  switch (Kind) {
5009  case RK_Arithmetic:
5010  propagateIRFlags(Op, I);
5011  return Op;
5012  case RK_Min:
5013  case RK_Max:
5014  case RK_UMin:
5015  case RK_UMax:
5016  if (auto *SI = dyn_cast<SelectInst>(Op)) {
5017  propagateIRFlags(SI->getCondition(),
5018  cast<SelectInst>(I)->getCondition());
5019  }
5020  propagateIRFlags(Op, I);
5021  return Op;
5022  case RK_None:
5023  break;
5024  }
5025  llvm_unreachable("Unknown reduction operation.");
5026  }
5027 
5028  TargetTransformInfo::ReductionFlags getFlags() const {
5030  Flags.NoNaN = NoNaN;
5031  switch (Kind) {
5032  case RK_Arithmetic:
5033  break;
5034  case RK_Min:
5035  Flags.IsSigned = Opcode == Instruction::ICmp;
5036  Flags.IsMaxOp = false;
5037  break;
5038  case RK_Max:
5039  Flags.IsSigned = Opcode == Instruction::ICmp;
5040  Flags.IsMaxOp = true;
5041  break;
5042  case RK_UMin:
5043  Flags.IsSigned = false;
5044  Flags.IsMaxOp = false;
5045  break;
5046  case RK_UMax:
5047  Flags.IsSigned = false;
5048  Flags.IsMaxOp = true;
5049  break;
5050  case RK_None:
5051  llvm_unreachable("Reduction kind is not set");
5052  }
5053  return Flags;
5054  }
5055  };
5056 
5057  Instruction *ReductionRoot = nullptr;
5058 
5059  /// The operation data of the reduction operation.
5060  OperationData ReductionData;
5061 
5062  /// The operation data of the values we perform a reduction on.
5063  OperationData ReducedValueData;
5064 
5065  /// Should we model this reduction as a pairwise reduction tree or a tree that
5066  /// splits the vector in halves and adds those halves.
5067  bool IsPairwiseReduction = false;
5068 
5069  /// Checks if the ParentStackElem.first should be marked as a reduction
5070  /// operation with an extra argument or as extra argument itself.
5071  void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5072  Value *ExtraArg) {
5073  if (ExtraArgs.count(ParentStackElem.first)) {
5074  ExtraArgs[ParentStackElem.first] = nullptr;
5075  // We ran into something like:
5076  // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5077  // The whole ParentStackElem.first should be considered as an extra value
5078  // in this case.
5079  // Do not perform analysis of remaining operands of ParentStackElem.first
5080  // instruction, this whole instruction is an extra argument.
5081  ParentStackElem.second = ParentStackElem.first->getNumOperands();
5082  } else {
5083  // We ran into something like:
5084  // ParentStackElem.first += ... + ExtraArg + ...
5085  ExtraArgs[ParentStackElem.first] = ExtraArg;
5086  }
5087  }
5088 
5089  static OperationData getOperationData(Value *V) {
5090  if (!V)
5091  return OperationData();
5092 
5093  Value *LHS;
5094  Value *RHS;
5095  if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5096  return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5097  RK_Arithmetic);
5098  }
5099  if (auto *Select = dyn_cast<SelectInst>(V)) {
5100  // Look for a min/max pattern.
5101  if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5102  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5103  } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5104  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5105  } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5106  m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5107  return OperationData(
5108  Instruction::FCmp, LHS, RHS, RK_Min,
5109  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5110  } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5111  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5112  } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5113  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5114  } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5115  m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5116  return OperationData(
5117  Instruction::FCmp, LHS, RHS, RK_Max,
5118  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5119  }
5120  }
5121  return OperationData(V);
5122  }
5123 
5124 public:
5125  HorizontalReduction() = default;
5126 
5127  /// \brief Try to find a reduction tree.
5128  bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5129  assert((!Phi || is_contained(Phi->operands(), B)) &&
5130  "Thi phi needs to use the binary operator");
5131 
5132  ReductionData = getOperationData(B);
5133 
5134  // We could have a initial reductions that is not an add.
5135  // r *= v1 + v2 + v3 + v4
5136  // In such a case start looking for a tree rooted in the first '+'.
5137  if (Phi) {
5138  if (ReductionData.getLHS() == Phi) {
5139  Phi = nullptr;
5140  B = dyn_cast<Instruction>(ReductionData.getRHS());
5141  ReductionData = getOperationData(B);
5142  } else if (ReductionData.getRHS() == Phi) {
5143  Phi = nullptr;
5144  B = dyn_cast<Instruction>(ReductionData.getLHS());
5145  ReductionData = getOperationData(B);
5146  }
5147  }
5148 
5149  if (!ReductionData.isVectorizable(B))
5150  return false;
5151 
5152  Type *Ty = B->getType();
5153  if (!isValidElementType(Ty))
5154  return false;
5155 
5156  ReducedValueData.clear();
5157  ReductionRoot = B;
5158 
5159  // Post order traverse the reduction tree starting at B. We only handle true
5160  // trees containing only binary operators.
5162  Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5163  ReductionData.initReductionOps(ReductionOps);
5164  while (!Stack.empty()) {
5165  Instruction *TreeN = Stack.back().first;
5166  unsigned EdgeToVist = Stack.back().second++;
5167  OperationData OpData = getOperationData(TreeN);
5168  bool IsReducedValue = OpData != ReductionData;
5169 
5170  // Postorder vist.
5171  if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5172  if (IsReducedValue)
5173  ReducedVals.push_back(TreeN);
5174  else {
5175  auto I = ExtraArgs.find(TreeN);
5176  if (I != ExtraArgs.end() && !I->second) {
5177  // Check if TreeN is an extra argument of its parent operation.
5178  if (Stack.size() <= 1) {
5179  // TreeN can't be an extra argument as it is a root reduction
5180  // operation.
5181  return false;
5182  }
5183  // Yes, TreeN is an extra argument, do not add it to a list of
5184  // reduction operations.
5185  // Stack[Stack.size() - 2] always points to the parent operation.
5186  markExtraArg(Stack[Stack.size() - 2], TreeN);
5187  ExtraArgs.erase(TreeN);
5188  } else
5189  ReductionData.addReductionOps(TreeN, ReductionOps);
5190  }
5191  // Retract.
5192  Stack.pop_back();
5193  continue;
5194  }
5195 
5196  // Visit left or right.
5197  Value *NextV = TreeN->getOperand(EdgeToVist);
5198  if (NextV != Phi) {
5199  auto *I = dyn_cast<Instruction>(NextV);
5200  OpData = getOperationData(I);
5201  // Continue analysis if the next operand is a reduction operation or
5202  // (possibly) a reduced value. If the reduced value opcode is not set,
5203  // the first met operation != reduction operation is considered as the
5204  // reduced value class.
5205  if (I && (!ReducedValueData || OpData == ReducedValueData ||
5206  OpData == ReductionData)) {
5207  const bool IsReductionOperation = OpData == ReductionData;
5208  // Only handle trees in the current basic block.
5209  if (!ReductionData.hasSameParent(I, B->getParent(),
5210  IsReductionOperation)) {
5211  // I is an extra argument for TreeN (its parent operation).
5212  markExtraArg(Stack.back(), I);
5213  continue;
5214  }
5215 
5216  // Each tree node needs to have minimal number of users except for the
5217  // ultimate reduction.
5218  if (!ReductionData.hasRequiredNumberOfUses(I,
5219  OpData == ReductionData) &&
5220  I != B) {
5221  // I is an extra argument for TreeN (its parent operation).
5222  markExtraArg(Stack.back(), I);
5223  continue;
5224  }
5225 
5226  if (IsReductionOperation) {
5227  // We need to be able to reassociate the reduction operations.
5228  if (!OpData.isAssociative(I)) {
5229  // I is an extra argument for TreeN (its parent operation).
5230  markExtraArg(Stack.back(), I);
5231  continue;
5232  }
5233  } else if (ReducedValueData &&
5234  ReducedValueData != OpData) {
5235  // Make sure that the opcodes of the operations that we are going to
5236  // reduce match.
5237  // I is an extra argument for TreeN (its parent operation).
5238  markExtraArg(Stack.back(), I);
5239  continue;
5240  } else if (!ReducedValueData)
5241  ReducedValueData = OpData;
5242 
5243  Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5244  continue;
5245  }
5246  }
5247  // NextV is an extra argument for TreeN (its parent operation).
5248  markExtraArg(Stack.back(), NextV);
5249  }
5250  return true;
5251  }
5252 
5253  /// \brief Attempt to vectorize the tree found by
5254  /// matchAssociativeReduction.
5255  bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5256  if (ReducedVals.empty())
5257  return false;
5258 
5259  // If there is a sufficient number of reduction values, reduce
5260  // to a nearby power-of-2. Can safely generate oversized
5261  // vectors and rely on the backend to split them to legal sizes.
5262  unsigned NumReducedVals = ReducedVals.size();
5263  if (NumReducedVals < 4)
5264  return false;
5265 
5266  unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5267 
5268  Value *VectorizedTree = nullptr;
5269  IRBuilder<> Builder(ReductionRoot);
5270  FastMathFlags Unsafe;
5271  Unsafe.setFast();
5272  Builder.setFastMathFlags(Unsafe);
5273  unsigned i = 0;
5274 
5275  BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5276  // The same extra argument may be used several time, so log each attempt
5277  // to use it.
5278  for (auto &Pair : ExtraArgs)
5279  ExternallyUsedValues[Pair.second].push_back(Pair.first);
5280  SmallVector<Value *, 16> IgnoreList;
5281  for (auto &V : ReductionOps)
5282  IgnoreList.append(V.begin(), V.end());
5283  while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5284  auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5285  V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5286  if (V.shouldReorder()) {
5287  SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
5288  V.buildTree(Reversed, ExternallyUsedValues, IgnoreList);
5289  }
5290  if (V.isTreeTinyAndNotFullyVectorizable())
5291  break;
5292 
5293  V.computeMinimumValueSizes();
5294 
5295  // Estimate cost.
5296  int Cost =
5297  V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5298  if (Cost >= -SLPCostThreshold) {
5299  V.getORE()->emit([&]() {
5300  return OptimizationRemarkMissed(
5301  SV_NAME, "HorSLPNotBeneficial", cast<Instruction>(VL[0]))
5302  << "Vectorizing horizontal reduction is possible"
5303  << "but not beneficial with cost "
5304  << ore::NV("Cost", Cost) << " and threshold "
5305  << ore::NV("Threshold", -SLPCostThreshold);
5306  });
5307  break;
5308  }
5309 
5310  DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Cost
5311  << ". (HorRdx)\n");
5312  V.getORE()->emit([&]() {
5313  return OptimizationRemark(
5314  SV_NAME, "VectorizedHorizontalReduction", cast<Instruction>(VL[0]))
5315  << "Vectorized horizontal reduction with cost "
5316  << ore::NV("Cost", Cost) << " and with tree size "
5317  << ore::NV("TreeSize", V.getTreeSize());
5318  });
5319 
5320  // Vectorize a tree.
5321  DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5322  Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5323 
5324  // Emit a reduction.
5325  Value *ReducedSubTree =
5326  emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5327  if (VectorizedTree) {
5328  Builder.SetCurrentDebugLocation(Loc);
5329  OperationData VectReductionData(ReductionData.getOpcode(),
5330  VectorizedTree, ReducedSubTree,
5331  ReductionData.getKind());
5332  VectorizedTree =
5333  VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5334  } else
5335  VectorizedTree = ReducedSubTree;
5336  i += ReduxWidth;
5337  ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5338  }
5339 
5340  if (VectorizedTree) {
5341  // Finish the reduction.
5342  for (; i < NumReducedVals; ++i) {
5343  auto *I = cast<Instruction>(ReducedVals[i]);
5344  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5345  OperationData VectReductionData(ReductionData.getOpcode(),
5346  VectorizedTree, I,
5347  ReductionData.getKind());
5348  VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5349  }
5350  for (auto &Pair : ExternallyUsedValues) {
5351  assert(!Pair.second.empty() &&
5352  "At least one DebugLoc must be inserted");
5353  // Add each externally used value to the final reduction.
5354  for (auto *I : Pair.second) {
5355  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5356  OperationData VectReductionData(ReductionData.getOpcode(),
5357  VectorizedTree, Pair.first,
5358  ReductionData.getKind());
5359  VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5360  }
5361  }
5362  // Update users.
5363  ReductionRoot->replaceAllUsesWith(VectorizedTree);
5364  }
5365  return VectorizedTree != nullptr;
5366  }
5367 
5368  unsigned numReductionValues() const {
5369  return ReducedVals.size();
5370  }
5371 
5372 private:
5373  /// \brief Calculate the cost of a reduction.
5374  int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5375  unsigned ReduxWidth) {
5376  Type *ScalarTy = FirstReducedVal->getType();
5377  Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5378 
5379  int PairwiseRdxCost;
5380  int SplittingRdxCost;
5381  switch (ReductionData.getKind()) {
5382  case RK_Arithmetic:
5383  PairwiseRdxCost =
5384  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5385  /*IsPairwiseForm=*/true);
5386  SplittingRdxCost =
5387  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5388  /*IsPairwiseForm=*/false);
5389  break;
5390  case RK_Min:
5391  case RK_Max:
5392  case RK_UMin:
5393  case RK_UMax: {
5394  Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5395  bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5396  ReductionData.getKind() == RK_UMax;
5397  PairwiseRdxCost =
5398  TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5399  /*IsPairwiseForm=*/true, IsUnsigned);
5400  SplittingRdxCost =
5401  TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5402  /*IsPairwiseForm=*/false, IsUnsigned);
5403  break;
5404  }
5405  case RK_None:
5406  llvm_unreachable("Expected arithmetic or min/max reduction operation");
5407  }
5408 
5409  IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5410  int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5411 
5412  int ScalarReduxCost;
5413  switch (ReductionData.getKind()) {
5414  case RK_Arithmetic:
5415  ScalarReduxCost =
5416  TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5417  break;
5418  case RK_Min:
5419  case RK_Max:
5420  case RK_UMin:
5421  case RK_UMax:
5422  ScalarReduxCost =
5423  TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5424  TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5425  CmpInst::makeCmpResultType(ScalarTy));
5426  break;
5427  case RK_None:
5428  llvm_unreachable("Expected arithmetic or min/max reduction operation");
5429  }
5430  ScalarReduxCost *= (ReduxWidth - 1);
5431 
5432  DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost
5433  << " for reduction that starts with " << *FirstReducedVal
5434  << " (It is a "
5435  << (IsPairwiseReduction ? "pairwise" : "splitting")
5436  << " reduction)\n");
5437 
5438  return VecReduxCost - ScalarReduxCost;
5439  }
5440 
5441  /// \brief Emit a horizontal reduction of the vectorized value.
5442  Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5443  unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5444  assert(VectorizedValue && "Need to have a vectorized tree node");
5445  assert(isPowerOf2_32(ReduxWidth) &&
5446  "We only handle power-of-two reductions for now");
5447 
5448  if (!IsPairwiseReduction)
5450  Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5451  ReductionData.getFlags(), ReductionOps.back());
5452 
5453  Value *TmpVec = VectorizedValue;
5454  for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5455  Value *LeftMask =
5456  createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5457  Value *RightMask =
5458  createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5459 
5460  Value *LeftShuf = Builder.CreateShuffleVector(
5461  TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5462  Value *RightShuf = Builder.CreateShuffleVector(
5463  TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5464  "rdx.shuf.r");
5465  OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5466  RightShuf, ReductionData.getKind());
5467  TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5468  }
5469 
5470  // The result is in the first element of the vector.
5471  return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5472  }
5473 };
5474 
5475 } // end anonymous namespace
5476 
5477 /// \brief Recognize construction of vectors like
5478 /// %ra = insertelement <4 x float> undef, float %s0, i32 0
5479 /// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5480 /// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5481 /// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5482 /// starting from the last insertelement instruction.
5483 ///
5484 /// Returns true if it matches
5485 static bool findBuildVector(InsertElementInst *LastInsertElem,
5486  SmallVectorImpl<Value *> &BuildVectorOpds) {
5487  Value *V = nullptr;
5488  do {
5489  BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5490  V = LastInsertElem->getOperand(0);
5491  if (isa<UndefValue>(V))
5492  break;
5493  LastInsertElem = dyn_cast<InsertElementInst>(V);
5494  if (!LastInsertElem || !LastInsertElem->hasOneUse())
5495  return false;
5496  } while (true);
5497  std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5498  return true;
5499 }
5500 
5501 /// \brief Like findBuildVector, but looks for construction of aggregate.
5502 ///
5503 /// \return true if it matches.
5505  SmallVectorImpl<Value *> &BuildVectorOpds) {
5506  Value *V;
5507  do {
5508  BuildVectorOpds.push_back(IV->getInsertedValueOperand());
5509  V = IV->getAggregateOperand();
5510  if (isa<UndefValue>(V))
5511  break;
5512  IV = dyn_cast<InsertValueInst>(V);
5513  if (!IV || !IV->hasOneUse())
5514  return false;
5515  } while (true);
5516  std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5517  return true;
5518 }
5519 
5520 static bool PhiTypeSorterFunc(Value *V, Value *V2) {
5521  return V->getType() < V2->getType();
5522 }
5523 
5524 /// \brief Try and get a reduction value from a phi node.
5525 ///
5526 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions
5527 /// if they come from either \p ParentBB or a containing loop latch.
5528 ///
5529 /// \returns A candidate reduction value if possible, or \code nullptr \endcode
5530 /// if not possible.
5532  BasicBlock *ParentBB, LoopInfo *LI) {
5533  // There are situations where the reduction value is not dominated by the
5534  // reduction phi. Vectorizing such cases has been reported to cause
5535  // miscompiles. See PR25787.
5536  auto DominatedReduxValue = [&](Value *R) {
5537  return (
5538  dyn_cast<Instruction>(R) &&
5539  DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
5540  };
5541 
5542  Value *Rdx = nullptr;
5543 
5544  // Return the incoming value if it comes from the same BB as the phi node.
5545  if (P->getIncomingBlock(0) == ParentBB) {
5546  Rdx = P->getIncomingValue(0);
5547  } else if (P->getIncomingBlock(1) == ParentBB) {
5548  Rdx = P->getIncomingValue(1);
5549  }
5550 
5551  if (Rdx && DominatedReduxValue(Rdx))
5552  return Rdx;
5553 
5554  // Otherwise, check whether we have a loop latch to look at.
5555  Loop *BBL = LI->getLoopFor(ParentBB);
5556  if (!BBL)
5557  return nullptr;
5558  BasicBlock *BBLatch = BBL->getLoopLatch();
5559  if (!BBLatch)
5560  return nullptr;
5561 
5562  // There is a loop latch, return the incoming value if it comes from
5563  // that. This reduction pattern occasionally turns up.
5564  if (P->getIncomingBlock(0) == BBLatch) {
5565  Rdx = P->getIncomingValue(0);
5566  } else if (P->getIncomingBlock(1) == BBLatch) {
5567  Rdx = P->getIncomingValue(1);
5568  }
5569 
5570  if (Rdx && DominatedReduxValue(Rdx))
5571  return Rdx;
5572 
5573  return nullptr;
5574 }
5575 
5576 /// Attempt to reduce a horizontal reduction.
5577 /// If it is legal to match a horizontal reduction feeding the phi node \a P
5578 /// with reduction operators \a Root (or one of its operands) in a basic block
5579 /// \a BB, then check if it can be done. If horizontal reduction is not found
5580 /// and root instruction is a binary operation, vectorization of the operands is
5581 /// attempted.
5582 /// \returns true if a horizontal reduction was matched and reduced or operands
5583 /// of one of the binary instruction were vectorized.
5584 /// \returns false if a horizontal reduction was not matched (or not possible)
5585 /// or no vectorization of any binary operation feeding \a Root instruction was
5586 /// performed.
5588  PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
5589  TargetTransformInfo *TTI,
5590  const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
5591  if (!ShouldVectorizeHor)
5592  return false;
5593 
5594  if (!Root)
5595  return false;
5596 
5597  if (Root->getParent() != BB || isa<PHINode>(Root))
5598  return false;
5599  // Start analysis starting from Root instruction. If horizontal reduction is
5600  // found, try to vectorize it. If it is not a horizontal reduction or
5601  // vectorization is not possible or not effective, and currently analyzed
5602  // instruction is a binary operation, try to vectorize the operands, using
5603  // pre-order DFS traversal order. If the operands were not vectorized, repeat
5604  // the same procedure considering each operand as a possible root of the
5605  // horizontal reduction.
5606  // Interrupt the process if the Root instruction itself was vectorized or all
5607  // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
5608  SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
5609  SmallSet<Value *, 8> VisitedInstrs;
5610  bool Res = false;
5611  while (!Stack.empty()) {
5612  Value *V;
5613  unsigned Level;
5614  std::tie(V, Level) = Stack.pop_back_val();
5615  if (!V)
5616  continue;
5617  auto *Inst = dyn_cast<Instruction>(V);
5618  if (!Inst)
5619  continue;
5620  auto *BI = dyn_cast<BinaryOperator>(Inst);
5621  auto *SI = dyn_cast<SelectInst>(Inst);
5622  if (BI || SI) {
5623  HorizontalReduction HorRdx;
5624  if (HorRdx.matchAssociativeReduction(P, Inst)) {
5625  if (HorRdx.tryToReduce(R, TTI)) {
5626  Res = true;
5627  // Set P to nullptr to avoid re-analysis of phi node in
5628  // matchAssociativeReduction function unless this is the root node.
5629  P = nullptr;
5630  continue;
5631  }
5632  }
5633  if (P && BI) {
5634  Inst = dyn_cast<Instruction>(BI->getOperand(0));
5635  if (Inst == P)
5636  Inst = dyn_cast<Instruction>(BI->getOperand(1));
5637  if (!Inst) {
5638  // Set P to nullptr to avoid re-analysis of phi node in
5639  // matchAssociativeReduction function unless this is the root node.
5640  P = nullptr;
5641  continue;
5642  }
5643  }
5644  }
5645  // Set P to nullptr to avoid re-analysis of phi node in
5646  // matchAssociativeReduction function unless this is the root node.
5647  P = nullptr;
5648  if (Vectorize(Inst, R)) {
5649  Res = true;
5650  continue;
5651  }
5652 
5653  // Try to vectorize operands.
5654  // Continue analysis for the instruction from the same basic block only to
5655  // save compile time.
5656  if (++Level < RecursionMaxDepth)
5657  for (auto *Op : Inst->operand_values())
5658  if (VisitedInstrs.insert(Op).second)
5659  if (auto *I = dyn_cast<Instruction>(Op))
5660  if (!isa<PHINode>(I) && I->getParent() == BB)
5661  Stack.emplace_back(Op, Level);
5662  }
5663  return Res;
5664 }
5665 
5666 bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
5667  BasicBlock *BB, BoUpSLP &R,
5668  TargetTransformInfo *TTI) {
5669  if (!V)
5670  return false;
5671  auto *I = dyn_cast<Instruction>(V);
5672  if (!I)
5673  return false;
5674 
5675  if (!isa<BinaryOperator>(I))
5676  P = nullptr;
5677  // Try to match and vectorize a horizontal reduction.
5678  auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
5679  return tryToVectorize(I, R);
5680  };
5681  return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
5682  ExtraVectorization);
5683 }
5684 
5685 bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
5686  BasicBlock *BB, BoUpSLP &R) {
5687  const DataLayout &DL = BB->getModule()->getDataLayout();
5688  if (!R.canMapToVector(IVI->getType(), DL))
5689  return false;
5690 
5691  SmallVector<Value *, 16> BuildVectorOpds;
5692  if (!findBuildAggregate(IVI, BuildVectorOpds))
5693  return false;
5694 
5695  DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
5696  // Aggregate value is unlikely to be processed in vector register, we need to
5697  // extract scalars into scalar registers, so NeedExtraction is set true.
5698  return tryToVectorizeList(BuildVectorOpds, R);
5699 }
5700 
5701 bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
5702  BasicBlock *BB, BoUpSLP &R) {
5703  SmallVector<Value *, 16> BuildVectorOpds;
5704  if (!findBuildVector(IEI, BuildVectorOpds))
5705  return false;
5706 
5707  // Vectorize starting with the build vector operands ignoring the BuildVector
5708  // instructions for the purpose of scheduling and user extraction.
5709  return tryToVectorizeList(BuildVectorOpds, R);
5710 }
5711 
5712 bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
5713  BoUpSLP &R) {
5714  if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
5715  return true;
5716 
5717  bool OpsChanged = false;
5718  for (int Idx = 0; Idx < 2; ++Idx) {
5719  OpsChanged |=
5720  vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
5721  }
5722  return OpsChanged;
5723 }
5724 
5725 bool SLPVectorizerPass::vectorizeSimpleInstructions(
5726  SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
5727  bool OpsChanged = false;
5728  for (auto &VH : reverse(Instructions)) {
5729  auto *I = dyn_cast_or_null<Instruction>(VH);
5730  if (!I)
5731  continue;
5732  if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
5733  OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
5734  else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
5735  OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
5736  else if (auto *CI = dyn_cast<CmpInst>(I))
5737  OpsChanged |= vectorizeCmpInst(CI, BB, R);
5738  }
5739  Instructions.clear();
5740  return OpsChanged;
5741 }
5742 
5743 bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
5744  bool Changed = false;
5745  SmallVector<Value *, 4> Incoming;
5746  SmallSet<Value *, 16> VisitedInstrs;
5747 
5748  bool HaveVectorizedPhiNodes = true;
5749  while (HaveVectorizedPhiNodes) {
5750  HaveVectorizedPhiNodes = false;
5751 
5752  // Collect the incoming values from the PHIs.
5753  Incoming.clear();
5754  for (Instruction &I : *BB) {
5755  PHINode *P = dyn_cast<PHINode>(&I);
5756  if (!P)
5757  break;
5758 
5759  if (!VisitedInstrs.count(P))
5760  Incoming.push_back(P);
5761  }
5762 
5763  // Sort by type.
5764  std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
5765 
5766  // Try to vectorize elements base on their type.
5767  for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
5768  E = Incoming.end();
5769  IncIt != E;) {
5770 
5771  // Look for the next elements with the same type.
5772  SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
5773  while (SameTypeIt != E &&
5774  (*SameTypeIt)->getType() == (*IncIt)->getType()) {
5775  VisitedInstrs.insert(*SameTypeIt);
5776  ++SameTypeIt;
5777  }
5778 
5779  // Try to vectorize them.
5780  unsigned NumElts = (SameTypeIt - IncIt);
5781  DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n");
5782  // The order in which the phi nodes appear in the program does not matter.
5783  // So allow tryToVectorizeList to reorder them if it is beneficial. This
5784  // is done when there are exactly two elements since tryToVectorizeList
5785  // asserts that there are only two values when AllowReorder is true.
5786  bool AllowReorder = NumElts == 2;
5787  if (NumElts > 1 &&
5788  tryToVectorizeList(makeArrayRef(IncIt, NumElts), R, AllowReorder)) {
5789  // Success start over because instructions might have been changed.
5790  HaveVectorizedPhiNodes = true;
5791  Changed = true;
5792  break;
5793  }
5794 
5795  // Start over at the next instruction of a different type (or the end).
5796  IncIt = SameTypeIt;
5797  }
5798  }
5799 
5800  VisitedInstrs.clear();
5801 
5802  SmallVector<WeakVH, 8> PostProcessInstructions;
5804  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
5805  // We may go through BB multiple times so skip the one we have checked.
5806  if (!VisitedInstrs.insert(&*it).second) {
5807  if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
5808  vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
5809  // We would like to start over since some instructions are deleted
5810  // and the iterator may become invalid value.
5811  Changed = true;
5812  it = BB->begin();
5813  e = BB->end();
5814  }
5815  continue;
5816  }
5817 
5818  if (isa<DbgInfoIntrinsic>(it))
5819  continue;
5820 
5821  // Try to vectorize reductions that use PHINodes.
5822  if (PHINode *P = dyn_cast<PHINode>(it)) {
5823  // Check that the PHI is a reduction PHI.
5824  if (P->getNumIncomingValues() != 2)
5825  return Changed;
5826 
5827  // Try to match and vectorize a horizontal reduction.
5828  if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
5829  TTI)) {
5830  Changed = true;
5831  it = BB->begin();
5832  e = BB->end();
5833  continue;
5834  }
5835  continue;
5836  }
5837 
5838  // Ran into an instruction without users, like terminator, or function call
5839  // with ignored return value, store. Ignore unused instructions (basing on
5840  // instruction type, except for CallInst and InvokeInst).
5841  if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
5842  isa<InvokeInst>(it))) {
5843  KeyNodes.insert(&*it);
5844  bool OpsChanged = false;
5845  if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
5846  for (auto *V : it->operand_values()) {
5847  // Try to match and vectorize a horizontal reduction.
5848  OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
5849  }
5850  }
5851  // Start vectorization of post-process list of instructions from the
5852  // top-tree instructions to try to vectorize as many instructions as
5853  // possible.
5854  OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
5855  if (OpsChanged) {
5856  // We would like to start over since some instructions are deleted
5857  // and the iterator may become invalid value.
5858  Changed = true;
5859  it = BB->begin();
5860  e = BB->end();
5861  continue;
5862  }
5863  }
5864 
5865  if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
5866  isa<InsertValueInst>(it))
5867  PostProcessInstructions.push_back(&*it);
5868 
5869  }
5870 
5871  return Changed;
5872 }
5873 
5874 bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
5875  auto Changed = false;
5876  for (auto &Entry : GEPs) {
5877  // If the getelementptr list has fewer than two elements, there's nothing
5878  // to do.
5879  if (Entry.second.size() < 2)
5880  continue;
5881 
5882