LLVM  6.0.0svn
SLPVectorizer.cpp
Go to the documentation of this file.
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
11 // stores that can be put together into vector-stores. Next, it attempts to
12 // construct vectorizable tree using the use-def chains. If a profitable tree
13 // was found, the SLP vectorizer performs vectorization on the tree.
14 //
15 // The pass is inspired by the work described in the paper:
16 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
17 //
18 //===----------------------------------------------------------------------===//
19 
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/DenseSet.h"
24 #include "llvm/ADT/MapVector.h"
25 #include "llvm/ADT/None.h"
26 #include "llvm/ADT/Optional.h"
28 #include "llvm/ADT/STLExtras.h"
29 #include "llvm/ADT/SetVector.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/SmallSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/Statistic.h"
34 #include "llvm/ADT/iterator.h"
41 #include "llvm/Analysis/LoopInfo.h"
50 #include "llvm/IR/Attributes.h"
51 #include "llvm/IR/BasicBlock.h"
52 #include "llvm/IR/Constant.h"
53 #include "llvm/IR/Constants.h"
54 #include "llvm/IR/DataLayout.h"
55 #include "llvm/IR/DebugLoc.h"
56 #include "llvm/IR/DerivedTypes.h"
57 #include "llvm/IR/Dominators.h"
58 #include "llvm/IR/Function.h"
59 #include "llvm/IR/IRBuilder.h"
60 #include "llvm/IR/InstrTypes.h"
61 #include "llvm/IR/Instruction.h"
62 #include "llvm/IR/Instructions.h"
63 #include "llvm/IR/IntrinsicInst.h"
64 #include "llvm/IR/Intrinsics.h"
65 #include "llvm/IR/Module.h"
66 #include "llvm/IR/NoFolder.h"
67 #include "llvm/IR/Operator.h"
68 #include "llvm/IR/PassManager.h"
69 #include "llvm/IR/PatternMatch.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/Pass.h"
77 #include "llvm/Support/Casting.h"
79 #include "llvm/Support/Compiler.h"
81 #include "llvm/Support/Debug.h"
84 #include "llvm/Support/KnownBits.h"
89 #include <algorithm>
90 #include <cassert>
91 #include <cstdint>
92 #include <iterator>
93 #include <memory>
94 #include <set>
95 #include <string>
96 #include <tuple>
97 #include <utility>
98 #include <vector>
99 
100 using namespace llvm;
101 using namespace llvm::PatternMatch;
102 using namespace slpvectorizer;
103 
104 #define SV_NAME "slp-vectorizer"
105 #define DEBUG_TYPE "SLP"
106 
107 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
108 
109 static cl::opt<int>
110  SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
111  cl::desc("Only vectorize if you gain more than this "
112  "number "));
113 
114 static cl::opt<bool>
115 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
116  cl::desc("Attempt to vectorize horizontal reductions"));
117 
119  "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
120  cl::desc(
121  "Attempt to vectorize horizontal reductions feeding into a store"));
122 
123 static cl::opt<int>
124 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
125  cl::desc("Attempt to vectorize for this register size in bits"));
126 
127 /// Limits the size of scheduling regions in a block.
128 /// It avoid long compile times for _very_ large blocks where vector
129 /// instructions are spread over a wide range.
130 /// This limit is way higher than needed by real-world functions.
131 static cl::opt<int>
132 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
133  cl::desc("Limit the size of the SLP scheduling region per block"));
134 
136  "slp-min-reg-size", cl::init(128), cl::Hidden,
137  cl::desc("Attempt to vectorize for this register size in bits"));
138 
140  "slp-recursion-max-depth", cl::init(12), cl::Hidden,
141  cl::desc("Limit the recursion depth when building a vectorizable tree"));
142 
144  "slp-min-tree-size", cl::init(3), cl::Hidden,
145  cl::desc("Only vectorize small trees if they are fully vectorizable"));
146 
147 static cl::opt<bool>
148  ViewSLPTree("view-slp-tree", cl::Hidden,
149  cl::desc("Display the SLP trees with Graphviz"));
150 
151 // Limit the number of alias checks. The limit is chosen so that
152 // it has no negative effect on the llvm benchmarks.
153 static const unsigned AliasedCheckLimit = 10;
154 
155 // Another limit for the alias checks: The maximum distance between load/store
156 // instructions where alias checks are done.
157 // This limit is useful for very large basic blocks.
158 static const unsigned MaxMemDepDistance = 160;
159 
160 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
161 /// regions to be handled.
162 static const int MinScheduleRegionSize = 16;
163 
164 /// \brief Predicate for the element types that the SLP vectorizer supports.
165 ///
166 /// The most important thing to filter here are types which are invalid in LLVM
167 /// vectors. We also filter target specific types which have absolutely no
168 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
169 /// avoids spending time checking the cost model and realizing that they will
170 /// be inevitably scalarized.
171 static bool isValidElementType(Type *Ty) {
172  return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
173  !Ty->isPPC_FP128Ty();
174 }
175 
176 /// \returns true if all of the instructions in \p VL are in the same block or
177 /// false otherwise.
179  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
180  if (!I0)
181  return false;
182  BasicBlock *BB = I0->getParent();
183  for (int i = 1, e = VL.size(); i < e; i++) {
184  Instruction *I = dyn_cast<Instruction>(VL[i]);
185  if (!I)
186  return false;
187 
188  if (BB != I->getParent())
189  return false;
190  }
191  return true;
192 }
193 
194 /// \returns True if all of the values in \p VL are constants.
196  for (Value *i : VL)
197  if (!isa<Constant>(i))
198  return false;
199  return true;
200 }
201 
202 /// \returns True if all of the values in \p VL are identical.
203 static bool isSplat(ArrayRef<Value *> VL) {
204  for (unsigned i = 1, e = VL.size(); i < e; ++i)
205  if (VL[i] != VL[0])
206  return false;
207  return true;
208 }
209 
210 /// Checks if the vector of instructions can be represented as a shuffle, like:
211 /// %x0 = extractelement <4 x i8> %x, i32 0
212 /// %x3 = extractelement <4 x i8> %x, i32 3
213 /// %y1 = extractelement <4 x i8> %y, i32 1
214 /// %y2 = extractelement <4 x i8> %y, i32 2
215 /// %x0x0 = mul i8 %x0, %x0
216 /// %x3x3 = mul i8 %x3, %x3
217 /// %y1y1 = mul i8 %y1, %y1
218 /// %y2y2 = mul i8 %y2, %y2
219 /// %ins1 = insertelement <4 x i8> undef, i8 %x0x0, i32 0
220 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
221 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
222 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
223 /// ret <4 x i8> %ins4
224 /// can be transformed into:
225 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
226 /// i32 6>
227 /// %2 = mul <4 x i8> %1, %1
228 /// ret <4 x i8> %2
229 /// We convert this initially to something like:
230 /// %x0 = extractelement <4 x i8> %x, i32 0
231 /// %x3 = extractelement <4 x i8> %x, i32 3
232 /// %y1 = extractelement <4 x i8> %y, i32 1
233 /// %y2 = extractelement <4 x i8> %y, i32 2
234 /// %1 = insertelement <4 x i8> undef, i8 %x0, i32 0
235 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
236 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
237 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
238 /// %5 = mul <4 x i8> %4, %4
239 /// %6 = extractelement <4 x i8> %5, i32 0
240 /// %ins1 = insertelement <4 x i8> undef, i8 %6, i32 0
241 /// %7 = extractelement <4 x i8> %5, i32 1
242 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
243 /// %8 = extractelement <4 x i8> %5, i32 2
244 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
245 /// %9 = extractelement <4 x i8> %5, i32 3
246 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
247 /// ret <4 x i8> %ins4
248 /// InstCombiner transforms this into a shuffle and vector mul
251  auto *EI0 = cast<ExtractElementInst>(VL[0]);
252  unsigned Size = EI0->getVectorOperandType()->getVectorNumElements();
253  Value *Vec1 = nullptr;
254  Value *Vec2 = nullptr;
255  enum ShuffleMode {Unknown, FirstAlternate, SecondAlternate, Permute};
256  ShuffleMode CommonShuffleMode = Unknown;
257  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
258  auto *EI = cast<ExtractElementInst>(VL[I]);
259  auto *Vec = EI->getVectorOperand();
260  // All vector operands must have the same number of vector elements.
261  if (Vec->getType()->getVectorNumElements() != Size)
262  return None;
263  auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
264  if (!Idx)
265  return None;
266  // Undefined behavior if Idx is negative or >= Size.
267  if (Idx->getValue().uge(Size))
268  continue;
269  unsigned IntIdx = Idx->getValue().getZExtValue();
270  // We can extractelement from undef vector.
271  if (isa<UndefValue>(Vec))
272  continue;
273  // For correct shuffling we have to have at most 2 different vector operands
274  // in all extractelement instructions.
275  if (Vec1 && Vec2 && Vec != Vec1 && Vec != Vec2)
276  return None;
277  if (CommonShuffleMode == Permute)
278  continue;
279  // If the extract index is not the same as the operation number, it is a
280  // permutation.
281  if (IntIdx != I) {
282  CommonShuffleMode = Permute;
283  continue;
284  }
285  // Check the shuffle mode for the current operation.
286  if (!Vec1)
287  Vec1 = Vec;
288  else if (Vec != Vec1)
289  Vec2 = Vec;
290  // Example: shufflevector A, B, <0,5,2,7>
291  // I is odd and IntIdx for A == I - FirstAlternate shuffle.
292  // I is even and IntIdx for B == I - FirstAlternate shuffle.
293  // Example: shufflevector A, B, <4,1,6,3>
294  // I is even and IntIdx for A == I - SecondAlternate shuffle.
295  // I is odd and IntIdx for B == I - SecondAlternate shuffle.
296  const bool IIsEven = I & 1;
297  const bool CurrVecIsA = Vec == Vec1;
298  const bool IIsOdd = !IIsEven;
299  const bool CurrVecIsB = !CurrVecIsA;
300  ShuffleMode CurrentShuffleMode =
301  ((IIsOdd && CurrVecIsA) || (IIsEven && CurrVecIsB)) ? FirstAlternate
302  : SecondAlternate;
303  // Common mode is not set or the same as the shuffle mode of the current
304  // operation - alternate.
305  if (CommonShuffleMode == Unknown)
306  CommonShuffleMode = CurrentShuffleMode;
307  // Common shuffle mode is not the same as the shuffle mode of the current
308  // operation - permutation.
309  if (CommonShuffleMode != CurrentShuffleMode)
310  CommonShuffleMode = Permute;
311  }
312  // If we're not crossing lanes in different vectors, consider it as blending.
313  if ((CommonShuffleMode == FirstAlternate ||
314  CommonShuffleMode == SecondAlternate) &&
315  Vec2)
317  // If Vec2 was never used, we have a permutation of a single vector, otherwise
318  // we have permutation of 2 vectors.
321 }
322 
323 ///\returns Opcode that can be clubbed with \p Op to create an alternate
324 /// sequence which can later be merged as a ShuffleVector instruction.
325 static unsigned getAltOpcode(unsigned Op) {
326  switch (Op) {
327  case Instruction::FAdd:
328  return Instruction::FSub;
329  case Instruction::FSub:
330  return Instruction::FAdd;
331  case Instruction::Add:
332  return Instruction::Sub;
333  case Instruction::Sub:
334  return Instruction::Add;
335  default:
336  return 0;
337  }
338 }
339 
340 static bool isOdd(unsigned Value) {
341  return Value & 1;
342 }
343 
344 static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
345  unsigned CheckedOpcode) {
346  return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
347 }
348 
349 /// Chooses the correct key for scheduling data. If \p Op has the same (or
350 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
351 /// OpValue.
352 static Value *isOneOf(Value *OpValue, Value *Op) {
353  auto *I = dyn_cast<Instruction>(Op);
354  if (!I)
355  return OpValue;
356  auto *OpInst = cast<Instruction>(OpValue);
357  unsigned OpInstOpcode = OpInst->getOpcode();
358  unsigned IOpcode = I->getOpcode();
359  if (sameOpcodeOrAlt(OpInstOpcode, getAltOpcode(OpInstOpcode), IOpcode))
360  return Op;
361  return OpValue;
362 }
363 
364 namespace {
365 
366 /// Contains data for the instructions going to be vectorized.
367 struct RawInstructionsData {
368  /// Main Opcode of the instructions going to be vectorized.
369  unsigned Opcode = 0;
370 
371  /// The list of instructions have some instructions with alternate opcodes.
372  bool HasAltOpcodes = false;
373 };
374 
375 } // end anonymous namespace
376 
377 /// Checks the list of the vectorized instructions \p VL and returns info about
378 /// this list.
379 static RawInstructionsData getMainOpcode(ArrayRef<Value *> VL) {
380  auto *I0 = dyn_cast<Instruction>(VL[0]);
381  if (!I0)
382  return {};
383  RawInstructionsData Res;
384  unsigned Opcode = I0->getOpcode();
385  // Walk through the list of the vectorized instructions
386  // in order to check its structure described by RawInstructionsData.
387  for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {
388  auto *I = dyn_cast<Instruction>(VL[Cnt]);
389  if (!I)
390  return {};
391  if (Opcode != I->getOpcode())
392  Res.HasAltOpcodes = true;
393  }
394  Res.Opcode = Opcode;
395  return Res;
396 }
397 
398 namespace {
399 
400 /// Main data required for vectorization of instructions.
401 struct InstructionsState {
402  /// The very first instruction in the list with the main opcode.
403  Value *OpValue = nullptr;
404 
405  /// The main opcode for the list of instructions.
406  unsigned Opcode = 0;
407 
408  /// Some of the instructions in the list have alternate opcodes.
409  bool IsAltShuffle = false;
410 
411  InstructionsState() = default;
412  InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
413  : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
414 };
415 
416 } // end anonymous namespace
417 
418 /// \returns analysis of the Instructions in \p VL described in
419 /// InstructionsState, the Opcode that we suppose the whole list
420 /// could be vectorized even if its structure is diverse.
421 static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
422  auto Res = getMainOpcode(VL);
423  unsigned Opcode = Res.Opcode;
424  if (!Res.HasAltOpcodes)
425  return InstructionsState(VL[0], Opcode, false);
426  auto *OpInst = cast<Instruction>(VL[0]);
427  unsigned AltOpcode = getAltOpcode(Opcode);
428  // Examine each element in the list instructions VL to determine
429  // if some operations there could be considered as an alternative
430  // (for example as subtraction relates to addition operation).
431  for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
432  auto *I = cast<Instruction>(VL[Cnt]);
433  unsigned InstOpcode = I->getOpcode();
434  if ((Res.HasAltOpcodes &&
435  InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||
436  (!Res.HasAltOpcodes && InstOpcode != Opcode)) {
437  return InstructionsState(OpInst, 0, false);
438  }
439  }
440  return InstructionsState(OpInst, Opcode, Res.HasAltOpcodes);
441 }
442 
443 /// \returns true if all of the values in \p VL have the same type or false
444 /// otherwise.
446  Type *Ty = VL[0]->getType();
447  for (int i = 1, e = VL.size(); i < e; i++)
448  if (VL[i]->getType() != Ty)
449  return false;
450 
451  return true;
452 }
453 
454 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
455 static bool matchExtractIndex(Instruction *E, unsigned Idx, unsigned Opcode) {
456  assert(Opcode == Instruction::ExtractElement ||
457  Opcode == Instruction::ExtractValue);
458  if (Opcode == Instruction::ExtractElement) {
460  return CI && CI->getZExtValue() == Idx;
461  } else {
462  ExtractValueInst *EI = cast<ExtractValueInst>(E);
463  return EI->getNumIndices() == 1 && *EI->idx_begin() == Idx;
464  }
465 }
466 
467 /// \returns True if in-tree use also needs extract. This refers to
468 /// possible scalar operand in vectorized instruction.
470  TargetLibraryInfo *TLI) {
471  unsigned Opcode = UserInst->getOpcode();
472  switch (Opcode) {
473  case Instruction::Load: {
474  LoadInst *LI = cast<LoadInst>(UserInst);
475  return (LI->getPointerOperand() == Scalar);
476  }
477  case Instruction::Store: {
478  StoreInst *SI = cast<StoreInst>(UserInst);
479  return (SI->getPointerOperand() == Scalar);
480  }
481  case Instruction::Call: {
482  CallInst *CI = cast<CallInst>(UserInst);
484  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
485  return (CI->getArgOperand(1) == Scalar);
486  }
488  }
489  default:
490  return false;
491  }
492 }
493 
494 /// \returns the AA location that is being access by the instruction.
496  if (StoreInst *SI = dyn_cast<StoreInst>(I))
497  return MemoryLocation::get(SI);
498  if (LoadInst *LI = dyn_cast<LoadInst>(I))
499  return MemoryLocation::get(LI);
500  return MemoryLocation();
501 }
502 
503 /// \returns True if the instruction is not a volatile or atomic load/store.
504 static bool isSimple(Instruction *I) {
505  if (LoadInst *LI = dyn_cast<LoadInst>(I))
506  return LI->isSimple();
507  if (StoreInst *SI = dyn_cast<StoreInst>(I))
508  return SI->isSimple();
509  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
510  return !MI->isVolatile();
511  return true;
512 }
513 
514 namespace llvm {
515 
516 namespace slpvectorizer {
517 
518 /// Bottom Up SLP Vectorizer.
519 class BoUpSLP {
520 public:
527 
531  const DataLayout *DL, OptimizationRemarkEmitter *ORE)
532  : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
533  DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
534  CodeMetrics::collectEphemeralValues(F, AC, EphValues);
535  // Use the vector register size specified by the target unless overridden
536  // by a command-line option.
537  // TODO: It would be better to limit the vectorization factor based on
538  // data type rather than just register size. For example, x86 AVX has
539  // 256-bit registers, but it does not support integer operations
540  // at that width (that requires AVX2).
541  if (MaxVectorRegSizeOption.getNumOccurrences())
542  MaxVecRegSize = MaxVectorRegSizeOption;
543  else
544  MaxVecRegSize = TTI->getRegisterBitWidth(true);
545 
546  if (MinVectorRegSizeOption.getNumOccurrences())
547  MinVecRegSize = MinVectorRegSizeOption;
548  else
549  MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
550  }
551 
552  /// \brief Vectorize the tree that starts with the elements in \p VL.
553  /// Returns the vectorized root.
554  Value *vectorizeTree();
555 
556  /// Vectorize the tree but with the list of externally used values \p
557  /// ExternallyUsedValues. Values in this MapVector can be replaced but the
558  /// generated extractvalue instructions.
559  Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
560 
561  /// \returns the cost incurred by unwanted spills and fills, caused by
562  /// holding live values over call sites.
563  int getSpillCost();
564 
565  /// \returns the vectorization cost of the subtree that starts at \p VL.
566  /// A negative number means that this is profitable.
567  int getTreeCost();
568 
569  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
570  /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
571  void buildTree(ArrayRef<Value *> Roots,
572  ArrayRef<Value *> UserIgnoreLst = None);
573 
574  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
575  /// the purpose of scheduling and extraction in the \p UserIgnoreLst taking
576  /// into account (anf updating it, if required) list of externally used
577  /// values stored in \p ExternallyUsedValues.
578  void buildTree(ArrayRef<Value *> Roots,
579  ExtraValueToDebugLocsMap &ExternallyUsedValues,
580  ArrayRef<Value *> UserIgnoreLst = None);
581 
582  /// Clear the internal data structures that are created by 'buildTree'.
583  void deleteTree() {
584  VectorizableTree.clear();
585  ScalarToTreeEntry.clear();
586  MustGather.clear();
587  ExternalUses.clear();
588  NumLoadsWantToKeepOrder = 0;
589  NumLoadsWantToChangeOrder = 0;
590  for (auto &Iter : BlocksSchedules) {
591  BlockScheduling *BS = Iter.second.get();
592  BS->clear();
593  }
594  MinBWs.clear();
595  }
596 
597  unsigned getTreeSize() const { return VectorizableTree.size(); }
598 
599  /// \brief Perform LICM and CSE on the newly generated gather sequences.
600  void optimizeGatherSequence();
601 
602  /// \returns true if it is beneficial to reverse the vector order.
603  bool shouldReorder() const {
604  return NumLoadsWantToChangeOrder > NumLoadsWantToKeepOrder;
605  }
606 
607  /// \return The vector element size in bits to use when vectorizing the
608  /// expression tree ending at \p V. If V is a store, the size is the width of
609  /// the stored value. Otherwise, the size is the width of the largest loaded
610  /// value reaching V. This method is used by the vectorizer to calculate
611  /// vectorization factors.
612  unsigned getVectorElementSize(Value *V);
613 
614  /// Compute the minimum type sizes required to represent the entries in a
615  /// vectorizable tree.
617 
618  // \returns maximum vector register size as set by TTI or overridden by cl::opt.
619  unsigned getMaxVecRegSize() const {
620  return MaxVecRegSize;
621  }
622 
623  // \returns minimum vector register size as set by cl::opt.
624  unsigned getMinVecRegSize() const {
625  return MinVecRegSize;
626  }
627 
628  /// \brief Check if ArrayType or StructType is isomorphic to some VectorType.
629  ///
630  /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
631  unsigned canMapToVector(Type *T, const DataLayout &DL) const;
632 
633  /// \returns True if the VectorizableTree is both tiny and not fully
634  /// vectorizable. We do not vectorize such trees.
635  bool isTreeTinyAndNotFullyVectorizable();
636 
638 
639 private:
640  struct TreeEntry;
641 
642  /// Checks if all users of \p I are the part of the vectorization tree.
643  bool areAllUsersVectorized(Instruction *I) const;
644 
645  /// \returns the cost of the vectorizable entry.
646  int getEntryCost(TreeEntry *E);
647 
648  /// This is the recursive part of buildTree.
649  void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth, int);
650 
651  /// \returns True if the ExtractElement/ExtractValue instructions in VL can
652  /// be vectorized to use the original vector (or aggregate "bitcast" to a vector).
653  bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const;
654 
655  /// Vectorize a single entry in the tree.
656  Value *vectorizeTree(TreeEntry *E);
657 
658  /// Vectorize a single entry in the tree, starting in \p VL.
659  Value *vectorizeTree(ArrayRef<Value *> VL);
660 
661  /// \returns the pointer to the vectorized value if \p VL is already
662  /// vectorized, or NULL. They may happen in cycles.
663  Value *alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const;
664 
665  /// \returns the scalarization cost for this type. Scalarization in this
666  /// context means the creation of vectors from a group of scalars.
667  int getGatherCost(Type *Ty);
668 
669  /// \returns the scalarization cost for this list of values. Assuming that
670  /// this subtree gets vectorized, we may need to extract the values from the
671  /// roots. This method calculates the cost of extracting the values.
672  int getGatherCost(ArrayRef<Value *> VL);
673 
674  /// \brief Set the Builder insert point to one after the last instruction in
675  /// the bundle
676  void setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue);
677 
678  /// \returns a vector from a collection of scalars in \p VL.
679  Value *Gather(ArrayRef<Value *> VL, VectorType *Ty);
680 
681  /// \returns whether the VectorizableTree is fully vectorizable and will
682  /// be beneficial even the tree height is tiny.
683  bool isFullyVectorizableTinyTree();
684 
685  /// \reorder commutative operands in alt shuffle if they result in
686  /// vectorized code.
687  void reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
690 
691  /// \reorder commutative operands to get better probability of
692  /// generating vectorized code.
693  void reorderInputsAccordingToOpcode(unsigned Opcode, ArrayRef<Value *> VL,
695  SmallVectorImpl<Value *> &Right);
696  struct TreeEntry {
697  TreeEntry(std::vector<TreeEntry> &Container) : Container(Container) {}
698 
699  /// \returns true if the scalars in VL are equal to this entry.
700  bool isSame(ArrayRef<Value *> VL) const {
701  assert(VL.size() == Scalars.size() && "Invalid size");
702  return std::equal(VL.begin(), VL.end(), Scalars.begin());
703  }
704 
705  /// A vector of scalars.
706  ValueList Scalars;
707 
708  /// The Scalars are vectorized into this value. It is initialized to Null.
709  Value *VectorizedValue = nullptr;
710 
711  /// Do we need to gather this sequence ?
712  bool NeedToGather = false;
713 
714  /// Points back to the VectorizableTree.
715  ///
716  /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
717  /// to be a pointer and needs to be able to initialize the child iterator.
718  /// Thus we need a reference back to the container to translate the indices
719  /// to entries.
720  std::vector<TreeEntry> &Container;
721 
722  /// The TreeEntry index containing the user of this entry. We can actually
723  /// have multiple users so the data structure is not truly a tree.
724  SmallVector<int, 1> UserTreeIndices;
725  };
726 
727  /// Create a new VectorizableTree entry.
728  TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,
729  int &UserTreeIdx) {
730  VectorizableTree.emplace_back(VectorizableTree);
731  int idx = VectorizableTree.size() - 1;
732  TreeEntry *Last = &VectorizableTree[idx];
733  Last->Scalars.insert(Last->Scalars.begin(), VL.begin(), VL.end());
734  Last->NeedToGather = !Vectorized;
735  if (Vectorized) {
736  for (int i = 0, e = VL.size(); i != e; ++i) {
737  assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");
738  ScalarToTreeEntry[VL[i]] = idx;
739  }
740  } else {
741  MustGather.insert(VL.begin(), VL.end());
742  }
743 
744  if (UserTreeIdx >= 0)
745  Last->UserTreeIndices.push_back(UserTreeIdx);
746  UserTreeIdx = idx;
747  return Last;
748  }
749 
750  /// -- Vectorization State --
751  /// Holds all of the tree entries.
752  std::vector<TreeEntry> VectorizableTree;
753 
754  TreeEntry *getTreeEntry(Value *V) {
755  auto I = ScalarToTreeEntry.find(V);
756  if (I != ScalarToTreeEntry.end())
757  return &VectorizableTree[I->second];
758  return nullptr;
759  }
760 
761  const TreeEntry *getTreeEntry(Value *V) const {
762  auto I = ScalarToTreeEntry.find(V);
763  if (I != ScalarToTreeEntry.end())
764  return &VectorizableTree[I->second];
765  return nullptr;
766  }
767 
768  /// Maps a specific scalar to its tree entry.
769  SmallDenseMap<Value*, int> ScalarToTreeEntry;
770 
771  /// A list of scalars that we found that we need to keep as scalars.
772  ValueSet MustGather;
773 
774  /// This POD struct describes one external user in the vectorized tree.
775  struct ExternalUser {
776  ExternalUser(Value *S, llvm::User *U, int L)
777  : Scalar(S), User(U), Lane(L) {}
778 
779  // Which scalar in our function.
780  Value *Scalar;
781 
782  // Which user that uses the scalar.
783  llvm::User *User;
784 
785  // Which lane does the scalar belong to.
786  int Lane;
787  };
789 
790  /// Checks if two instructions may access the same memory.
791  ///
792  /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
793  /// is invariant in the calling loop.
794  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
795  Instruction *Inst2) {
796  // First check if the result is already in the cache.
797  AliasCacheKey key = std::make_pair(Inst1, Inst2);
798  Optional<bool> &result = AliasCache[key];
799  if (result.hasValue()) {
800  return result.getValue();
801  }
802  MemoryLocation Loc2 = getLocation(Inst2, AA);
803  bool aliased = true;
804  if (Loc1.Ptr && Loc2.Ptr && isSimple(Inst1) && isSimple(Inst2)) {
805  // Do the alias check.
806  aliased = AA->alias(Loc1, Loc2);
807  }
808  // Store the result in the cache.
809  result = aliased;
810  return aliased;
811  }
812 
813  using AliasCacheKey = std::pair<Instruction *, Instruction *>;
814 
815  /// Cache for alias results.
816  /// TODO: consider moving this to the AliasAnalysis itself.
818 
819  /// Removes an instruction from its block and eventually deletes it.
820  /// It's like Instruction::eraseFromParent() except that the actual deletion
821  /// is delayed until BoUpSLP is destructed.
822  /// This is required to ensure that there are no incorrect collisions in the
823  /// AliasCache, which can happen if a new instruction is allocated at the
824  /// same address as a previously deleted instruction.
825  void eraseInstruction(Instruction *I) {
826  I->removeFromParent();
827  I->dropAllReferences();
828  DeletedInstructions.emplace_back(I);
829  }
830 
831  /// Temporary store for deleted instructions. Instructions will be deleted
832  /// eventually when the BoUpSLP is destructed.
833  SmallVector<unique_value, 8> DeletedInstructions;
834 
835  /// A list of values that need to extracted out of the tree.
836  /// This list holds pairs of (Internal Scalar : External User). External User
837  /// can be nullptr, it means that this Internal Scalar will be used later,
838  /// after vectorization.
839  UserList ExternalUses;
840 
841  /// Values used only by @llvm.assume calls.
843 
844  /// Holds all of the instructions that we gathered.
845  SetVector<Instruction *> GatherSeq;
846 
847  /// A list of blocks that we are going to CSE.
848  SetVector<BasicBlock *> CSEBlocks;
849 
850  /// Contains all scheduling relevant data for an instruction.
851  /// A ScheduleData either represents a single instruction or a member of an
852  /// instruction bundle (= a group of instructions which is combined into a
853  /// vector instruction).
854  struct ScheduleData {
855  // The initial value for the dependency counters. It means that the
856  // dependencies are not calculated yet.
857  enum { InvalidDeps = -1 };
858 
859  ScheduleData() = default;
860 
861  void init(int BlockSchedulingRegionID, Value *OpVal) {
862  FirstInBundle = this;
863  NextInBundle = nullptr;
864  NextLoadStore = nullptr;
865  IsScheduled = false;
866  SchedulingRegionID = BlockSchedulingRegionID;
867  UnscheduledDepsInBundle = UnscheduledDeps;
868  clearDependencies();
869  OpValue = OpVal;
870  }
871 
872  /// Returns true if the dependency information has been calculated.
873  bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
874 
875  /// Returns true for single instructions and for bundle representatives
876  /// (= the head of a bundle).
877  bool isSchedulingEntity() const { return FirstInBundle == this; }
878 
879  /// Returns true if it represents an instruction bundle and not only a
880  /// single instruction.
881  bool isPartOfBundle() const {
882  return NextInBundle != nullptr || FirstInBundle != this;
883  }
884 
885  /// Returns true if it is ready for scheduling, i.e. it has no more
886  /// unscheduled depending instructions/bundles.
887  bool isReady() const {
888  assert(isSchedulingEntity() &&
889  "can't consider non-scheduling entity for ready list");
890  return UnscheduledDepsInBundle == 0 && !IsScheduled;
891  }
892 
893  /// Modifies the number of unscheduled dependencies, also updating it for
894  /// the whole bundle.
895  int incrementUnscheduledDeps(int Incr) {
896  UnscheduledDeps += Incr;
897  return FirstInBundle->UnscheduledDepsInBundle += Incr;
898  }
899 
900  /// Sets the number of unscheduled dependencies to the number of
901  /// dependencies.
902  void resetUnscheduledDeps() {
903  incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
904  }
905 
906  /// Clears all dependency information.
907  void clearDependencies() {
908  Dependencies = InvalidDeps;
909  resetUnscheduledDeps();
910  MemoryDependencies.clear();
911  }
912 
913  void dump(raw_ostream &os) const {
914  if (!isSchedulingEntity()) {
915  os << "/ " << *Inst;
916  } else if (NextInBundle) {
917  os << '[' << *Inst;
918  ScheduleData *SD = NextInBundle;
919  while (SD) {
920  os << ';' << *SD->Inst;
921  SD = SD->NextInBundle;
922  }
923  os << ']';
924  } else {
925  os << *Inst;
926  }
927  }
928 
929  Instruction *Inst = nullptr;
930 
931  /// Points to the head in an instruction bundle (and always to this for
932  /// single instructions).
933  ScheduleData *FirstInBundle = nullptr;
934 
935  /// Single linked list of all instructions in a bundle. Null if it is a
936  /// single instruction.
937  ScheduleData *NextInBundle = nullptr;
938 
939  /// Single linked list of all memory instructions (e.g. load, store, call)
940  /// in the block - until the end of the scheduling region.
941  ScheduleData *NextLoadStore = nullptr;
942 
943  /// The dependent memory instructions.
944  /// This list is derived on demand in calculateDependencies().
945  SmallVector<ScheduleData *, 4> MemoryDependencies;
946 
947  /// This ScheduleData is in the current scheduling region if this matches
948  /// the current SchedulingRegionID of BlockScheduling.
949  int SchedulingRegionID = 0;
950 
951  /// Used for getting a "good" final ordering of instructions.
952  int SchedulingPriority = 0;
953 
954  /// The number of dependencies. Constitutes of the number of users of the
955  /// instruction plus the number of dependent memory instructions (if any).
956  /// This value is calculated on demand.
957  /// If InvalidDeps, the number of dependencies is not calculated yet.
958  int Dependencies = InvalidDeps;
959 
960  /// The number of dependencies minus the number of dependencies of scheduled
961  /// instructions. As soon as this is zero, the instruction/bundle gets ready
962  /// for scheduling.
963  /// Note that this is negative as long as Dependencies is not calculated.
964  int UnscheduledDeps = InvalidDeps;
965 
966  /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
967  /// single instructions.
968  int UnscheduledDepsInBundle = InvalidDeps;
969 
970  /// True if this instruction is scheduled (or considered as scheduled in the
971  /// dry-run).
972  bool IsScheduled = false;
973 
974  /// Opcode of the current instruction in the schedule data.
975  Value *OpValue = nullptr;
976  };
977 
978 #ifndef NDEBUG
979  friend inline raw_ostream &operator<<(raw_ostream &os,
980  const BoUpSLP::ScheduleData &SD) {
981  SD.dump(os);
982  return os;
983  }
984 #endif
985 
986  friend struct GraphTraits<BoUpSLP *>;
987  friend struct DOTGraphTraits<BoUpSLP *>;
988 
989  /// Contains all scheduling data for a basic block.
990  struct BlockScheduling {
991  BlockScheduling(BasicBlock *BB)
992  : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
993 
994  void clear() {
995  ReadyInsts.clear();
996  ScheduleStart = nullptr;
997  ScheduleEnd = nullptr;
998  FirstLoadStoreInRegion = nullptr;
999  LastLoadStoreInRegion = nullptr;
1000 
1001  // Reduce the maximum schedule region size by the size of the
1002  // previous scheduling run.
1003  ScheduleRegionSizeLimit -= ScheduleRegionSize;
1004  if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
1005  ScheduleRegionSizeLimit = MinScheduleRegionSize;
1006  ScheduleRegionSize = 0;
1007 
1008  // Make a new scheduling region, i.e. all existing ScheduleData is not
1009  // in the new region yet.
1010  ++SchedulingRegionID;
1011  }
1012 
1013  ScheduleData *getScheduleData(Value *V) {
1014  ScheduleData *SD = ScheduleDataMap[V];
1015  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1016  return SD;
1017  return nullptr;
1018  }
1019 
1020  ScheduleData *getScheduleData(Value *V, Value *Key) {
1021  if (V == Key)
1022  return getScheduleData(V);
1023  auto I = ExtraScheduleDataMap.find(V);
1024  if (I != ExtraScheduleDataMap.end()) {
1025  ScheduleData *SD = I->second[Key];
1026  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
1027  return SD;
1028  }
1029  return nullptr;
1030  }
1031 
1032  bool isInSchedulingRegion(ScheduleData *SD) {
1033  return SD->SchedulingRegionID == SchedulingRegionID;
1034  }
1035 
1036  /// Marks an instruction as scheduled and puts all dependent ready
1037  /// instructions into the ready-list.
1038  template <typename ReadyListType>
1039  void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
1040  SD->IsScheduled = true;
1041  DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
1042 
1043  ScheduleData *BundleMember = SD;
1044  while (BundleMember) {
1045  if (BundleMember->Inst != BundleMember->OpValue) {
1046  BundleMember = BundleMember->NextInBundle;
1047  continue;
1048  }
1049  // Handle the def-use chain dependencies.
1050  for (Use &U : BundleMember->Inst->operands()) {
1051  auto *I = dyn_cast<Instruction>(U.get());
1052  if (!I)
1053  continue;
1054  doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
1055  if (OpDef && OpDef->hasValidDependencies() &&
1056  OpDef->incrementUnscheduledDeps(-1) == 0) {
1057  // There are no more unscheduled dependencies after
1058  // decrementing, so we can put the dependent instruction
1059  // into the ready list.
1060  ScheduleData *DepBundle = OpDef->FirstInBundle;
1061  assert(!DepBundle->IsScheduled &&
1062  "already scheduled bundle gets ready");
1063  ReadyList.insert(DepBundle);
1064  DEBUG(dbgs()
1065  << "SLP: gets ready (def): " << *DepBundle << "\n");
1066  }
1067  });
1068  }
1069  // Handle the memory dependencies.
1070  for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
1071  if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
1072  // There are no more unscheduled dependencies after decrementing,
1073  // so we can put the dependent instruction into the ready list.
1074  ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
1075  assert(!DepBundle->IsScheduled &&
1076  "already scheduled bundle gets ready");
1077  ReadyList.insert(DepBundle);
1078  DEBUG(dbgs() << "SLP: gets ready (mem): " << *DepBundle
1079  << "\n");
1080  }
1081  }
1082  BundleMember = BundleMember->NextInBundle;
1083  }
1084  }
1085 
1086  void doForAllOpcodes(Value *V,
1087  function_ref<void(ScheduleData *SD)> Action) {
1088  if (ScheduleData *SD = getScheduleData(V))
1089  Action(SD);
1090  auto I = ExtraScheduleDataMap.find(V);
1091  if (I != ExtraScheduleDataMap.end())
1092  for (auto &P : I->second)
1093  if (P.second->SchedulingRegionID == SchedulingRegionID)
1094  Action(P.second);
1095  }
1096 
1097  /// Put all instructions into the ReadyList which are ready for scheduling.
1098  template <typename ReadyListType>
1099  void initialFillReadyList(ReadyListType &ReadyList) {
1100  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
1101  doForAllOpcodes(I, [&](ScheduleData *SD) {
1102  if (SD->isSchedulingEntity() && SD->isReady()) {
1103  ReadyList.insert(SD);
1104  DEBUG(dbgs() << "SLP: initially in ready list: " << *I << "\n");
1105  }
1106  });
1107  }
1108  }
1109 
1110  /// Checks if a bundle of instructions can be scheduled, i.e. has no
1111  /// cyclic dependencies. This is only a dry-run, no instructions are
1112  /// actually moved at this stage.
1113  bool tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP, Value *OpValue);
1114 
1115  /// Un-bundles a group of instructions.
1116  void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
1117 
1118  /// Allocates schedule data chunk.
1119  ScheduleData *allocateScheduleDataChunks();
1120 
1121  /// Extends the scheduling region so that V is inside the region.
1122  /// \returns true if the region size is within the limit.
1123  bool extendSchedulingRegion(Value *V, Value *OpValue);
1124 
1125  /// Initialize the ScheduleData structures for new instructions in the
1126  /// scheduling region.
1127  void initScheduleData(Instruction *FromI, Instruction *ToI,
1128  ScheduleData *PrevLoadStore,
1129  ScheduleData *NextLoadStore);
1130 
1131  /// Updates the dependency information of a bundle and of all instructions/
1132  /// bundles which depend on the original bundle.
1133  void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
1134  BoUpSLP *SLP);
1135 
1136  /// Sets all instruction in the scheduling region to un-scheduled.
1137  void resetSchedule();
1138 
1139  BasicBlock *BB;
1140 
1141  /// Simple memory allocation for ScheduleData.
1142  std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
1143 
1144  /// The size of a ScheduleData array in ScheduleDataChunks.
1145  int ChunkSize;
1146 
1147  /// The allocator position in the current chunk, which is the last entry
1148  /// of ScheduleDataChunks.
1149  int ChunkPos;
1150 
1151  /// Attaches ScheduleData to Instruction.
1152  /// Note that the mapping survives during all vectorization iterations, i.e.
1153  /// ScheduleData structures are recycled.
1154  DenseMap<Value *, ScheduleData *> ScheduleDataMap;
1155 
1156  /// Attaches ScheduleData to Instruction with the leading key.
1158  ExtraScheduleDataMap;
1159 
1160  struct ReadyList : SmallVector<ScheduleData *, 8> {
1161  void insert(ScheduleData *SD) { push_back(SD); }
1162  };
1163 
1164  /// The ready-list for scheduling (only used for the dry-run).
1165  ReadyList ReadyInsts;
1166 
1167  /// The first instruction of the scheduling region.
1168  Instruction *ScheduleStart = nullptr;
1169 
1170  /// The first instruction _after_ the scheduling region.
1171  Instruction *ScheduleEnd = nullptr;
1172 
1173  /// The first memory accessing instruction in the scheduling region
1174  /// (can be null).
1175  ScheduleData *FirstLoadStoreInRegion = nullptr;
1176 
1177  /// The last memory accessing instruction in the scheduling region
1178  /// (can be null).
1179  ScheduleData *LastLoadStoreInRegion = nullptr;
1180 
1181  /// The current size of the scheduling region.
1182  int ScheduleRegionSize = 0;
1183 
1184  /// The maximum size allowed for the scheduling region.
1185  int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
1186 
1187  /// The ID of the scheduling region. For a new vectorization iteration this
1188  /// is incremented which "removes" all ScheduleData from the region.
1189  // Make sure that the initial SchedulingRegionID is greater than the
1190  // initial SchedulingRegionID in ScheduleData (which is 0).
1191  int SchedulingRegionID = 1;
1192  };
1193 
1194  /// Attaches the BlockScheduling structures to basic blocks.
1196 
1197  /// Performs the "real" scheduling. Done before vectorization is actually
1198  /// performed in a basic block.
1199  void scheduleBlock(BlockScheduling *BS);
1200 
1201  /// List of users to ignore during scheduling and that don't need extracting.
1202  ArrayRef<Value *> UserIgnoreList;
1203 
1204  // Number of load bundles that contain consecutive loads.
1205  int NumLoadsWantToKeepOrder = 0;
1206 
1207  // Number of load bundles that contain consecutive loads in reversed order.
1208  int NumLoadsWantToChangeOrder = 0;
1209 
1210  // Analysis and block reference.
1211  Function *F;
1212  ScalarEvolution *SE;
1213  TargetTransformInfo *TTI;
1214  TargetLibraryInfo *TLI;
1215  AliasAnalysis *AA;
1216  LoopInfo *LI;
1217  DominatorTree *DT;
1218  AssumptionCache *AC;
1219  DemandedBits *DB;
1220  const DataLayout *DL;
1222 
1223  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
1224  unsigned MinVecRegSize; // Set by cl::opt (default: 128).
1225 
1226  /// Instruction builder to construct the vectorized tree.
1227  IRBuilder<> Builder;
1228 
1229  /// A map of scalar integer values to the smallest bit width with which they
1230  /// can legally be represented. The values map to (width, signed) pairs,
1231  /// where "width" indicates the minimum bit width and "signed" is True if the
1232  /// value must be signed-extended, rather than zero-extended, back to its
1233  /// original width.
1235 };
1236 
1237 } // end namespace slpvectorizer
1238 
1239 template <> struct GraphTraits<BoUpSLP *> {
1240  using TreeEntry = BoUpSLP::TreeEntry;
1241 
1242  /// NodeRef has to be a pointer per the GraphWriter.
1243  using NodeRef = TreeEntry *;
1244 
1245  /// \brief Add the VectorizableTree to the index iterator to be able to return
1246  /// TreeEntry pointers.
1247  struct ChildIteratorType
1248  : public iterator_adaptor_base<ChildIteratorType,
1249  SmallVector<int, 1>::iterator> {
1250  std::vector<TreeEntry> &VectorizableTree;
1251 
1253  std::vector<TreeEntry> &VT)
1254  : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
1255 
1256  NodeRef operator*() { return &VectorizableTree[*I]; }
1257  };
1258 
1259  static NodeRef getEntryNode(BoUpSLP &R) { return &R.VectorizableTree[0]; }
1260 
1261  static ChildIteratorType child_begin(NodeRef N) {
1262  return {N->UserTreeIndices.begin(), N->Container};
1263  }
1264 
1265  static ChildIteratorType child_end(NodeRef N) {
1266  return {N->UserTreeIndices.end(), N->Container};
1267  }
1268 
1269  /// For the node iterator we just need to turn the TreeEntry iterator into a
1270  /// TreeEntry* iterator so that it dereferences to NodeRef.
1272 
1273  static nodes_iterator nodes_begin(BoUpSLP *R) {
1274  return nodes_iterator(R->VectorizableTree.begin());
1275  }
1276 
1277  static nodes_iterator nodes_end(BoUpSLP *R) {
1278  return nodes_iterator(R->VectorizableTree.end());
1279  }
1280 
1281  static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
1282 };
1283 
1284 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
1285  using TreeEntry = BoUpSLP::TreeEntry;
1286 
1288 
1289  std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
1290  std::string Str;
1291  raw_string_ostream OS(Str);
1292  if (isSplat(Entry->Scalars)) {
1293  OS << "<splat> " << *Entry->Scalars[0];
1294  return Str;
1295  }
1296  for (auto V : Entry->Scalars) {
1297  OS << *V;
1298  if (std::any_of(
1299  R->ExternalUses.begin(), R->ExternalUses.end(),
1300  [&](const BoUpSLP::ExternalUser &EU) { return EU.Scalar == V; }))
1301  OS << " <extract>";
1302  OS << "\n";
1303  }
1304  return Str;
1305  }
1306 
1307  static std::string getNodeAttributes(const TreeEntry *Entry,
1308  const BoUpSLP *) {
1309  if (Entry->NeedToGather)
1310  return "color=red";
1311  return "";
1312  }
1313 };
1314 
1315 } // end namespace llvm
1316 
1317 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1318  ArrayRef<Value *> UserIgnoreLst) {
1319  ExtraValueToDebugLocsMap ExternallyUsedValues;
1320  buildTree(Roots, ExternallyUsedValues, UserIgnoreLst);
1321 }
1322 
1323 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
1324  ExtraValueToDebugLocsMap &ExternallyUsedValues,
1325  ArrayRef<Value *> UserIgnoreLst) {
1326  deleteTree();
1327  UserIgnoreList = UserIgnoreLst;
1328  if (!allSameType(Roots))
1329  return;
1330  buildTree_rec(Roots, 0, -1);
1331 
1332  // Collect the values that we need to extract from the tree.
1333  for (TreeEntry &EIdx : VectorizableTree) {
1334  TreeEntry *Entry = &EIdx;
1335 
1336  // No need to handle users of gathered values.
1337  if (Entry->NeedToGather)
1338  continue;
1339 
1340  // For each lane:
1341  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
1342  Value *Scalar = Entry->Scalars[Lane];
1343 
1344  // Check if the scalar is externally used as an extra arg.
1345  auto ExtI = ExternallyUsedValues.find(Scalar);
1346  if (ExtI != ExternallyUsedValues.end()) {
1347  DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane " <<
1348  Lane << " from " << *Scalar << ".\n");
1349  ExternalUses.emplace_back(Scalar, nullptr, Lane);
1350  continue;
1351  }
1352  for (User *U : Scalar->users()) {
1353  DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
1354 
1355  Instruction *UserInst = dyn_cast<Instruction>(U);
1356  if (!UserInst)
1357  continue;
1358 
1359  // Skip in-tree scalars that become vectors
1360  if (TreeEntry *UseEntry = getTreeEntry(U)) {
1361  Value *UseScalar = UseEntry->Scalars[0];
1362  // Some in-tree scalars will remain as scalar in vectorized
1363  // instructions. If that is the case, the one in Lane 0 will
1364  // be used.
1365  if (UseScalar != U ||
1366  !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
1367  DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
1368  << ".\n");
1369  assert(!UseEntry->NeedToGather && "Bad state");
1370  continue;
1371  }
1372  }
1373 
1374  // Ignore users in the user ignore list.
1375  if (is_contained(UserIgnoreList, UserInst))
1376  continue;
1377 
1378  DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane " <<
1379  Lane << " from " << *Scalar << ".\n");
1380  ExternalUses.push_back(ExternalUser(Scalar, U, Lane));
1381  }
1382  }
1383  }
1384 }
1385 
1386 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1387  int UserTreeIdx) {
1388  assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
1389 
1390  InstructionsState S = getSameOpcode(VL);
1391  if (Depth == RecursionMaxDepth) {
1392  DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
1393  newTreeEntry(VL, false, UserTreeIdx);
1394  return;
1395  }
1396 
1397  // Don't handle vectors.
1398  if (S.OpValue->getType()->isVectorTy()) {
1399  DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
1400  newTreeEntry(VL, false, UserTreeIdx);
1401  return;
1402  }
1403 
1404  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
1405  if (SI->getValueOperand()->getType()->isVectorTy()) {
1406  DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
1407  newTreeEntry(VL, false, UserTreeIdx);
1408  return;
1409  }
1410 
1411  // If all of the operands are identical or constant we have a simple solution.
1412  if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {
1413  DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
1414  newTreeEntry(VL, false, UserTreeIdx);
1415  return;
1416  }
1417 
1418  // We now know that this is a vector of instructions of the same type from
1419  // the same block.
1420 
1421  // Don't vectorize ephemeral values.
1422  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1423  if (EphValues.count(VL[i])) {
1424  DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1425  ") is ephemeral.\n");
1426  newTreeEntry(VL, false, UserTreeIdx);
1427  return;
1428  }
1429  }
1430 
1431  // Check if this is a duplicate of another entry.
1432  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
1433  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1434  DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n");
1435  if (E->Scalars[i] != VL[i]) {
1436  DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
1437  newTreeEntry(VL, false, UserTreeIdx);
1438  return;
1439  }
1440  }
1441  // Record the reuse of the tree node. FIXME, currently this is only used to
1442  // properly draw the graph rather than for the actual vectorization.
1443  E->UserTreeIndices.push_back(UserTreeIdx);
1444  DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue << ".\n");
1445  return;
1446  }
1447 
1448  // Check that none of the instructions in the bundle are already in the tree.
1449  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1450  auto *I = dyn_cast<Instruction>(VL[i]);
1451  if (!I)
1452  continue;
1453  if (getTreeEntry(I)) {
1454  DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<
1455  ") is already in tree.\n");
1456  newTreeEntry(VL, false, UserTreeIdx);
1457  return;
1458  }
1459  }
1460 
1461  // If any of the scalars is marked as a value that needs to stay scalar, then
1462  // we need to gather the scalars.
1463  for (unsigned i = 0, e = VL.size(); i != e; ++i) {
1464  if (MustGather.count(VL[i])) {
1465  DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
1466  newTreeEntry(VL, false, UserTreeIdx);
1467  return;
1468  }
1469  }
1470 
1471  // Check that all of the users of the scalars that we want to vectorize are
1472  // schedulable.
1473  auto *VL0 = cast<Instruction>(S.OpValue);
1474  BasicBlock *BB = VL0->getParent();
1475 
1476  if (!DT->isReachableFromEntry(BB)) {
1477  // Don't go into unreachable blocks. They may contain instructions with
1478  // dependency cycles which confuse the final scheduling.
1479  DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
1480  newTreeEntry(VL, false, UserTreeIdx);
1481  return;
1482  }
1483 
1484  // Check that every instruction appears once in this bundle.
1485  for (unsigned i = 0, e = VL.size(); i < e; ++i)
1486  for (unsigned j = i + 1; j < e; ++j)
1487  if (VL[i] == VL[j]) {
1488  DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
1489  newTreeEntry(VL, false, UserTreeIdx);
1490  return;
1491  }
1492 
1493  auto &BSRef = BlocksSchedules[BB];
1494  if (!BSRef)
1495  BSRef = llvm::make_unique<BlockScheduling>(BB);
1496 
1497  BlockScheduling &BS = *BSRef.get();
1498 
1499  if (!BS.tryScheduleBundle(VL, this, S.OpValue)) {
1500  DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
1501  assert((!BS.getScheduleData(VL0) ||
1502  !BS.getScheduleData(VL0)->isPartOfBundle()) &&
1503  "tryScheduleBundle should cancelScheduling on failure");
1504  newTreeEntry(VL, false, UserTreeIdx);
1505  return;
1506  }
1507  DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
1508 
1509  unsigned ShuffleOrOp = S.IsAltShuffle ?
1510  (unsigned) Instruction::ShuffleVector : S.Opcode;
1511  switch (ShuffleOrOp) {
1512  case Instruction::PHI: {
1513  PHINode *PH = dyn_cast<PHINode>(VL0);
1514 
1515  // Check for terminator values (e.g. invoke).
1516  for (unsigned j = 0; j < VL.size(); ++j)
1517  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1519  cast<PHINode>(VL[j])->getIncomingValueForBlock(PH->getIncomingBlock(i)));
1520  if (Term) {
1521  DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n");
1522  BS.cancelScheduling(VL, VL0);
1523  newTreeEntry(VL, false, UserTreeIdx);
1524  return;
1525  }
1526  }
1527 
1528  newTreeEntry(VL, true, UserTreeIdx);
1529  DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
1530 
1531  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
1532  ValueList Operands;
1533  // Prepare the operand vector.
1534  for (Value *j : VL)
1535  Operands.push_back(cast<PHINode>(j)->getIncomingValueForBlock(
1536  PH->getIncomingBlock(i)));
1537 
1538  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1539  }
1540  return;
1541  }
1542  case Instruction::ExtractValue:
1543  case Instruction::ExtractElement: {
1544  bool Reuse = canReuseExtract(VL, VL0);
1545  if (Reuse) {
1546  DEBUG(dbgs() << "SLP: Reusing extract sequence.\n");
1547  } else {
1548  BS.cancelScheduling(VL, VL0);
1549  }
1550  newTreeEntry(VL, Reuse, UserTreeIdx);
1551  return;
1552  }
1553  case Instruction::Load: {
1554  // Check that a vectorized load would load the same memory as a scalar
1555  // load. For example, we don't want to vectorize loads that are smaller
1556  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
1557  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
1558  // from such a struct, we read/write packed bits disagreeing with the
1559  // unvectorized version.
1560  Type *ScalarTy = VL0->getType();
1561 
1562  if (DL->getTypeSizeInBits(ScalarTy) !=
1563  DL->getTypeAllocSizeInBits(ScalarTy)) {
1564  BS.cancelScheduling(VL, VL0);
1565  newTreeEntry(VL, false, UserTreeIdx);
1566  DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
1567  return;
1568  }
1569 
1570  // Make sure all loads in the bundle are simple - we can't vectorize
1571  // atomic or volatile loads.
1572  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1573  LoadInst *L = cast<LoadInst>(VL[i]);
1574  if (!L->isSimple()) {
1575  BS.cancelScheduling(VL, VL0);
1576  newTreeEntry(VL, false, UserTreeIdx);
1577  DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
1578  return;
1579  }
1580  }
1581 
1582  // Check if the loads are consecutive, reversed, or neither.
1583  // TODO: What we really want is to sort the loads, but for now, check
1584  // the two likely directions.
1585  bool Consecutive = true;
1586  bool ReverseConsecutive = true;
1587  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i) {
1588  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1589  Consecutive = false;
1590  break;
1591  } else {
1592  ReverseConsecutive = false;
1593  }
1594  }
1595 
1596  if (Consecutive) {
1597  ++NumLoadsWantToKeepOrder;
1598  newTreeEntry(VL, true, UserTreeIdx);
1599  DEBUG(dbgs() << "SLP: added a vector of loads.\n");
1600  return;
1601  }
1602 
1603  // If none of the load pairs were consecutive when checked in order,
1604  // check the reverse order.
1605  if (ReverseConsecutive)
1606  for (unsigned i = VL.size() - 1; i > 0; --i)
1607  if (!isConsecutiveAccess(VL[i], VL[i - 1], *DL, *SE)) {
1608  ReverseConsecutive = false;
1609  break;
1610  }
1611 
1612  BS.cancelScheduling(VL, VL0);
1613  newTreeEntry(VL, false, UserTreeIdx);
1614 
1615  if (ReverseConsecutive) {
1616  ++NumLoadsWantToChangeOrder;
1617  DEBUG(dbgs() << "SLP: Gathering reversed loads.\n");
1618  } else {
1619  DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
1620  }
1621  return;
1622  }
1623  case Instruction::ZExt:
1624  case Instruction::SExt:
1625  case Instruction::FPToUI:
1626  case Instruction::FPToSI:
1627  case Instruction::FPExt:
1628  case Instruction::PtrToInt:
1629  case Instruction::IntToPtr:
1630  case Instruction::SIToFP:
1631  case Instruction::UIToFP:
1632  case Instruction::Trunc:
1633  case Instruction::FPTrunc:
1634  case Instruction::BitCast: {
1635  Type *SrcTy = VL0->getOperand(0)->getType();
1636  for (unsigned i = 0; i < VL.size(); ++i) {
1637  Type *Ty = cast<Instruction>(VL[i])->getOperand(0)->getType();
1638  if (Ty != SrcTy || !isValidElementType(Ty)) {
1639  BS.cancelScheduling(VL, VL0);
1640  newTreeEntry(VL, false, UserTreeIdx);
1641  DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n");
1642  return;
1643  }
1644  }
1645  newTreeEntry(VL, true, UserTreeIdx);
1646  DEBUG(dbgs() << "SLP: added a vector of casts.\n");
1647 
1648  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1649  ValueList Operands;
1650  // Prepare the operand vector.
1651  for (Value *j : VL)
1652  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1653 
1654  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1655  }
1656  return;
1657  }
1658  case Instruction::ICmp:
1659  case Instruction::FCmp: {
1660  // Check that all of the compares have the same predicate.
1661  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
1662  Type *ComparedTy = VL0->getOperand(0)->getType();
1663  for (unsigned i = 1, e = VL.size(); i < e; ++i) {
1664  CmpInst *Cmp = cast<CmpInst>(VL[i]);
1665  if (Cmp->getPredicate() != P0 ||
1666  Cmp->getOperand(0)->getType() != ComparedTy) {
1667  BS.cancelScheduling(VL, VL0);
1668  newTreeEntry(VL, false, UserTreeIdx);
1669  DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
1670  return;
1671  }
1672  }
1673 
1674  newTreeEntry(VL, true, UserTreeIdx);
1675  DEBUG(dbgs() << "SLP: added a vector of compares.\n");
1676 
1677  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1678  ValueList Operands;
1679  // Prepare the operand vector.
1680  for (Value *j : VL)
1681  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1682 
1683  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1684  }
1685  return;
1686  }
1687  case Instruction::Select:
1688  case Instruction::Add:
1689  case Instruction::FAdd:
1690  case Instruction::Sub:
1691  case Instruction::FSub:
1692  case Instruction::Mul:
1693  case Instruction::FMul:
1694  case Instruction::UDiv:
1695  case Instruction::SDiv:
1696  case Instruction::FDiv:
1697  case Instruction::URem:
1698  case Instruction::SRem:
1699  case Instruction::FRem:
1700  case Instruction::Shl:
1701  case Instruction::LShr:
1702  case Instruction::AShr:
1703  case Instruction::And:
1704  case Instruction::Or:
1705  case Instruction::Xor:
1706  newTreeEntry(VL, true, UserTreeIdx);
1707  DEBUG(dbgs() << "SLP: added a vector of bin op.\n");
1708 
1709  // Sort operands of the instructions so that each side is more likely to
1710  // have the same opcode.
1711  if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
1712  ValueList Left, Right;
1713  reorderInputsAccordingToOpcode(S.Opcode, VL, Left, Right);
1714  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1715  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1716  return;
1717  }
1718 
1719  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1720  ValueList Operands;
1721  // Prepare the operand vector.
1722  for (Value *j : VL)
1723  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1724 
1725  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1726  }
1727  return;
1728 
1729  case Instruction::GetElementPtr: {
1730  // We don't combine GEPs with complicated (nested) indexing.
1731  for (unsigned j = 0; j < VL.size(); ++j) {
1732  if (cast<Instruction>(VL[j])->getNumOperands() != 2) {
1733  DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
1734  BS.cancelScheduling(VL, VL0);
1735  newTreeEntry(VL, false, UserTreeIdx);
1736  return;
1737  }
1738  }
1739 
1740  // We can't combine several GEPs into one vector if they operate on
1741  // different types.
1742  Type *Ty0 = VL0->getOperand(0)->getType();
1743  for (unsigned j = 0; j < VL.size(); ++j) {
1744  Type *CurTy = cast<Instruction>(VL[j])->getOperand(0)->getType();
1745  if (Ty0 != CurTy) {
1746  DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");
1747  BS.cancelScheduling(VL, VL0);
1748  newTreeEntry(VL, false, UserTreeIdx);
1749  return;
1750  }
1751  }
1752 
1753  // We don't combine GEPs with non-constant indexes.
1754  for (unsigned j = 0; j < VL.size(); ++j) {
1755  auto Op = cast<Instruction>(VL[j])->getOperand(1);
1756  if (!isa<ConstantInt>(Op)) {
1757  DEBUG(
1758  dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");
1759  BS.cancelScheduling(VL, VL0);
1760  newTreeEntry(VL, false, UserTreeIdx);
1761  return;
1762  }
1763  }
1764 
1765  newTreeEntry(VL, true, UserTreeIdx);
1766  DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
1767  for (unsigned i = 0, e = 2; i < e; ++i) {
1768  ValueList Operands;
1769  // Prepare the operand vector.
1770  for (Value *j : VL)
1771  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1772 
1773  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1774  }
1775  return;
1776  }
1777  case Instruction::Store: {
1778  // Check if the stores are consecutive or of we need to swizzle them.
1779  for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)
1780  if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {
1781  BS.cancelScheduling(VL, VL0);
1782  newTreeEntry(VL, false, UserTreeIdx);
1783  DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
1784  return;
1785  }
1786 
1787  newTreeEntry(VL, true, UserTreeIdx);
1788  DEBUG(dbgs() << "SLP: added a vector of stores.\n");
1789 
1790  ValueList Operands;
1791  for (Value *j : VL)
1792  Operands.push_back(cast<Instruction>(j)->getOperand(0));
1793 
1794  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1795  return;
1796  }
1797  case Instruction::Call: {
1798  // Check if the calls are all to the same vectorizable intrinsic.
1799  CallInst *CI = cast<CallInst>(VL0);
1800  // Check if this is an Intrinsic call or something that can be
1801  // represented by an intrinsic call
1803  if (!isTriviallyVectorizable(ID)) {
1804  BS.cancelScheduling(VL, VL0);
1805  newTreeEntry(VL, false, UserTreeIdx);
1806  DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
1807  return;
1808  }
1809  Function *Int = CI->getCalledFunction();
1810  Value *A1I = nullptr;
1811  if (hasVectorInstrinsicScalarOpd(ID, 1))
1812  A1I = CI->getArgOperand(1);
1813  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
1814  CallInst *CI2 = dyn_cast<CallInst>(VL[i]);
1815  if (!CI2 || CI2->getCalledFunction() != Int ||
1816  getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
1817  !CI->hasIdenticalOperandBundleSchema(*CI2)) {
1818  BS.cancelScheduling(VL, VL0);
1819  newTreeEntry(VL, false, UserTreeIdx);
1820  DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]
1821  << "\n");
1822  return;
1823  }
1824  // ctlz,cttz and powi are special intrinsics whose second argument
1825  // should be same in order for them to be vectorized.
1826  if (hasVectorInstrinsicScalarOpd(ID, 1)) {
1827  Value *A1J = CI2->getArgOperand(1);
1828  if (A1I != A1J) {
1829  BS.cancelScheduling(VL, VL0);
1830  newTreeEntry(VL, false, UserTreeIdx);
1831  DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
1832  << " argument "<< A1I<<"!=" << A1J
1833  << "\n");
1834  return;
1835  }
1836  }
1837  // Verify that the bundle operands are identical between the two calls.
1838  if (CI->hasOperandBundles() &&
1840  CI->op_begin() + CI->getBundleOperandsEndIndex(),
1841  CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
1842  BS.cancelScheduling(VL, VL0);
1843  newTreeEntry(VL, false, UserTreeIdx);
1844  DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="
1845  << *VL[i] << '\n');
1846  return;
1847  }
1848  }
1849 
1850  newTreeEntry(VL, true, UserTreeIdx);
1851  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
1852  ValueList Operands;
1853  // Prepare the operand vector.
1854  for (Value *j : VL) {
1855  CallInst *CI2 = dyn_cast<CallInst>(j);
1856  Operands.push_back(CI2->getArgOperand(i));
1857  }
1858  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1859  }
1860  return;
1861  }
1862  case Instruction::ShuffleVector:
1863  // If this is not an alternate sequence of opcode like add-sub
1864  // then do not vectorize this instruction.
1865  if (!S.IsAltShuffle) {
1866  BS.cancelScheduling(VL, VL0);
1867  newTreeEntry(VL, false, UserTreeIdx);
1868  DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
1869  return;
1870  }
1871  newTreeEntry(VL, true, UserTreeIdx);
1872  DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
1873 
1874  // Reorder operands if reordering would enable vectorization.
1875  if (isa<BinaryOperator>(VL0)) {
1876  ValueList Left, Right;
1877  reorderAltShuffleOperands(S.Opcode, VL, Left, Right);
1878  buildTree_rec(Left, Depth + 1, UserTreeIdx);
1879  buildTree_rec(Right, Depth + 1, UserTreeIdx);
1880  return;
1881  }
1882 
1883  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
1884  ValueList Operands;
1885  // Prepare the operand vector.
1886  for (Value *j : VL)
1887  Operands.push_back(cast<Instruction>(j)->getOperand(i));
1888 
1889  buildTree_rec(Operands, Depth + 1, UserTreeIdx);
1890  }
1891  return;
1892 
1893  default:
1894  BS.cancelScheduling(VL, VL0);
1895  newTreeEntry(VL, false, UserTreeIdx);
1896  DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
1897  return;
1898  }
1899 }
1900 
1901 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
1902  unsigned N;
1903  Type *EltTy;
1904  auto *ST = dyn_cast<StructType>(T);
1905  if (ST) {
1906  N = ST->getNumElements();
1907  EltTy = *ST->element_begin();
1908  } else {
1909  N = cast<ArrayType>(T)->getNumElements();
1910  EltTy = cast<ArrayType>(T)->getElementType();
1911  }
1912  if (!isValidElementType(EltTy))
1913  return 0;
1914  uint64_t VTSize = DL.getTypeStoreSizeInBits(VectorType::get(EltTy, N));
1915  if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
1916  return 0;
1917  if (ST) {
1918  // Check that struct is homogeneous.
1919  for (const auto *Ty : ST->elements())
1920  if (Ty != EltTy)
1921  return 0;
1922  }
1923  return N;
1924 }
1925 
1926 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue) const {
1927  Instruction *E0 = cast<Instruction>(OpValue);
1928  assert(E0->getOpcode() == Instruction::ExtractElement ||
1929  E0->getOpcode() == Instruction::ExtractValue);
1930  assert(E0->getOpcode() == getSameOpcode(VL).Opcode && "Invalid opcode");
1931  // Check if all of the extracts come from the same vector and from the
1932  // correct offset.
1933  Value *Vec = E0->getOperand(0);
1934 
1935  // We have to extract from a vector/aggregate with the same number of elements.
1936  unsigned NElts;
1937  if (E0->getOpcode() == Instruction::ExtractValue) {
1938  const DataLayout &DL = E0->getModule()->getDataLayout();
1939  NElts = canMapToVector(Vec->getType(), DL);
1940  if (!NElts)
1941  return false;
1942  // Check if load can be rewritten as load of vector.
1943  LoadInst *LI = dyn_cast<LoadInst>(Vec);
1944  if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
1945  return false;
1946  } else {
1947  NElts = Vec->getType()->getVectorNumElements();
1948  }
1949 
1950  if (NElts != VL.size())
1951  return false;
1952 
1953  // Check that all of the indices extract from the correct offset.
1954  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
1955  Instruction *Inst = cast<Instruction>(VL[I]);
1956  if (!matchExtractIndex(Inst, I, Inst->getOpcode()))
1957  return false;
1958  if (Inst->getOperand(0) != Vec)
1959  return false;
1960  }
1961 
1962  return true;
1963 }
1964 
1965 bool BoUpSLP::areAllUsersVectorized(Instruction *I) const {
1966  return I->hasOneUse() ||
1967  std::all_of(I->user_begin(), I->user_end(), [this](User *U) {
1968  return ScalarToTreeEntry.count(U) > 0;
1969  });
1970 }
1971 
1972 int BoUpSLP::getEntryCost(TreeEntry *E) {
1973  ArrayRef<Value*> VL = E->Scalars;
1974 
1975  Type *ScalarTy = VL[0]->getType();
1976  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
1977  ScalarTy = SI->getValueOperand()->getType();
1978  else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
1979  ScalarTy = CI->getOperand(0)->getType();
1980  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
1981 
1982  // If we have computed a smaller type for the expression, update VecTy so
1983  // that the costs will be accurate.
1984  if (MinBWs.count(VL[0]))
1985  VecTy = VectorType::get(
1986  IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
1987 
1988  if (E->NeedToGather) {
1989  if (allConstant(VL))
1990  return 0;
1991  if (isSplat(VL)) {
1992  return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy, 0);
1993  }
1994  if (getSameOpcode(VL).Opcode == Instruction::ExtractElement) {
1996  if (ShuffleKind.hasValue()) {
1997  int Cost = TTI->getShuffleCost(ShuffleKind.getValue(), VecTy);
1998  for (auto *V : VL) {
1999  // If all users of instruction are going to be vectorized and this
2000  // instruction itself is not going to be vectorized, consider this
2001  // instruction as dead and remove its cost from the final cost of the
2002  // vectorized tree.
2003  if (areAllUsersVectorized(cast<Instruction>(V)) &&
2004  !ScalarToTreeEntry.count(V)) {
2005  auto *IO = cast<ConstantInt>(
2006  cast<ExtractElementInst>(V)->getIndexOperand());
2007  Cost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
2008  IO->getZExtValue());
2009  }
2010  }
2011  return Cost;
2012  }
2013  }
2014  return getGatherCost(E->Scalars);
2015  }
2016  InstructionsState S = getSameOpcode(VL);
2017  assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
2018  Instruction *VL0 = cast<Instruction>(S.OpValue);
2019  unsigned ShuffleOrOp = S.IsAltShuffle ?
2020  (unsigned) Instruction::ShuffleVector : S.Opcode;
2021  switch (ShuffleOrOp) {
2022  case Instruction::PHI:
2023  return 0;
2024 
2025  case Instruction::ExtractValue:
2026  case Instruction::ExtractElement:
2027  if (canReuseExtract(VL, S.OpValue)) {
2028  int DeadCost = 0;
2029  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
2030  Instruction *E = cast<Instruction>(VL[i]);
2031  // If all users are going to be vectorized, instruction can be
2032  // considered as dead.
2033  // The same, if have only one user, it will be vectorized for sure.
2034  if (areAllUsersVectorized(E))
2035  // Take credit for instruction that will become dead.
2036  DeadCost +=
2037  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, i);
2038  }
2039  return -DeadCost;
2040  }
2041  return getGatherCost(VecTy);
2042 
2043  case Instruction::ZExt:
2044  case Instruction::SExt:
2045  case Instruction::FPToUI:
2046  case Instruction::FPToSI:
2047  case Instruction::FPExt:
2048  case Instruction::PtrToInt:
2049  case Instruction::IntToPtr:
2050  case Instruction::SIToFP:
2051  case Instruction::UIToFP:
2052  case Instruction::Trunc:
2053  case Instruction::FPTrunc:
2054  case Instruction::BitCast: {
2055  Type *SrcTy = VL0->getOperand(0)->getType();
2056 
2057  // Calculate the cost of this instruction.
2058  int ScalarCost = VL.size() * TTI->getCastInstrCost(VL0->getOpcode(),
2059  VL0->getType(), SrcTy, VL0);
2060 
2061  VectorType *SrcVecTy = VectorType::get(SrcTy, VL.size());
2062  int VecCost = TTI->getCastInstrCost(VL0->getOpcode(), VecTy, SrcVecTy, VL0);
2063  return VecCost - ScalarCost;
2064  }
2065  case Instruction::FCmp:
2066  case Instruction::ICmp:
2067  case Instruction::Select: {
2068  // Calculate the cost of this instruction.
2069  VectorType *MaskTy = VectorType::get(Builder.getInt1Ty(), VL.size());
2070  int ScalarCost = VecTy->getNumElements() *
2071  TTI->getCmpSelInstrCost(S.Opcode, ScalarTy, Builder.getInt1Ty(), VL0);
2072  int VecCost = TTI->getCmpSelInstrCost(S.Opcode, VecTy, MaskTy, VL0);
2073  return VecCost - ScalarCost;
2074  }
2075  case Instruction::Add:
2076  case Instruction::FAdd:
2077  case Instruction::Sub:
2078  case Instruction::FSub:
2079  case Instruction::Mul:
2080  case Instruction::FMul:
2081  case Instruction::UDiv:
2082  case Instruction::SDiv:
2083  case Instruction::FDiv:
2084  case Instruction::URem:
2085  case Instruction::SRem:
2086  case Instruction::FRem:
2087  case Instruction::Shl:
2088  case Instruction::LShr:
2089  case Instruction::AShr:
2090  case Instruction::And:
2091  case Instruction::Or:
2092  case Instruction::Xor: {
2093  // Certain instructions can be cheaper to vectorize if they have a
2094  // constant second vector operand.
2103 
2104  // If all operands are exactly the same ConstantInt then set the
2105  // operand kind to OK_UniformConstantValue.
2106  // If instead not all operands are constants, then set the operand kind
2107  // to OK_AnyValue. If all operands are constants but not the same,
2108  // then set the operand kind to OK_NonUniformConstantValue.
2109  ConstantInt *CInt = nullptr;
2110  for (unsigned i = 0; i < VL.size(); ++i) {
2111  const Instruction *I = cast<Instruction>(VL[i]);
2112  if (!isa<ConstantInt>(I->getOperand(1))) {
2114  break;
2115  }
2116  if (i == 0) {
2117  CInt = cast<ConstantInt>(I->getOperand(1));
2118  continue;
2119  }
2121  CInt != cast<ConstantInt>(I->getOperand(1)))
2123  }
2124  // FIXME: Currently cost of model modification for division by power of
2125  // 2 is handled for X86 and AArch64. Add support for other targets.
2126  if (Op2VK == TargetTransformInfo::OK_UniformConstantValue && CInt &&
2127  CInt->getValue().isPowerOf2())
2129 
2131  int ScalarCost =
2132  VecTy->getNumElements() *
2133  TTI->getArithmeticInstrCost(S.Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,
2134  Op2VP, Operands);
2135  int VecCost = TTI->getArithmeticInstrCost(S.Opcode, VecTy, Op1VK, Op2VK,
2136  Op1VP, Op2VP, Operands);
2137  return VecCost - ScalarCost;
2138  }
2139  case Instruction::GetElementPtr: {
2144 
2145  int ScalarCost =
2146  VecTy->getNumElements() *
2147  TTI->getArithmeticInstrCost(Instruction::Add, ScalarTy, Op1VK, Op2VK);
2148  int VecCost =
2149  TTI->getArithmeticInstrCost(Instruction::Add, VecTy, Op1VK, Op2VK);
2150 
2151  return VecCost - ScalarCost;
2152  }
2153  case Instruction::Load: {
2154  // Cost of wide load - cost of scalar loads.
2155  unsigned alignment = dyn_cast<LoadInst>(VL0)->getAlignment();
2156  int ScalarLdCost = VecTy->getNumElements() *
2157  TTI->getMemoryOpCost(Instruction::Load, ScalarTy, alignment, 0, VL0);
2158  int VecLdCost = TTI->getMemoryOpCost(Instruction::Load,
2159  VecTy, alignment, 0, VL0);
2160  return VecLdCost - ScalarLdCost;
2161  }
2162  case Instruction::Store: {
2163  // We know that we can merge the stores. Calculate the cost.
2164  unsigned alignment = dyn_cast<StoreInst>(VL0)->getAlignment();
2165  int ScalarStCost = VecTy->getNumElements() *
2166  TTI->getMemoryOpCost(Instruction::Store, ScalarTy, alignment, 0, VL0);
2167  int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
2168  VecTy, alignment, 0, VL0);
2169  return VecStCost - ScalarStCost;
2170  }
2171  case Instruction::Call: {
2172  CallInst *CI = cast<CallInst>(VL0);
2174 
2175  // Calculate the cost of the scalar and vector calls.
2176  SmallVector<Type*, 4> ScalarTys;
2177  for (unsigned op = 0, opc = CI->getNumArgOperands(); op!= opc; ++op)
2178  ScalarTys.push_back(CI->getArgOperand(op)->getType());
2179 
2180  FastMathFlags FMF;
2181  if (auto *FPMO = dyn_cast<FPMathOperator>(CI))
2182  FMF = FPMO->getFastMathFlags();
2183 
2184  int ScalarCallCost = VecTy->getNumElements() *
2185  TTI->getIntrinsicInstrCost(ID, ScalarTy, ScalarTys, FMF);
2186 
2188  int VecCallCost = TTI->getIntrinsicInstrCost(ID, CI->getType(), Args, FMF,
2189  VecTy->getNumElements());
2190 
2191  DEBUG(dbgs() << "SLP: Call cost "<< VecCallCost - ScalarCallCost
2192  << " (" << VecCallCost << "-" << ScalarCallCost << ")"
2193  << " for " << *CI << "\n");
2194 
2195  return VecCallCost - ScalarCallCost;
2196  }
2197  case Instruction::ShuffleVector: {
2202  int ScalarCost = 0;
2203  int VecCost = 0;
2204  for (Value *i : VL) {
2205  Instruction *I = cast<Instruction>(i);
2206  if (!I)
2207  break;
2208  ScalarCost +=
2209  TTI->getArithmeticInstrCost(I->getOpcode(), ScalarTy, Op1VK, Op2VK);
2210  }
2211  // VecCost is equal to sum of the cost of creating 2 vectors
2212  // and the cost of creating shuffle.
2213  Instruction *I0 = cast<Instruction>(VL[0]);
2214  VecCost =
2215  TTI->getArithmeticInstrCost(I0->getOpcode(), VecTy, Op1VK, Op2VK);
2216  Instruction *I1 = cast<Instruction>(VL[1]);
2217  VecCost +=
2218  TTI->getArithmeticInstrCost(I1->getOpcode(), VecTy, Op1VK, Op2VK);
2219  VecCost +=
2220  TTI->getShuffleCost(TargetTransformInfo::SK_Alternate, VecTy, 0);
2221  return VecCost - ScalarCost;
2222  }
2223  default:
2224  llvm_unreachable("Unknown instruction");
2225  }
2226 }
2227 
2228 bool BoUpSLP::isFullyVectorizableTinyTree() {
2229  DEBUG(dbgs() << "SLP: Check whether the tree with height " <<
2230  VectorizableTree.size() << " is fully vectorizable .\n");
2231 
2232  // We only handle trees of heights 1 and 2.
2233  if (VectorizableTree.size() == 1 && !VectorizableTree[0].NeedToGather)
2234  return true;
2235 
2236  if (VectorizableTree.size() != 2)
2237  return false;
2238 
2239  // Handle splat and all-constants stores.
2240  if (!VectorizableTree[0].NeedToGather &&
2241  (allConstant(VectorizableTree[1].Scalars) ||
2242  isSplat(VectorizableTree[1].Scalars)))
2243  return true;
2244 
2245  // Gathering cost would be too much for tiny trees.
2246  if (VectorizableTree[0].NeedToGather || VectorizableTree[1].NeedToGather)
2247  return false;
2248 
2249  return true;
2250 }
2251 
2252 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() {
2253  // We can vectorize the tree if its size is greater than or equal to the
2254  // minimum size specified by the MinTreeSize command line option.
2255  if (VectorizableTree.size() >= MinTreeSize)
2256  return false;
2257 
2258  // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
2259  // can vectorize it if we can prove it fully vectorizable.
2260  if (isFullyVectorizableTinyTree())
2261  return false;
2262 
2263  assert(VectorizableTree.empty()
2264  ? ExternalUses.empty()
2265  : true && "We shouldn't have any external users");
2266 
2267  // Otherwise, we can't vectorize the tree. It is both tiny and not fully
2268  // vectorizable.
2269  return true;
2270 }
2271 
2272 int BoUpSLP::getSpillCost() {
2273  // Walk from the bottom of the tree to the top, tracking which values are
2274  // live. When we see a call instruction that is not part of our tree,
2275  // query TTI to see if there is a cost to keeping values live over it
2276  // (for example, if spills and fills are required).
2277  unsigned BundleWidth = VectorizableTree.front().Scalars.size();
2278  int Cost = 0;
2279 
2280  SmallPtrSet<Instruction*, 4> LiveValues;
2281  Instruction *PrevInst = nullptr;
2282 
2283  for (const auto &N : VectorizableTree) {
2284  Instruction *Inst = dyn_cast<Instruction>(N.Scalars[0]);
2285  if (!Inst)
2286  continue;
2287 
2288  if (!PrevInst) {
2289  PrevInst = Inst;
2290  continue;
2291  }
2292 
2293  // Update LiveValues.
2294  LiveValues.erase(PrevInst);
2295  for (auto &J : PrevInst->operands()) {
2296  if (isa<Instruction>(&*J) && getTreeEntry(&*J))
2297  LiveValues.insert(cast<Instruction>(&*J));
2298  }
2299 
2300  DEBUG(
2301  dbgs() << "SLP: #LV: " << LiveValues.size();
2302  for (auto *X : LiveValues)
2303  dbgs() << " " << X->getName();
2304  dbgs() << ", Looking at ";
2305  Inst->dump();
2306  );
2307 
2308  // Now find the sequence of instructions between PrevInst and Inst.
2310  PrevInstIt =
2311  PrevInst->getIterator().getReverse();
2312  while (InstIt != PrevInstIt) {
2313  if (PrevInstIt == PrevInst->getParent()->rend()) {
2314  PrevInstIt = Inst->getParent()->rbegin();
2315  continue;
2316  }
2317 
2318  if (isa<CallInst>(&*PrevInstIt) && &*PrevInstIt != PrevInst) {
2320  for (auto *II : LiveValues)
2321  V.push_back(VectorType::get(II->getType(), BundleWidth));
2322  Cost += TTI->getCostOfKeepingLiveOverCall(V);
2323  }
2324 
2325  ++PrevInstIt;
2326  }
2327 
2328  PrevInst = Inst;
2329  }
2330 
2331  return Cost;
2332 }
2333 
2334 int BoUpSLP::getTreeCost() {
2335  int Cost = 0;
2336  DEBUG(dbgs() << "SLP: Calculating cost for tree of size " <<
2337  VectorizableTree.size() << ".\n");
2338 
2339  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
2340 
2341  for (TreeEntry &TE : VectorizableTree) {
2342  int C = getEntryCost(&TE);
2343  DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "
2344  << *TE.Scalars[0] << ".\n");
2345  Cost += C;
2346  }
2347 
2348  SmallSet<Value *, 16> ExtractCostCalculated;
2349  int ExtractCost = 0;
2350  for (ExternalUser &EU : ExternalUses) {
2351  // We only add extract cost once for the same scalar.
2352  if (!ExtractCostCalculated.insert(EU.Scalar).second)
2353  continue;
2354 
2355  // Uses by ephemeral values are free (because the ephemeral value will be
2356  // removed prior to code generation, and so the extraction will be
2357  // removed as well).
2358  if (EphValues.count(EU.User))
2359  continue;
2360 
2361  // If we plan to rewrite the tree in a smaller type, we will need to sign
2362  // extend the extracted value back to the original type. Here, we account
2363  // for the extract and the added cost of the sign extend if needed.
2364  auto *VecTy = VectorType::get(EU.Scalar->getType(), BundleWidth);
2365  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
2366  if (MinBWs.count(ScalarRoot)) {
2367  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
2368  auto Extend =
2369  MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
2370  VecTy = VectorType::get(MinTy, BundleWidth);
2371  ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
2372  VecTy, EU.Lane);
2373  } else {
2374  ExtractCost +=
2375  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
2376  }
2377  }
2378 
2379  int SpillCost = getSpillCost();
2380  Cost += SpillCost + ExtractCost;
2381 
2382  std::string Str;
2383  {
2384  raw_string_ostream OS(Str);
2385  OS << "SLP: Spill Cost = " << SpillCost << ".\n"
2386  << "SLP: Extract Cost = " << ExtractCost << ".\n"
2387  << "SLP: Total Cost = " << Cost << ".\n";
2388  }
2389  DEBUG(dbgs() << Str);
2390 
2391  if (ViewSLPTree)
2392  ViewGraph(this, "SLP" + F->getName(), false, Str);
2393 
2394  return Cost;
2395 }
2396 
2397 int BoUpSLP::getGatherCost(Type *Ty) {
2398  int Cost = 0;
2399  for (unsigned i = 0, e = cast<VectorType>(Ty)->getNumElements(); i < e; ++i)
2400  Cost += TTI->getVectorInstrCost(Instruction::InsertElement, Ty, i);
2401  return Cost;
2402 }
2403 
2404 int BoUpSLP::getGatherCost(ArrayRef<Value *> VL) {
2405  // Find the type of the operands in VL.
2406  Type *ScalarTy = VL[0]->getType();
2407  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
2408  ScalarTy = SI->getValueOperand()->getType();
2409  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2410  // Find the cost of inserting/extracting values from the vector.
2411  return getGatherCost(VecTy);
2412 }
2413 
2414 // Reorder commutative operations in alternate shuffle if the resulting vectors
2415 // are consecutive loads. This would allow us to vectorize the tree.
2416 // If we have something like-
2417 // load a[0] - load b[0]
2418 // load b[1] + load a[1]
2419 // load a[2] - load b[2]
2420 // load a[3] + load b[3]
2421 // Reordering the second load b[1] load a[1] would allow us to vectorize this
2422 // code.
2423 void BoUpSLP::reorderAltShuffleOperands(unsigned Opcode, ArrayRef<Value *> VL,
2426  // Push left and right operands of binary operation into Left and Right
2427  unsigned AltOpcode = getAltOpcode(Opcode);
2428  (void)AltOpcode;
2429  for (Value *V : VL) {
2430  auto *I = cast<Instruction>(V);
2431  assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&
2432  "Incorrect instruction in vector");
2433  Left.push_back(I->getOperand(0));
2434  Right.push_back(I->getOperand(1));
2435  }
2436 
2437  // Reorder if we have a commutative operation and consecutive access
2438  // are on either side of the alternate instructions.
2439  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2440  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2441  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2442  Instruction *VL1 = cast<Instruction>(VL[j]);
2443  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2444  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2445  std::swap(Left[j], Right[j]);
2446  continue;
2447  } else if (VL2->isCommutative() &&
2448  isConsecutiveAccess(L, L1, *DL, *SE)) {
2449  std::swap(Left[j + 1], Right[j + 1]);
2450  continue;
2451  }
2452  // else unchanged
2453  }
2454  }
2455  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2456  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2457  Instruction *VL1 = cast<Instruction>(VL[j]);
2458  Instruction *VL2 = cast<Instruction>(VL[j + 1]);
2459  if (VL1->isCommutative() && isConsecutiveAccess(L, L1, *DL, *SE)) {
2460  std::swap(Left[j], Right[j]);
2461  continue;
2462  } else if (VL2->isCommutative() &&
2463  isConsecutiveAccess(L, L1, *DL, *SE)) {
2464  std::swap(Left[j + 1], Right[j + 1]);
2465  continue;
2466  }
2467  // else unchanged
2468  }
2469  }
2470  }
2471 }
2472 
2473 // Return true if I should be commuted before adding it's left and right
2474 // operands to the arrays Left and Right.
2475 //
2476 // The vectorizer is trying to either have all elements one side being
2477 // instruction with the same opcode to enable further vectorization, or having
2478 // a splat to lower the vectorizing cost.
2480  int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,
2481  ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,
2482  bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {
2483  VLeft = I.getOperand(0);
2484  VRight = I.getOperand(1);
2485  // If we have "SplatRight", try to see if commuting is needed to preserve it.
2486  if (SplatRight) {
2487  if (VRight == Right[i - 1])
2488  // Preserve SplatRight
2489  return false;
2490  if (VLeft == Right[i - 1]) {
2491  // Commuting would preserve SplatRight, but we don't want to break
2492  // SplatLeft either, i.e. preserve the original order if possible.
2493  // (FIXME: why do we care?)
2494  if (SplatLeft && VLeft == Left[i - 1])
2495  return false;
2496  return true;
2497  }
2498  }
2499  // Symmetrically handle Right side.
2500  if (SplatLeft) {
2501  if (VLeft == Left[i - 1])
2502  // Preserve SplatLeft
2503  return false;
2504  if (VRight == Left[i - 1])
2505  return true;
2506  }
2507 
2508  Instruction *ILeft = dyn_cast<Instruction>(VLeft);
2509  Instruction *IRight = dyn_cast<Instruction>(VRight);
2510 
2511  // If we have "AllSameOpcodeRight", try to see if the left operands preserves
2512  // it and not the right, in this case we want to commute.
2513  if (AllSameOpcodeRight) {
2514  unsigned RightPrevOpcode = cast<Instruction>(Right[i - 1])->getOpcode();
2515  if (IRight && RightPrevOpcode == IRight->getOpcode())
2516  // Do not commute, a match on the right preserves AllSameOpcodeRight
2517  return false;
2518  if (ILeft && RightPrevOpcode == ILeft->getOpcode()) {
2519  // We have a match and may want to commute, but first check if there is
2520  // not also a match on the existing operands on the Left to preserve
2521  // AllSameOpcodeLeft, i.e. preserve the original order if possible.
2522  // (FIXME: why do we care?)
2523  if (AllSameOpcodeLeft && ILeft &&
2524  cast<Instruction>(Left[i - 1])->getOpcode() == ILeft->getOpcode())
2525  return false;
2526  return true;
2527  }
2528  }
2529  // Symmetrically handle Left side.
2530  if (AllSameOpcodeLeft) {
2531  unsigned LeftPrevOpcode = cast<Instruction>(Left[i - 1])->getOpcode();
2532  if (ILeft && LeftPrevOpcode == ILeft->getOpcode())
2533  return false;
2534  if (IRight && LeftPrevOpcode == IRight->getOpcode())
2535  return true;
2536  }
2537  return false;
2538 }
2539 
2540 void BoUpSLP::reorderInputsAccordingToOpcode(unsigned Opcode,
2541  ArrayRef<Value *> VL,
2543  SmallVectorImpl<Value *> &Right) {
2544  if (!VL.empty()) {
2545  // Peel the first iteration out of the loop since there's nothing
2546  // interesting to do anyway and it simplifies the checks in the loop.
2547  auto *I = cast<Instruction>(VL[0]);
2548  Value *VLeft = I->getOperand(0);
2549  Value *VRight = I->getOperand(1);
2550  if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))
2551  // Favor having instruction to the right. FIXME: why?
2552  std::swap(VLeft, VRight);
2553  Left.push_back(VLeft);
2554  Right.push_back(VRight);
2555  }
2556 
2557  // Keep track if we have instructions with all the same opcode on one side.
2558  bool AllSameOpcodeLeft = isa<Instruction>(Left[0]);
2559  bool AllSameOpcodeRight = isa<Instruction>(Right[0]);
2560  // Keep track if we have one side with all the same value (broadcast).
2561  bool SplatLeft = true;
2562  bool SplatRight = true;
2563 
2564  for (unsigned i = 1, e = VL.size(); i != e; ++i) {
2565  Instruction *I = cast<Instruction>(VL[i]);
2566  assert(((I->getOpcode() == Opcode && I->isCommutative()) ||
2567  (I->getOpcode() != Opcode && Instruction::isCommutative(Opcode))) &&
2568  "Can only process commutative instruction");
2569  // Commute to favor either a splat or maximizing having the same opcodes on
2570  // one side.
2571  Value *VLeft;
2572  Value *VRight;
2573  if (shouldReorderOperands(i, Opcode, *I, Left, Right, AllSameOpcodeLeft,
2574  AllSameOpcodeRight, SplatLeft, SplatRight, VLeft,
2575  VRight)) {
2576  Left.push_back(VRight);
2577  Right.push_back(VLeft);
2578  } else {
2579  Left.push_back(VLeft);
2580  Right.push_back(VRight);
2581  }
2582  // Update Splat* and AllSameOpcode* after the insertion.
2583  SplatRight = SplatRight && (Right[i - 1] == Right[i]);
2584  SplatLeft = SplatLeft && (Left[i - 1] == Left[i]);
2585  AllSameOpcodeLeft = AllSameOpcodeLeft && isa<Instruction>(Left[i]) &&
2586  (cast<Instruction>(Left[i - 1])->getOpcode() ==
2587  cast<Instruction>(Left[i])->getOpcode());
2588  AllSameOpcodeRight = AllSameOpcodeRight && isa<Instruction>(Right[i]) &&
2589  (cast<Instruction>(Right[i - 1])->getOpcode() ==
2590  cast<Instruction>(Right[i])->getOpcode());
2591  }
2592 
2593  // If one operand end up being broadcast, return this operand order.
2594  if (SplatRight || SplatLeft)
2595  return;
2596 
2597  // Finally check if we can get longer vectorizable chain by reordering
2598  // without breaking the good operand order detected above.
2599  // E.g. If we have something like-
2600  // load a[0] load b[0]
2601  // load b[1] load a[1]
2602  // load a[2] load b[2]
2603  // load a[3] load b[3]
2604  // Reordering the second load b[1] load a[1] would allow us to vectorize
2605  // this code and we still retain AllSameOpcode property.
2606  // FIXME: This load reordering might break AllSameOpcode in some rare cases
2607  // such as-
2608  // add a[0],c[0] load b[0]
2609  // add a[1],c[2] load b[1]
2610  // b[2] load b[2]
2611  // add a[3],c[3] load b[3]
2612  for (unsigned j = 0; j < VL.size() - 1; ++j) {
2613  if (LoadInst *L = dyn_cast<LoadInst>(Left[j])) {
2614  if (LoadInst *L1 = dyn_cast<LoadInst>(Right[j + 1])) {
2615  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2616  std::swap(Left[j + 1], Right[j + 1]);
2617  continue;
2618  }
2619  }
2620  }
2621  if (LoadInst *L = dyn_cast<LoadInst>(Right[j])) {
2622  if (LoadInst *L1 = dyn_cast<LoadInst>(Left[j + 1])) {
2623  if (isConsecutiveAccess(L, L1, *DL, *SE)) {
2624  std::swap(Left[j + 1], Right[j + 1]);
2625  continue;
2626  }
2627  }
2628  }
2629  // else unchanged
2630  }
2631 }
2632 
2633 void BoUpSLP::setInsertPointAfterBundle(ArrayRef<Value *> VL, Value *OpValue) {
2634  // Get the basic block this bundle is in. All instructions in the bundle
2635  // should be in this block.
2636  auto *Front = cast<Instruction>(OpValue);
2637  auto *BB = Front->getParent();
2638  const unsigned Opcode = cast<Instruction>(OpValue)->getOpcode();
2639  const unsigned AltOpcode = getAltOpcode(Opcode);
2640  assert(llvm::all_of(make_range(VL.begin(), VL.end()), [=](Value *V) -> bool {
2641  return !sameOpcodeOrAlt(Opcode, AltOpcode,
2642  cast<Instruction>(V)->getOpcode()) ||
2643  cast<Instruction>(V)->getParent() == BB;
2644  }));
2645 
2646  // The last instruction in the bundle in program order.
2647  Instruction *LastInst = nullptr;
2648 
2649  // Find the last instruction. The common case should be that BB has been
2650  // scheduled, and the last instruction is VL.back(). So we start with
2651  // VL.back() and iterate over schedule data until we reach the end of the
2652  // bundle. The end of the bundle is marked by null ScheduleData.
2653  if (BlocksSchedules.count(BB)) {
2654  auto *Bundle =
2655  BlocksSchedules[BB]->getScheduleData(isOneOf(OpValue, VL.back()));
2656  if (Bundle && Bundle->isPartOfBundle())
2657  for (; Bundle; Bundle = Bundle->NextInBundle)
2658  if (Bundle->OpValue == Bundle->Inst)
2659  LastInst = Bundle->Inst;
2660  }
2661 
2662  // LastInst can still be null at this point if there's either not an entry
2663  // for BB in BlocksSchedules or there's no ScheduleData available for
2664  // VL.back(). This can be the case if buildTree_rec aborts for various
2665  // reasons (e.g., the maximum recursion depth is reached, the maximum region
2666  // size is reached, etc.). ScheduleData is initialized in the scheduling
2667  // "dry-run".
2668  //
2669  // If this happens, we can still find the last instruction by brute force. We
2670  // iterate forwards from Front (inclusive) until we either see all
2671  // instructions in the bundle or reach the end of the block. If Front is the
2672  // last instruction in program order, LastInst will be set to Front, and we
2673  // will visit all the remaining instructions in the block.
2674  //
2675  // One of the reasons we exit early from buildTree_rec is to place an upper
2676  // bound on compile-time. Thus, taking an additional compile-time hit here is
2677  // not ideal. However, this should be exceedingly rare since it requires that
2678  // we both exit early from buildTree_rec and that the bundle be out-of-order
2679  // (causing us to iterate all the way to the end of the block).
2680  if (!LastInst) {
2681  SmallPtrSet<Value *, 16> Bundle(VL.begin(), VL.end());
2682  for (auto &I : make_range(BasicBlock::iterator(Front), BB->end())) {
2683  if (Bundle.erase(&I) && sameOpcodeOrAlt(Opcode, AltOpcode, I.getOpcode()))
2684  LastInst = &I;
2685  if (Bundle.empty())
2686  break;
2687  }
2688  }
2689 
2690  // Set the insertion point after the last instruction in the bundle. Set the
2691  // debug location to Front.
2692  Builder.SetInsertPoint(BB, ++LastInst->getIterator());
2693  Builder.SetCurrentDebugLocation(Front->getDebugLoc());
2694 }
2695 
2696 Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
2697  Value *Vec = UndefValue::get(Ty);
2698  // Generate the 'InsertElement' instruction.
2699  for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
2700  Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
2701  if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
2702  GatherSeq.insert(Insrt);
2703  CSEBlocks.insert(Insrt->getParent());
2704 
2705  // Add to our 'need-to-extract' list.
2706  if (TreeEntry *E = getTreeEntry(VL[i])) {
2707  // Find which lane we need to extract.
2708  int FoundLane = -1;
2709  for (unsigned Lane = 0, LE = VL.size(); Lane != LE; ++Lane) {
2710  // Is this the lane of the scalar that we are looking for ?
2711  if (E->Scalars[Lane] == VL[i]) {
2712  FoundLane = Lane;
2713  break;
2714  }
2715  }
2716  assert(FoundLane >= 0 && "Could not find the correct lane");
2717  ExternalUses.push_back(ExternalUser(VL[i], Insrt, FoundLane));
2718  }
2719  }
2720  }
2721 
2722  return Vec;
2723 }
2724 
2725 Value *BoUpSLP::alreadyVectorized(ArrayRef<Value *> VL, Value *OpValue) const {
2726  if (const TreeEntry *En = getTreeEntry(OpValue)) {
2727  if (En->isSame(VL) && En->VectorizedValue)
2728  return En->VectorizedValue;
2729  }
2730  return nullptr;
2731 }
2732 
2733 Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
2734  InstructionsState S = getSameOpcode(VL);
2735  if (S.Opcode) {
2736  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
2737  if (E->isSame(VL))
2738  return vectorizeTree(E);
2739  }
2740  }
2741 
2742  Type *ScalarTy = S.OpValue->getType();
2743  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
2744  ScalarTy = SI->getValueOperand()->getType();
2745  VectorType *VecTy = VectorType::get(ScalarTy, VL.size());
2746 
2747  return Gather(VL, VecTy);
2748 }
2749 
2750 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
2751  IRBuilder<>::InsertPointGuard Guard(Builder);
2752 
2753  if (E->VectorizedValue) {
2754  DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");
2755  return E->VectorizedValue;
2756  }
2757 
2758  InstructionsState S = getSameOpcode(E->Scalars);
2759  Instruction *VL0 = cast<Instruction>(E->Scalars[0]);
2760  Type *ScalarTy = VL0->getType();
2761  if (StoreInst *SI = dyn_cast<StoreInst>(VL0))
2762  ScalarTy = SI->getValueOperand()->getType();
2763  VectorType *VecTy = VectorType::get(ScalarTy, E->Scalars.size());
2764 
2765  if (E->NeedToGather) {
2766  setInsertPointAfterBundle(E->Scalars, VL0);
2767  auto *V = Gather(E->Scalars, VecTy);
2768  E->VectorizedValue = V;
2769  return V;
2770  }
2771 
2772  unsigned ShuffleOrOp = S.IsAltShuffle ?
2773  (unsigned) Instruction::ShuffleVector : S.Opcode;
2774  switch (ShuffleOrOp) {
2775  case Instruction::PHI: {
2776  PHINode *PH = dyn_cast<PHINode>(VL0);
2777  Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
2778  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2779  PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
2780  E->VectorizedValue = NewPhi;
2781 
2782  // PHINodes may have multiple entries from the same block. We want to
2783  // visit every block once.
2784  SmallSet<BasicBlock*, 4> VisitedBBs;
2785 
2786  for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {
2787  ValueList Operands;
2788  BasicBlock *IBB = PH->getIncomingBlock(i);
2789 
2790  if (!VisitedBBs.insert(IBB).second) {
2791  NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
2792  continue;
2793  }
2794 
2795  // Prepare the operand vector.
2796  for (Value *V : E->Scalars)
2797  Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(IBB));
2798 
2799  Builder.SetInsertPoint(IBB->getTerminator());
2800  Builder.SetCurrentDebugLocation(PH->getDebugLoc());
2801  Value *Vec = vectorizeTree(Operands);
2802  NewPhi->addIncoming(Vec, IBB);
2803  }
2804 
2805  assert(NewPhi->getNumIncomingValues() == PH->getNumIncomingValues() &&
2806  "Invalid number of incoming values");
2807  return NewPhi;
2808  }
2809 
2810  case Instruction::ExtractElement: {
2811  if (canReuseExtract(E->Scalars, VL0)) {
2812  Value *V = VL0->getOperand(0);
2813  E->VectorizedValue = V;
2814  return V;
2815  }
2816  setInsertPointAfterBundle(E->Scalars, VL0);
2817  auto *V = Gather(E->Scalars, VecTy);
2818  E->VectorizedValue = V;
2819  return V;
2820  }
2821  case Instruction::ExtractValue: {
2822  if (canReuseExtract(E->Scalars, VL0)) {
2823  LoadInst *LI = cast<LoadInst>(VL0->getOperand(0));
2824  Builder.SetInsertPoint(LI);
2825  PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
2826  Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
2827  LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
2828  E->VectorizedValue = V;
2829  return propagateMetadata(V, E->Scalars);
2830  }
2831  setInsertPointAfterBundle(E->Scalars, VL0);
2832  auto *V = Gather(E->Scalars, VecTy);
2833  E->VectorizedValue = V;
2834  return V;
2835  }
2836  case Instruction::ZExt:
2837  case Instruction::SExt:
2838  case Instruction::FPToUI:
2839  case Instruction::FPToSI:
2840  case Instruction::FPExt:
2841  case Instruction::PtrToInt:
2842  case Instruction::IntToPtr:
2843  case Instruction::SIToFP:
2844  case Instruction::UIToFP:
2845  case Instruction::Trunc:
2846  case Instruction::FPTrunc:
2847  case Instruction::BitCast: {
2848  ValueList INVL;
2849  for (Value *V : E->Scalars)
2850  INVL.push_back(cast<Instruction>(V)->getOperand(0));
2851 
2852  setInsertPointAfterBundle(E->Scalars, VL0);
2853 
2854  Value *InVec = vectorizeTree(INVL);
2855 
2856  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2857  return V;
2858 
2859  CastInst *CI = dyn_cast<CastInst>(VL0);
2860  Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
2861  E->VectorizedValue = V;
2862  ++NumVectorInstructions;
2863  return V;
2864  }
2865  case Instruction::FCmp:
2866  case Instruction::ICmp: {
2867  ValueList LHSV, RHSV;
2868  for (Value *V : E->Scalars) {
2869  LHSV.push_back(cast<Instruction>(V)->getOperand(0));
2870  RHSV.push_back(cast<Instruction>(V)->getOperand(1));
2871  }
2872 
2873  setInsertPointAfterBundle(E->Scalars, VL0);
2874 
2875  Value *L = vectorizeTree(LHSV);
2876  Value *R = vectorizeTree(RHSV);
2877 
2878  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2879  return V;
2880 
2881  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
2882  Value *V;
2883  if (S.Opcode == Instruction::FCmp)
2884  V = Builder.CreateFCmp(P0, L, R);
2885  else
2886  V = Builder.CreateICmp(P0, L, R);
2887 
2888  E->VectorizedValue = V;
2889  propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2890  ++NumVectorInstructions;
2891  return V;
2892  }
2893  case Instruction::Select: {
2894  ValueList TrueVec, FalseVec, CondVec;
2895  for (Value *V : E->Scalars) {
2896  CondVec.push_back(cast<Instruction>(V)->getOperand(0));
2897  TrueVec.push_back(cast<Instruction>(V)->getOperand(1));
2898  FalseVec.push_back(cast<Instruction>(V)->getOperand(2));
2899  }
2900 
2901  setInsertPointAfterBundle(E->Scalars, VL0);
2902 
2903  Value *Cond = vectorizeTree(CondVec);
2904  Value *True = vectorizeTree(TrueVec);
2905  Value *False = vectorizeTree(FalseVec);
2906 
2907  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2908  return V;
2909 
2910  Value *V = Builder.CreateSelect(Cond, True, False);
2911  E->VectorizedValue = V;
2912  ++NumVectorInstructions;
2913  return V;
2914  }
2915  case Instruction::Add:
2916  case Instruction::FAdd:
2917  case Instruction::Sub:
2918  case Instruction::FSub:
2919  case Instruction::Mul:
2920  case Instruction::FMul:
2921  case Instruction::UDiv:
2922  case Instruction::SDiv:
2923  case Instruction::FDiv:
2924  case Instruction::URem:
2925  case Instruction::SRem:
2926  case Instruction::FRem:
2927  case Instruction::Shl:
2928  case Instruction::LShr:
2929  case Instruction::AShr:
2930  case Instruction::And:
2931  case Instruction::Or:
2932  case Instruction::Xor: {
2933  ValueList LHSVL, RHSVL;
2934  if (isa<BinaryOperator>(VL0) && VL0->isCommutative())
2935  reorderInputsAccordingToOpcode(S.Opcode, E->Scalars, LHSVL,
2936  RHSVL);
2937  else
2938  for (Value *V : E->Scalars) {
2939  auto *I = cast<Instruction>(V);
2940  LHSVL.push_back(I->getOperand(0));
2941  RHSVL.push_back(I->getOperand(1));
2942  }
2943 
2944  setInsertPointAfterBundle(E->Scalars, VL0);
2945 
2946  Value *LHS = vectorizeTree(LHSVL);
2947  Value *RHS = vectorizeTree(RHSVL);
2948 
2949  if (Value *V = alreadyVectorized(E->Scalars, VL0))
2950  return V;
2951 
2952  Value *V = Builder.CreateBinOp(
2953  static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
2954  E->VectorizedValue = V;
2955  propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
2956  ++NumVectorInstructions;
2957 
2958  if (Instruction *I = dyn_cast<Instruction>(V))
2959  return propagateMetadata(I, E->Scalars);
2960 
2961  return V;
2962  }
2963  case Instruction::Load: {
2964  // Loads are inserted at the head of the tree because we don't want to
2965  // sink them all the way down past store instructions.
2966  setInsertPointAfterBundle(E->Scalars, VL0);
2967 
2968  LoadInst *LI = cast<LoadInst>(VL0);
2969  Type *ScalarLoadTy = LI->getType();
2970  unsigned AS = LI->getPointerAddressSpace();
2971 
2972  Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
2973  VecTy->getPointerTo(AS));
2974 
2975  // The pointer operand uses an in-tree scalar so we add the new BitCast to
2976  // ExternalUses list to make sure that an extract will be generated in the
2977  // future.
2978  Value *PO = LI->getPointerOperand();
2979  if (getTreeEntry(PO))
2980  ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
2981 
2982  unsigned Alignment = LI->getAlignment();
2983  LI = Builder.CreateLoad(VecPtr);
2984  if (!Alignment) {
2985  Alignment = DL->getABITypeAlignment(ScalarLoadTy);
2986  }
2987  LI->setAlignment(Alignment);
2988  E->VectorizedValue = LI;
2989  ++NumVectorInstructions;
2990  return propagateMetadata(LI, E->Scalars);
2991  }
2992  case Instruction::Store: {
2993  StoreInst *SI = cast<StoreInst>(VL0);
2994  unsigned Alignment = SI->getAlignment();
2995  unsigned AS = SI->getPointerAddressSpace();
2996 
2997  ValueList ScalarStoreValues;
2998  for (Value *V : E->Scalars)
2999  ScalarStoreValues.push_back(cast<StoreInst>(V)->getValueOperand());
3000 
3001  setInsertPointAfterBundle(E->Scalars, VL0);
3002 
3003  Value *VecValue = vectorizeTree(ScalarStoreValues);
3004  Value *ScalarPtr = SI->getPointerOperand();
3005  Value *VecPtr = Builder.CreateBitCast(ScalarPtr, VecTy->getPointerTo(AS));
3006  StoreInst *S = Builder.CreateStore(VecValue, VecPtr);
3007 
3008  // The pointer operand uses an in-tree scalar, so add the new BitCast to
3009  // ExternalUses to make sure that an extract will be generated in the
3010  // future.
3011  if (getTreeEntry(ScalarPtr))
3012  ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
3013 
3014  if (!Alignment)
3015  Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
3016 
3017  S->setAlignment(Alignment);
3018  E->VectorizedValue = S;
3019  ++NumVectorInstructions;
3020  return propagateMetadata(S, E->Scalars);
3021  }
3022  case Instruction::GetElementPtr: {
3023  setInsertPointAfterBundle(E->Scalars, VL0);
3024 
3025  ValueList Op0VL;
3026  for (Value *V : E->Scalars)
3027  Op0VL.push_back(cast<GetElementPtrInst>(V)->getOperand(0));
3028 
3029  Value *Op0 = vectorizeTree(Op0VL);
3030 
3031  std::vector<Value *> OpVecs;
3032  for (int j = 1, e = cast<GetElementPtrInst>(VL0)->getNumOperands(); j < e;
3033  ++j) {
3034  ValueList OpVL;
3035  for (Value *V : E->Scalars)
3036  OpVL.push_back(cast<GetElementPtrInst>(V)->getOperand(j));
3037 
3038  Value *OpVec = vectorizeTree(OpVL);
3039  OpVecs.push_back(OpVec);
3040  }
3041 
3042  Value *V = Builder.CreateGEP(
3043  cast<GetElementPtrInst>(VL0)->getSourceElementType(), Op0, OpVecs);
3044  E->VectorizedValue = V;
3045  ++NumVectorInstructions;
3046 
3047  if (Instruction *I = dyn_cast<Instruction>(V))
3048  return propagateMetadata(I, E->Scalars);
3049 
3050  return V;
3051  }
3052  case Instruction::Call: {
3053  CallInst *CI = cast<CallInst>(VL0);
3054  setInsertPointAfterBundle(E->Scalars, VL0);
3055  Function *FI;
3057  Value *ScalarArg = nullptr;
3058  if (CI && (FI = CI->getCalledFunction())) {
3059  IID = FI->getIntrinsicID();
3060  }
3061  std::vector<Value *> OpVecs;
3062  for (int j = 0, e = CI->getNumArgOperands(); j < e; ++j) {
3063  ValueList OpVL;
3064  // ctlz,cttz and powi are special intrinsics whose second argument is
3065  // a scalar. This argument should not be vectorized.
3066  if (hasVectorInstrinsicScalarOpd(IID, 1) && j == 1) {
3067  CallInst *CEI = cast<CallInst>(VL0);
3068  ScalarArg = CEI->getArgOperand(j);
3069  OpVecs.push_back(CEI->getArgOperand(j));
3070  continue;
3071  }
3072  for (Value *V : E->Scalars) {
3073  CallInst *CEI = cast<CallInst>(V);
3074  OpVL.push_back(CEI->getArgOperand(j));
3075  }
3076 
3077  Value *OpVec = vectorizeTree(OpVL);
3078  DEBUG(dbgs() << "SLP: OpVec[" << j << "]: " << *OpVec << "\n");
3079  OpVecs.push_back(OpVec);
3080  }
3081 
3082  Module *M = F->getParent();
3084  Type *Tys[] = { VectorType::get(CI->getType(), E->Scalars.size()) };
3085  Function *CF = Intrinsic::getDeclaration(M, ID, Tys);
3087  CI->getOperandBundlesAsDefs(OpBundles);
3088  Value *V = Builder.CreateCall(CF, OpVecs, OpBundles);
3089 
3090  // The scalar argument uses an in-tree scalar so we add the new vectorized
3091  // call to ExternalUses list to make sure that an extract will be
3092  // generated in the future.
3093  if (ScalarArg && getTreeEntry(ScalarArg))
3094  ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
3095 
3096  E->VectorizedValue = V;
3097  propagateIRFlags(E->VectorizedValue, E->Scalars, VL0);
3098  ++NumVectorInstructions;
3099  return V;
3100  }
3101  case Instruction::ShuffleVector: {
3102  ValueList LHSVL, RHSVL;
3103  assert(Instruction::isBinaryOp(S.Opcode) &&
3104  "Invalid Shuffle Vector Operand");
3105  reorderAltShuffleOperands(S.Opcode, E->Scalars, LHSVL, RHSVL);
3106  setInsertPointAfterBundle(E->Scalars, VL0);
3107 
3108  Value *LHS = vectorizeTree(LHSVL);
3109  Value *RHS = vectorizeTree(RHSVL);
3110 
3111  if (Value *V = alreadyVectorized(E->Scalars, VL0))
3112  return V;
3113 
3114  // Create a vector of LHS op1 RHS
3115  Value *V0 = Builder.CreateBinOp(
3116  static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
3117 
3118  unsigned AltOpcode = getAltOpcode(S.Opcode);
3119  // Create a vector of LHS op2 RHS
3120  Value *V1 = Builder.CreateBinOp(
3121  static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
3122 
3123  // Create shuffle to take alternate operations from the vector.
3124  // Also, gather up odd and even scalar ops to propagate IR flags to
3125  // each vector operation.
3126  ValueList OddScalars, EvenScalars;
3127  unsigned e = E->Scalars.size();
3129  for (unsigned i = 0; i < e; ++i) {
3130  if (isOdd(i)) {
3131  Mask[i] = Builder.getInt32(e + i);
3132  OddScalars.push_back(E->Scalars[i]);
3133  } else {
3134  Mask[i] = Builder.getInt32(i);
3135  EvenScalars.push_back(E->Scalars[i]);
3136  }
3137  }
3138 
3139  Value *ShuffleMask = ConstantVector::get(Mask);
3140  propagateIRFlags(V0, EvenScalars);
3141  propagateIRFlags(V1, OddScalars);
3142 
3143  Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
3144  E->VectorizedValue = V;
3145  ++NumVectorInstructions;
3146  if (Instruction *I = dyn_cast<Instruction>(V))
3147  return propagateMetadata(I, E->Scalars);
3148 
3149  return V;
3150  }
3151  default:
3152  llvm_unreachable("unknown inst");
3153  }
3154  return nullptr;
3155 }
3156 
3157 Value *BoUpSLP::vectorizeTree() {
3158  ExtraValueToDebugLocsMap ExternallyUsedValues;
3159  return vectorizeTree(ExternallyUsedValues);
3160 }
3161 
3162 Value *
3163 BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3164  // All blocks must be scheduled before any instructions are inserted.
3165  for (auto &BSIter : BlocksSchedules) {
3166  scheduleBlock(BSIter.second.get());
3167  }
3168 
3169  Builder.SetInsertPoint(&F->getEntryBlock().front());
3170  auto *VectorRoot = vectorizeTree(&VectorizableTree[0]);
3171 
3172  // If the vectorized tree can be rewritten in a smaller type, we truncate the
3173  // vectorized root. InstCombine will then rewrite the entire expression. We
3174  // sign extend the extracted values below.
3175  auto *ScalarRoot = VectorizableTree[0].Scalars[0];
3176  if (MinBWs.count(ScalarRoot)) {
3177  if (auto *I = dyn_cast<Instruction>(VectorRoot))
3178  Builder.SetInsertPoint(&*++BasicBlock::iterator(I));
3179  auto BundleWidth = VectorizableTree[0].Scalars.size();
3180  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
3181  auto *VecTy = VectorType::get(MinTy, BundleWidth);
3182  auto *Trunc = Builder.CreateTrunc(VectorRoot, VecTy);
3183  VectorizableTree[0].VectorizedValue = Trunc;
3184  }
3185 
3186  DEBUG(dbgs() << "SLP: Extracting " << ExternalUses.size() << " values .\n");
3187 
3188  // If necessary, sign-extend or zero-extend ScalarRoot to the larger type
3189  // specified by ScalarType.
3190  auto extend = [&](Value *ScalarRoot, Value *Ex, Type *ScalarType) {
3191  if (!MinBWs.count(ScalarRoot))
3192  return Ex;
3193  if (MinBWs[ScalarRoot].second)
3194  return Builder.CreateSExt(Ex, ScalarType);
3195  return Builder.CreateZExt(Ex, ScalarType);
3196  };
3197 
3198  // Extract all of the elements with the external uses.
3199  for (const auto &ExternalUse : ExternalUses) {
3200  Value *Scalar = ExternalUse.Scalar;
3201  llvm::User *User = ExternalUse.User;
3202 
3203  // Skip users that we already RAUW. This happens when one instruction
3204  // has multiple uses of the same value.
3205  if (User && !is_contained(Scalar->users(), User))
3206  continue;
3207  TreeEntry *E = getTreeEntry(Scalar);
3208  assert(E && "Invalid scalar");
3209  assert(!E->NeedToGather && "Extracting from a gather list");
3210 
3211  Value *Vec = E->VectorizedValue;
3212  assert(Vec && "Can't find vectorizable value");
3213 
3214  Value *Lane = Builder.getInt32(ExternalUse.Lane);
3215  // If User == nullptr, the Scalar is used as extra arg. Generate
3216  // ExtractElement instruction and update the record for this scalar in
3217  // ExternallyUsedValues.
3218  if (!User) {
3219  assert(ExternallyUsedValues.count(Scalar) &&
3220  "Scalar with nullptr as an external user must be registered in "
3221  "ExternallyUsedValues map");
3222  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3223  Builder.SetInsertPoint(VecI->getParent(),
3224  std::next(VecI->getIterator()));
3225  } else {
3226  Builder.SetInsertPoint(&F->getEntryBlock().front());
3227  }
3228  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3229  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3230  CSEBlocks.insert(cast<Instruction>(Scalar)->getParent());
3231  auto &Locs = ExternallyUsedValues[Scalar];
3232  ExternallyUsedValues.insert({Ex, Locs});
3233  ExternallyUsedValues.erase(Scalar);
3234  continue;
3235  }
3236 
3237  // Generate extracts for out-of-tree users.
3238  // Find the insertion point for the extractelement lane.
3239  if (auto *VecI = dyn_cast<Instruction>(Vec)) {
3240  if (PHINode *PH = dyn_cast<PHINode>(User)) {
3241  for (int i = 0, e = PH->getNumIncomingValues(); i != e; ++i) {
3242  if (PH->getIncomingValue(i) == Scalar) {
3243  TerminatorInst *IncomingTerminator =
3244  PH->getIncomingBlock(i)->getTerminator();
3245  if (isa<CatchSwitchInst>(IncomingTerminator)) {
3246  Builder.SetInsertPoint(VecI->getParent(),
3247  std::next(VecI->getIterator()));
3248  } else {
3249  Builder.SetInsertPoint(PH->getIncomingBlock(i)->getTerminator());
3250  }
3251  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3252  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3253  CSEBlocks.insert(PH->getIncomingBlock(i));
3254  PH->setOperand(i, Ex);
3255  }
3256  }
3257  } else {
3258  Builder.SetInsertPoint(cast<Instruction>(User));
3259  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3260  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3261  CSEBlocks.insert(cast<Instruction>(User)->getParent());
3262  User->replaceUsesOfWith(Scalar, Ex);
3263  }
3264  } else {
3265  Builder.SetInsertPoint(&F->getEntryBlock().front());
3266  Value *Ex = Builder.CreateExtractElement(Vec, Lane);
3267  Ex = extend(ScalarRoot, Ex, Scalar->getType());
3268  CSEBlocks.insert(&F->getEntryBlock());
3269  User->replaceUsesOfWith(Scalar, Ex);
3270  }
3271 
3272  DEBUG(dbgs() << "SLP: Replaced:" << *User << ".\n");
3273  }
3274 
3275  // For each vectorized value:
3276  for (TreeEntry &EIdx : VectorizableTree) {
3277  TreeEntry *Entry = &EIdx;
3278 
3279  // No need to handle users of gathered values.
3280  if (Entry->NeedToGather)
3281  continue;
3282 
3283  assert(Entry->VectorizedValue && "Can't find vectorizable value");
3284 
3285  // For each lane:
3286  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3287  Value *Scalar = Entry->Scalars[Lane];
3288 
3289  Type *Ty = Scalar->getType();
3290  if (!Ty->isVoidTy()) {
3291 #ifndef NDEBUG
3292  for (User *U : Scalar->users()) {
3293  DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
3294 
3295  // It is legal to replace users in the ignorelist by undef.
3296  assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
3297  "Replacing out-of-tree value with undef");
3298  }
3299 #endif
3300  Value *Undef = UndefValue::get(Ty);
3301  Scalar->replaceAllUsesWith(Undef);
3302  }
3303  DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
3304  eraseInstruction(cast<Instruction>(Scalar));
3305  }
3306  }
3307 
3308  Builder.ClearInsertionPoint();
3309 
3310  return VectorizableTree[0].VectorizedValue;
3311 }
3312 
3313 void BoUpSLP::optimizeGatherSequence() {
3314  DEBUG(dbgs() << "SLP: Optimizing " << GatherSeq.size()
3315  << " gather sequences instructions.\n");
3316  // LICM InsertElementInst sequences.
3317  for (Instruction *it : GatherSeq) {
3319 
3320  if (!Insert)
3321  continue;
3322 
3323  // Check if this block is inside a loop.
3324  Loop *L = LI->getLoopFor(Insert->getParent());
3325  if (!L)
3326  continue;
3327 
3328  // Check if it has a preheader.
3329  BasicBlock *PreHeader = L->getLoopPreheader();
3330  if (!PreHeader)
3331  continue;
3332 
3333  // If the vector or the element that we insert into it are
3334  // instructions that are defined in this basic block then we can't
3335  // hoist this instruction.
3336  Instruction *CurrVec = dyn_cast<Instruction>(Insert->getOperand(0));
3337  Instruction *NewElem = dyn_cast<Instruction>(Insert->getOperand(1));
3338  if (CurrVec && L->contains(CurrVec))
3339  continue;
3340  if (NewElem && L->contains(NewElem))
3341  continue;
3342 
3343  // We can hoist this instruction. Move it to the pre-header.
3344  Insert->moveBefore(PreHeader->getTerminator());
3345  }
3346 
3347  // Make a list of all reachable blocks in our CSE queue.
3349  CSEWorkList.reserve(CSEBlocks.size());
3350  for (BasicBlock *BB : CSEBlocks)
3351  if (DomTreeNode *N = DT->getNode(BB)) {
3352  assert(DT->isReachableFromEntry(N));
3353  CSEWorkList.push_back(N);
3354  }
3355 
3356  // Sort blocks by domination. This ensures we visit a block after all blocks
3357  // dominating it are visited.
3358  std::stable_sort(CSEWorkList.begin(), CSEWorkList.end(),
3359  [this](const DomTreeNode *A, const DomTreeNode *B) {
3360  return DT->properlyDominates(A, B);
3361  });
3362 
3363  // Perform O(N^2) search over the gather sequences and merge identical
3364  // instructions. TODO: We can further optimize this scan if we split the
3365  // instructions into different buckets based on the insert lane.
3367  for (auto I = CSEWorkList.begin(), E = CSEWorkList.end(); I != E; ++I) {
3368  assert((I == CSEWorkList.begin() || !DT->dominates(*I, *std::prev(I))) &&
3369  "Worklist not sorted properly!");
3370  BasicBlock *BB = (*I)->getBlock();
3371  // For all instructions in blocks containing gather sequences:
3372  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
3373  Instruction *In = &*it++;
3374  if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
3375  continue;
3376 
3377  // Check if we can replace this instruction with any of the
3378  // visited instructions.
3379  for (Instruction *v : Visited) {
3380  if (In->isIdenticalTo(v) &&
3381  DT->dominates(v->getParent(), In->getParent())) {
3382  In->replaceAllUsesWith(v);
3383  eraseInstruction(In);
3384  In = nullptr;
3385  break;
3386  }
3387  }
3388  if (In) {
3389  assert(!is_contained(Visited, In));
3390  Visited.push_back(In);
3391  }
3392  }
3393  }
3394  CSEBlocks.clear();
3395  GatherSeq.clear();
3396 }
3397 
3398 // Groups the instructions to a bundle (which is then a single scheduling entity)
3399 // and schedules instructions until the bundle gets ready.
3400 bool BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL,
3401  BoUpSLP *SLP, Value *OpValue) {
3402  if (isa<PHINode>(OpValue))
3403  return true;
3404 
3405  // Initialize the instruction bundle.
3406  Instruction *OldScheduleEnd = ScheduleEnd;
3407  ScheduleData *PrevInBundle = nullptr;
3408  ScheduleData *Bundle = nullptr;
3409  bool ReSchedule = false;
3410  DEBUG(dbgs() << "SLP: bundle: " << *OpValue << "\n");
3411 
3412  // Make sure that the scheduling region contains all
3413  // instructions of the bundle.
3414  for (Value *V : VL) {
3415  if (!extendSchedulingRegion(V, OpValue))
3416  return false;
3417  }
3418 
3419  for (Value *V : VL) {
3420  ScheduleData *BundleMember = getScheduleData(V);
3421  assert(BundleMember &&
3422  "no ScheduleData for bundle member (maybe not in same basic block)");
3423  if (BundleMember->IsScheduled) {
3424  // A bundle member was scheduled as single instruction before and now
3425  // needs to be scheduled as part of the bundle. We just get rid of the
3426  // existing schedule.
3427  DEBUG(dbgs() << "SLP: reset schedule because " << *BundleMember
3428  << " was already scheduled\n");
3429  ReSchedule = true;
3430  }
3431  assert(BundleMember->isSchedulingEntity() &&
3432  "bundle member already part of other bundle");
3433  if (PrevInBundle) {
3434  PrevInBundle->NextInBundle = BundleMember;
3435  } else {
3436  Bundle = BundleMember;
3437  }
3438  BundleMember->UnscheduledDepsInBundle = 0;
3439  Bundle->UnscheduledDepsInBundle += BundleMember->UnscheduledDeps;
3440 
3441  // Group the instructions to a bundle.
3442  BundleMember->FirstInBundle = Bundle;
3443  PrevInBundle = BundleMember;
3444  }
3445  if (ScheduleEnd != OldScheduleEnd) {
3446  // The scheduling region got new instructions at the lower end (or it is a
3447  // new region for the first bundle). This makes it necessary to
3448  // recalculate all dependencies.
3449  // It is seldom that this needs to be done a second time after adding the
3450  // initial bundle to the region.
3451  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3452  doForAllOpcodes(I, [](ScheduleData *SD) {
3453  SD->clearDependencies();
3454  });
3455  }
3456  ReSchedule = true;
3457  }
3458  if (ReSchedule) {
3459  resetSchedule();
3460  initialFillReadyList(ReadyInsts);
3461  }
3462 
3463  DEBUG(dbgs() << "SLP: try schedule bundle " << *Bundle << " in block "
3464  << BB->getName() << "\n");
3465 
3466  calculateDependencies(Bundle, true, SLP);
3467 
3468  // Now try to schedule the new bundle. As soon as the bundle is "ready" it
3469  // means that there are no cyclic dependencies and we can schedule it.
3470  // Note that's important that we don't "schedule" the bundle yet (see
3471  // cancelScheduling).
3472  while (!Bundle->isReady() && !ReadyInsts.empty()) {
3473 
3474  ScheduleData *pickedSD = ReadyInsts.back();
3475  ReadyInsts.pop_back();
3476 
3477  if (pickedSD->isSchedulingEntity() && pickedSD->isReady()) {
3478  schedule(pickedSD, ReadyInsts);
3479  }
3480  }
3481  if (!Bundle->isReady()) {
3482  cancelScheduling(VL, OpValue);
3483  return false;
3484  }
3485  return true;
3486 }
3487 
3488 void BoUpSLP::BlockScheduling::cancelScheduling(ArrayRef<Value *> VL,
3489  Value *OpValue) {
3490  if (isa<PHINode>(OpValue))
3491  return;
3492 
3493  ScheduleData *Bundle = getScheduleData(OpValue);
3494  DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");
3495  assert(!Bundle->IsScheduled &&
3496  "Can't cancel bundle which is already scheduled");
3497  assert(Bundle->isSchedulingEntity() && Bundle->isPartOfBundle() &&
3498  "tried to unbundle something which is not a bundle");
3499 
3500  // Un-bundle: make single instructions out of the bundle.
3501  ScheduleData *BundleMember = Bundle;
3502  while (BundleMember) {
3503  assert(BundleMember->FirstInBundle == Bundle && "corrupt bundle links");
3504  BundleMember->FirstInBundle = BundleMember;
3505  ScheduleData *Next = BundleMember->NextInBundle;
3506  BundleMember->NextInBundle = nullptr;
3507  BundleMember->UnscheduledDepsInBundle = BundleMember->UnscheduledDeps;
3508  if (BundleMember->UnscheduledDepsInBundle == 0) {
3509  ReadyInsts.insert(BundleMember);
3510  }
3511  BundleMember = Next;
3512  }
3513 }
3514 
3515 BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
3516  // Allocate a new ScheduleData for the instruction.
3517  if (ChunkPos >= ChunkSize) {
3518  ScheduleDataChunks.push_back(llvm::make_unique<ScheduleData[]>(ChunkSize));
3519  ChunkPos = 0;
3520  }
3521  return &(ScheduleDataChunks.back()[ChunkPos++]);
3522 }
3523 
3524 bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
3525  Value *OpValue) {
3526  if (getScheduleData(V, isOneOf(OpValue, V)))
3527  return true;
3528  Instruction *I = dyn_cast<Instruction>(V);
3529  assert(I && "bundle member must be an instruction");
3530  assert(!isa<PHINode>(I) && "phi nodes don't need to be scheduled");
3531  auto &&CheckSheduleForI = [this, OpValue](Instruction *I) -> bool {
3532  ScheduleData *ISD = getScheduleData(I);
3533  if (!ISD)
3534  return false;
3535  assert(isInSchedulingRegion(ISD) &&
3536  "ScheduleData not in scheduling region");
3537  ScheduleData *SD = allocateScheduleDataChunks();
3538  SD->Inst = I;
3539  SD->init(SchedulingRegionID, OpValue);
3540  ExtraScheduleDataMap[I][OpValue] = SD;
3541  return true;
3542  };
3543  if (CheckSheduleForI(I))
3544  return true;
3545  if (!ScheduleStart) {
3546  // It's the first instruction in the new region.
3547  initScheduleData(I, I->getNextNode(), nullptr, nullptr);
3548  ScheduleStart = I;
3549  ScheduleEnd = I->getNextNode();
3550  if (isOneOf(OpValue, I) != I)
3551  CheckSheduleForI(I);
3552  assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3553  DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n");
3554  return true;
3555  }
3556  // Search up and down at the same time, because we don't know if the new
3557  // instruction is above or below the existing scheduling region.
3559  ++ScheduleStart->getIterator().getReverse();
3560  BasicBlock::reverse_iterator UpperEnd = BB->rend();
3561  BasicBlock::iterator DownIter = ScheduleEnd->getIterator();
3562  BasicBlock::iterator LowerEnd = BB->end();
3563  while (true) {
3564  if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
3565  DEBUG(dbgs() << "SLP: exceeded schedule region size limit\n");
3566  return false;
3567  }
3568 
3569  if (UpIter != UpperEnd) {
3570  if (&*UpIter == I) {
3571  initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
3572  ScheduleStart = I;
3573  if (isOneOf(OpValue, I) != I)
3574  CheckSheduleForI(I);
3575  DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n");
3576  return true;
3577  }
3578  UpIter++;
3579  }
3580  if (DownIter != LowerEnd) {
3581  if (&*DownIter == I) {
3582  initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
3583  nullptr);
3584  ScheduleEnd = I->getNextNode();
3585  if (isOneOf(OpValue, I) != I)
3586  CheckSheduleForI(I);
3587  assert(ScheduleEnd && "tried to vectorize a TerminatorInst?");
3588  DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n");
3589  return true;
3590  }
3591  DownIter++;
3592  }
3593  assert((UpIter != UpperEnd || DownIter != LowerEnd) &&
3594  "instruction not found in block");
3595  }
3596  return true;
3597 }
3598 
3599 void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
3600  Instruction *ToI,
3601  ScheduleData *PrevLoadStore,
3602  ScheduleData *NextLoadStore) {
3603  ScheduleData *CurrentLoadStore = PrevLoadStore;
3604  for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
3605  ScheduleData *SD = ScheduleDataMap[I];
3606  if (!SD) {
3607  SD = allocateScheduleDataChunks();
3608  ScheduleDataMap[I] = SD;
3609  SD->Inst = I;
3610  }
3611  assert(!isInSchedulingRegion(SD) &&
3612  "new ScheduleData already in scheduling region");
3613  SD->init(SchedulingRegionID, I);
3614 
3615  if (I->mayReadOrWriteMemory()) {
3616  // Update the linked list of memory accessing instructions.
3617  if (CurrentLoadStore) {
3618  CurrentLoadStore->NextLoadStore = SD;
3619  } else {
3620  FirstLoadStoreInRegion = SD;
3621  }
3622  CurrentLoadStore = SD;
3623  }
3624  }
3625  if (NextLoadStore) {
3626  if (CurrentLoadStore)
3627  CurrentLoadStore->NextLoadStore = NextLoadStore;
3628  } else {
3629  LastLoadStoreInRegion = CurrentLoadStore;
3630  }
3631 }
3632 
3633 void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
3634  bool InsertInReadyList,
3635  BoUpSLP *SLP) {
3636  assert(SD->isSchedulingEntity());
3637 
3639  WorkList.push_back(SD);
3640 
3641  while (!WorkList.empty()) {
3642  ScheduleData *SD = WorkList.back();
3643  WorkList.pop_back();
3644 
3645  ScheduleData *BundleMember = SD;
3646  while (BundleMember) {
3647  assert(isInSchedulingRegion(BundleMember));
3648  if (!BundleMember->hasValidDependencies()) {
3649 
3650  DEBUG(dbgs() << "SLP: update deps of " << *BundleMember << "\n");
3651  BundleMember->Dependencies = 0;
3652  BundleMember->resetUnscheduledDeps();
3653 
3654  // Handle def-use chain dependencies.
3655  if (BundleMember->OpValue != BundleMember->Inst) {
3656  ScheduleData *UseSD = getScheduleData(BundleMember->Inst);
3657  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3658  BundleMember->Dependencies++;
3659  ScheduleData *DestBundle = UseSD->FirstInBundle;
3660  if (!DestBundle->IsScheduled)
3661  BundleMember->incrementUnscheduledDeps(1);
3662  if (!DestBundle->hasValidDependencies())
3663  WorkList.push_back(DestBundle);
3664  }
3665  } else {
3666  for (User *U : BundleMember->Inst->users()) {
3667  if (isa<Instruction>(U)) {
3668  ScheduleData *UseSD = getScheduleData(U);
3669  if (UseSD && isInSchedulingRegion(UseSD->FirstInBundle)) {
3670  BundleMember->Dependencies++;
3671  ScheduleData *DestBundle = UseSD->FirstInBundle;
3672  if (!DestBundle->IsScheduled)
3673  BundleMember->incrementUnscheduledDeps(1);
3674  if (!DestBundle->hasValidDependencies())
3675  WorkList.push_back(DestBundle);
3676  }
3677  } else {
3678  // I'm not sure if this can ever happen. But we need to be safe.
3679  // This lets the instruction/bundle never be scheduled and
3680  // eventually disable vectorization.
3681  BundleMember->Dependencies++;
3682  BundleMember->incrementUnscheduledDeps(1);
3683  }
3684  }
3685  }
3686 
3687  // Handle the memory dependencies.
3688  ScheduleData *DepDest = BundleMember->NextLoadStore;
3689  if (DepDest) {
3690  Instruction *SrcInst = BundleMember->Inst;
3691  MemoryLocation SrcLoc = getLocation(SrcInst, SLP->AA);
3692  bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
3693  unsigned numAliased = 0;
3694  unsigned DistToSrc = 1;
3695 
3696  while (DepDest) {
3697  assert(isInSchedulingRegion(DepDest));
3698 
3699  // We have two limits to reduce the complexity:
3700  // 1) AliasedCheckLimit: It's a small limit to reduce calls to
3701  // SLP->isAliased (which is the expensive part in this loop).
3702  // 2) MaxMemDepDistance: It's for very large blocks and it aborts
3703  // the whole loop (even if the loop is fast, it's quadratic).
3704  // It's important for the loop break condition (see below) to
3705  // check this limit even between two read-only instructions.
3706  if (DistToSrc >= MaxMemDepDistance ||
3707  ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
3708  (numAliased >= AliasedCheckLimit ||
3709  SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
3710 
3711  // We increment the counter only if the locations are aliased
3712  // (instead of counting all alias checks). This gives a better
3713  // balance between reduced runtime and accurate dependencies.
3714  numAliased++;
3715 
3716  DepDest->MemoryDependencies.push_back(BundleMember);
3717  BundleMember->Dependencies++;
3718  ScheduleData *DestBundle = DepDest->FirstInBundle;
3719  if (!DestBundle->IsScheduled) {
3720  BundleMember->incrementUnscheduledDeps(1);
3721  }
3722  if (!DestBundle->hasValidDependencies()) {
3723  WorkList.push_back(DestBundle);
3724  }
3725  }
3726  DepDest = DepDest->NextLoadStore;
3727 
3728  // Example, explaining the loop break condition: Let's assume our
3729  // starting instruction is i0 and MaxMemDepDistance = 3.
3730  //
3731  // +--------v--v--v
3732  // i0,i1,i2,i3,i4,i5,i6,i7,i8
3733  // +--------^--^--^
3734  //
3735  // MaxMemDepDistance let us stop alias-checking at i3 and we add
3736  // dependencies from i0 to i3,i4,.. (even if they are not aliased).
3737  // Previously we already added dependencies from i3 to i6,i7,i8
3738  // (because of MaxMemDepDistance). As we added a dependency from
3739  // i0 to i3, we have transitive dependencies from i0 to i6,i7,i8
3740  // and we can abort this loop at i6.
3741  if (DistToSrc >= 2 * MaxMemDepDistance)
3742  break;
3743  DistToSrc++;
3744  }
3745  }
3746  }
3747  BundleMember = BundleMember->NextInBundle;
3748  }
3749  if (InsertInReadyList && SD->isReady()) {
3750  ReadyInsts.push_back(SD);
3751  DEBUG(dbgs() << "SLP: gets ready on update: " << *SD->Inst << "\n");
3752  }
3753  }
3754 }
3755 
3756 void BoUpSLP::BlockScheduling::resetSchedule() {
3757  assert(ScheduleStart &&
3758  "tried to reset schedule on block which has not been scheduled");
3759  for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
3760  doForAllOpcodes(I, [&](ScheduleData *SD) {
3761  assert(isInSchedulingRegion(SD) &&
3762  "ScheduleData not in scheduling region");
3763  SD->IsScheduled = false;
3764  SD->resetUnscheduledDeps();
3765  });
3766  }
3767  ReadyInsts.clear();
3768 }
3769 
3770 void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
3771  if (!BS->ScheduleStart)
3772  return;
3773 
3774  DEBUG(dbgs() << "SLP: schedule block " << BS->BB->getName() << "\n");
3775 
3776  BS->resetSchedule();
3777 
3778  // For the real scheduling we use a more sophisticated ready-list: it is
3779  // sorted by the original instruction location. This lets the final schedule
3780  // be as close as possible to the original instruction order.
3781  struct ScheduleDataCompare {
3782  bool operator()(ScheduleData *SD1, ScheduleData *SD2) const {
3783  return SD2->SchedulingPriority < SD1->SchedulingPriority;
3784  }
3785  };
3786  std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
3787 
3788  // Ensure that all dependency data is updated and fill the ready-list with
3789  // initial instructions.
3790  int Idx = 0;
3791  int NumToSchedule = 0;
3792  for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
3793  I = I->getNextNode()) {
3794  BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {
3795  assert(SD->isPartOfBundle() ==
3796  (getTreeEntry(SD->Inst) != nullptr) &&
3797  "scheduler and vectorizer bundle mismatch");
3798  SD->FirstInBundle->SchedulingPriority = Idx++;
3799  if (SD->isSchedulingEntity()) {
3800  BS->calculateDependencies(SD, false, this);
3801  NumToSchedule++;
3802  }
3803  });
3804  }
3805  BS->initialFillReadyList(ReadyInsts);
3806 
3807  Instruction *LastScheduledInst = BS->ScheduleEnd;
3808 
3809  // Do the "real" scheduling.
3810  while (!ReadyInsts.empty()) {
3811  ScheduleData *picked = *ReadyInsts.begin();
3812  ReadyInsts.erase(ReadyInsts.begin());
3813 
3814  // Move the scheduled instruction(s) to their dedicated places, if not
3815  // there yet.
3816  ScheduleData *BundleMember = picked;
3817  while (BundleMember) {
3818  Instruction *pickedInst = BundleMember->Inst;
3819  if (LastScheduledInst->getNextNode() != pickedInst) {
3820  BS->BB->getInstList().remove(pickedInst);
3821  BS->BB->getInstList().insert(LastScheduledInst->getIterator(),
3822  pickedInst);
3823  }
3824  LastScheduledInst = pickedInst;
3825  BundleMember = BundleMember->NextInBundle;
3826  }
3827 
3828  BS->schedule(picked, ReadyInsts);
3829  NumToSchedule--;
3830  }
3831  assert(NumToSchedule == 0 && "could not schedule all instructions");
3832 
3833  // Avoid duplicate scheduling of the block.
3834  BS->ScheduleStart = nullptr;
3835 }
3836 
3837 unsigned BoUpSLP::getVectorElementSize(Value *V) {
3838  // If V is a store, just return the width of the stored value without
3839  // traversing the expression tree. This is the common case.
3840  if (auto *Store = dyn_cast<StoreInst>(V))
3841  return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
3842 
3843  // If V is not a store, we can traverse the expression tree to find loads
3844  // that feed it. The type of the loaded value may indicate a more suitable
3845  // width than V's type. We want to base the vector element size on the width
3846  // of memory operations where possible.
3849  if (auto *I = dyn_cast<Instruction>(V))
3850  Worklist.push_back(I);
3851 
3852  // Traverse the expression tree in bottom-up order looking for loads. If we
3853  // encounter an instruciton we don't yet handle, we give up.
3854  auto MaxWidth = 0u;
3855  auto FoundUnknownInst = false;
3856  while (!Worklist.empty() && !FoundUnknownInst) {
3857  auto *I = Worklist.pop_back_val();
3858  Visited.insert(I);
3859 
3860  // We should only be looking at scalar instructions here. If the current
3861  // instruction has a vector type, give up.
3862  auto *Ty = I->getType();
3863  if (isa<VectorType>(Ty))
3864  FoundUnknownInst = true;
3865 
3866  // If the current instruction is a load, update MaxWidth to reflect the
3867  // width of the loaded value.
3868  else if (isa<LoadInst>(I))
3869  MaxWidth = std::max<unsigned>(MaxWidth, DL->getTypeSizeInBits(Ty));
3870 
3871  // Otherwise, we need to visit the operands of the instruction. We only
3872  // handle the interesting cases from buildTree here. If an operand is an
3873  // instruction we haven't yet visited, we add it to the worklist.
3874  else if (isa<PHINode>(I) || isa<CastInst>(I) || isa<GetElementPtrInst>(I) ||
3875  isa<CmpInst>(I) || isa<SelectInst>(I) || isa<BinaryOperator>(I)) {
3876  for (Use &U : I->operands())
3877  if (auto *J = dyn_cast<Instruction>(U.get()))
3878  if (!Visited.count(J))
3879  Worklist.push_back(J);
3880  }
3881 
3882  // If we don't yet handle the instruction, give up.
3883  else
3884  FoundUnknownInst = true;
3885  }
3886 
3887  // If we didn't encounter a memory access in the expression tree, or if we
3888  // gave up for some reason, just return the width of V.
3889  if (!MaxWidth || FoundUnknownInst)
3890  return DL->getTypeSizeInBits(V->getType());
3891 
3892  // Otherwise, return the maximum width we found.
3893  return MaxWidth;
3894 }
3895 
3896 // Determine if a value V in a vectorizable expression Expr can be demoted to a
3897 // smaller type with a truncation. We collect the values that will be demoted
3898 // in ToDemote and additional roots that require investigating in Roots.
3900  SmallVectorImpl<Value *> &ToDemote,
3901  SmallVectorImpl<Value *> &Roots) {
3902  // We can always demote constants.
3903  if (isa<Constant>(V)) {
3904  ToDemote.push_back(V);
3905  return true;
3906  }
3907 
3908  // If the value is not an instruction in the expression with only one use, it
3909  // cannot be demoted.
3910  auto *I = dyn_cast<Instruction>(V);
3911  if (!I || !I->hasOneUse() || !Expr.count(I))
3912  return false;
3913 
3914  switch (I->getOpcode()) {
3915 
3916  // We can always demote truncations and extensions. Since truncations can
3917  // seed additional demotion, we save the truncated value.
3918  case Instruction::Trunc:
3919  Roots.push_back(I->getOperand(0));
3920  case Instruction::ZExt:
3921  case Instruction::SExt:
3922  break;
3923 
3924  // We can demote certain binary operations if we can demote both of their
3925  // operands.
3926  case Instruction::Add:
3927  case Instruction::Sub:
3928  case Instruction::Mul:
3929  case Instruction::And:
3930  case Instruction::Or:
3931  case Instruction::Xor:
3932  if (!collectValuesToDemote(I->getOperand(0), Expr, ToDemote, Roots) ||
3933  !collectValuesToDemote(I->getOperand(1), Expr, ToDemote, Roots))
3934  return false;
3935  break;
3936 
3937  // We can demote selects if we can demote their true and false values.
3938  case Instruction::Select: {
3939  SelectInst *SI = cast<SelectInst>(I);
3940  if (!collectValuesToDemote(SI->getTrueValue(), Expr, ToDemote, Roots) ||
3941  !collectValuesToDemote(SI->getFalseValue(), Expr, ToDemote, Roots))
3942  return false;
3943  break;
3944  }
3945 
3946  // We can demote phis if we can demote all their incoming operands. Note that
3947  // we don't need to worry about cycles since we ensure single use above.
3948  case Instruction::PHI: {
3949  PHINode *PN = cast<PHINode>(I);
3950  for (Value *IncValue : PN->incoming_values())
3951  if (!collectValuesToDemote(IncValue, Expr, ToDemote, Roots))
3952  return false;
3953  break;
3954  }
3955 
3956  // Otherwise, conservatively give up.
3957  default:
3958  return false;
3959  }
3960 
3961  // Record the value that we can demote.
3962  ToDemote.push_back(V);
3963  return true;
3964 }
3965 
3967  // If there are no external uses, the expression tree must be rooted by a
3968  // store. We can't demote in-memory values, so there is nothing to do here.
3969  if (ExternalUses.empty())
3970  return;
3971 
3972  // We only attempt to truncate integer expressions.
3973  auto &TreeRoot = VectorizableTree[0].Scalars;
3974  auto *TreeRootIT = dyn_cast<IntegerType>(TreeRoot[0]->getType());
3975  if (!TreeRootIT)
3976  return;
3977 
3978  // If the expression is not rooted by a store, these roots should have
3979  // external uses. We will rely on InstCombine to rewrite the expression in
3980  // the narrower type. However, InstCombine only rewrites single-use values.
3981  // This means that if a tree entry other than a root is used externally, it
3982  // must have multiple uses and InstCombine will not rewrite it. The code
3983  // below ensures that only the roots are used externally.
3984  SmallPtrSet<Value *, 32> Expr(TreeRoot.begin(), TreeRoot.end());
3985  for (auto &EU : ExternalUses)
3986  if (!Expr.erase(EU.Scalar))
3987  return;
3988  if (!Expr.empty())
3989  return;
3990 
3991  // Collect the scalar values of the vectorizable expression. We will use this
3992  // context to determine which values can be demoted. If we see a truncation,
3993  // we mark it as seeding another demotion.
3994  for (auto &Entry : VectorizableTree)
3995  Expr.insert(Entry.Scalars.begin(), Entry.Scalars.end());
3996 
3997  // Ensure the roots of the vectorizable tree don't form a cycle. They must
3998  // have a single external user that is not in the vectorizable tree.
3999  for (auto *Root : TreeRoot)
4000  if (!Root->hasOneUse() || Expr.count(*Root->user_begin()))
4001  return;
4002 
4003  // Conservatively determine if we can actually truncate the roots of the
4004  // expression. Collect the values that can be demoted in ToDemote and
4005  // additional roots that require investigating in Roots.
4006  SmallVector<Value *, 32> ToDemote;
4008  for (auto *Root : TreeRoot)
4009  if (!collectValuesToDemote(Root, Expr, ToDemote, Roots))
4010  return;
4011 
4012  // The maximum bit width required to represent all the values that can be
4013  // demoted without loss of precision. It would be safe to truncate the roots
4014  // of the expression to this width.
4015  auto MaxBitWidth = 8u;
4016 
4017  // We first check if all the bits of the roots are demanded. If they're not,
4018  // we can truncate the roots to this narrower type.
4019  for (auto *Root : TreeRoot) {
4020  auto Mask = DB->getDemandedBits(cast<Instruction>(Root));
4021  MaxBitWidth = std::max<unsigned>(
4022  Mask.getBitWidth() - Mask.countLeadingZeros(), MaxBitWidth);
4023  }
4024 
4025  // True if the roots can be zero-extended back to their original type, rather
4026  // than sign-extended. We know that if the leading bits are not demanded, we
4027  // can safely zero-extend. So we initialize IsKnownPositive to True.
4028  bool IsKnownPositive = true;
4029 
4030  // If all the bits of the roots are demanded, we can try a little harder to
4031  // compute a narrower type. This can happen, for example, if the roots are
4032  // getelementptr indices. InstCombine promotes these indices to the pointer
4033  // width. Thus, all their bits are technically demanded even though the
4034  // address computation might be vectorized in a smaller type.
4035  //
4036  // We start by looking at each entry that can be demoted. We compute the
4037  // maximum bit width required to store the scalar by using ValueTracking to
4038  // compute the number of high-order bits we can truncate.
4039  if (MaxBitWidth == DL->getTypeSizeInBits(TreeRoot[0]->getType())) {
4040  MaxBitWidth = 8u;
4041 
4042  // Determine if the sign bit of all the roots is known to be zero. If not,
4043  // IsKnownPositive is set to False.
4044  IsKnownPositive = llvm::all_of(TreeRoot, [&](Value *R) {
4045  KnownBits Known = computeKnownBits(R, *DL);
4046  return Known.isNonNegative();
4047  });
4048 
4049  // Determine the maximum number of bits required to store the scalar
4050  // values.
4051  for (auto *Scalar : ToDemote) {
4052  auto NumSignBits = ComputeNumSignBits(Scalar, *DL, 0, AC, nullptr, DT);
4053  auto NumTypeBits = DL->getTypeSizeInBits(Scalar->getType());
4054  MaxBitWidth = std::max<unsigned>(NumTypeBits - NumSignBits, MaxBitWidth);
4055  }
4056 
4057  // If we can't prove that the sign bit is zero, we must add one to the
4058  // maximum bit width to account for the unknown sign bit. This preserves
4059  // the existing sign bit so we can safely sign-extend the root back to the
4060  // original type. Otherwise, if we know the sign bit is zero, we will
4061  // zero-extend the root instead.
4062  //
4063  // FIXME: This is somewhat suboptimal, as there will be cases where adding
4064  // one to the maximum bit width will yield a larger-than-necessary
4065  // type. In general, we need to add an extra bit only if we can't
4066  // prove that the upper bit of the original type is equal to the
4067  // upper bit of the proposed smaller type. If these two bits are the
4068  // same (either zero or one) we know that sign-extending from the
4069  // smaller type will result in the same value. Here, since we can't
4070  // yet prove this, we are just making the proposed smaller type
4071  // larger to ensure correctness.
4072  if (!IsKnownPositive)
4073  ++MaxBitWidth;
4074  }
4075 
4076  // Round MaxBitWidth up to the next power-of-two.
4077  if (!isPowerOf2_64(MaxBitWidth))
4078  MaxBitWidth = NextPowerOf2(MaxBitWidth);
4079 
4080  // If the maximum bit width we compute is less than the with of the roots'
4081  // type, we can proceed with the narrowing. Otherwise, do nothing.
4082  if (MaxBitWidth >= TreeRootIT->getBitWidth())
4083  return;
4084 
4085  // If we can truncate the root, we must collect additional values that might
4086  // be demoted as a result. That is, those seeded by truncations we will
4087  // modify.
4088  while (!Roots.empty())
4089  collectValuesToDemote(Roots.pop_back_val(), Expr, ToDemote, Roots);
4090 
4091  // Finally, map the values we can demote to the maximum bit with we computed.
4092  for (auto *Scalar : ToDemote)
4093  MinBWs[Scalar] = std::make_pair(MaxBitWidth, !IsKnownPositive);
4094 }
4095 
4096 namespace {
4097 
4098 /// The SLPVectorizer Pass.
4099 struct SLPVectorizer : public FunctionPass {
4100  SLPVectorizerPass Impl;
4101 
4102  /// Pass identification, replacement for typeid
4103  static char ID;
4104 
4105  explicit SLPVectorizer() : FunctionPass(ID) {
4107  }
4108 
4109  bool doInitialization(Module &M) override {
4110  return false;
4111  }
4112 
4113  bool runOnFunction(Function &F) override {
4114  if (skipFunction(F))
4115  return false;
4116 
4117  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
4118  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
4119  auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
4120  auto *TLI = TLIP ? &TLIP->getTLI() : nullptr;
4121  auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
4122  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
4123  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
4124  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
4125  auto *DB = &getAnalysis<DemandedBitsWrapperPass>().getDemandedBits();
4126  auto *ORE = &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
4127 
4128  return Impl.runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4129  }
4130 
4131  void getAnalysisUsage(AnalysisUsage &AU) const override {
4145  AU.setPreservesCFG();
4146  }
4147 };
4148 
4149 } // end anonymous namespace
4150 
4152  auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
4153  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
4154  auto *TLI = AM.getCachedResult<TargetLibraryAnalysis>(F);
4155  auto *AA = &AM.getResult<AAManager>(F);
4156  auto *LI = &AM.getResult<LoopAnalysis>(F);
4157  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
4158  auto *AC = &AM.getResult<AssumptionAnalysis>(F);
4159  auto *DB = &AM.getResult<DemandedBitsAnalysis>(F);
4160  auto *ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
4161 
4162  bool Changed = runImpl(F, SE, TTI, TLI, AA, LI, DT, AC, DB, ORE);
4163  if (!Changed)
4164  return PreservedAnalyses::all();
4165 
4166  PreservedAnalyses PA;
4167  PA.preserveSet<CFGAnalyses>();
4168  PA.preserve<AAManager>();
4169  PA.preserve<GlobalsAA>();
4170  return PA;
4171 }
4172 
4174  TargetTransformInfo *TTI_,
4175  TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
4176  LoopInfo *LI_, DominatorTree *DT_,
4177  AssumptionCache *AC_, DemandedBits *DB_,
4178  OptimizationRemarkEmitter *ORE_) {
4179  SE = SE_;
4180  TTI = TTI_;
4181  TLI = TLI_;
4182  AA = AA_;
4183  LI = LI_;
4184  DT = DT_;
4185  AC = AC_;
4186  DB = DB_;
4187  DL = &F.getParent()->getDataLayout();
4188 
4189  Stores.clear();
4190  GEPs.clear();
4191  bool Changed = false;
4192 
4193  // If the target claims to have no vector registers don't attempt
4194  // vectorization.
4195  if (!TTI->getNumberOfRegisters(true))
4196  return false;
4197 
4198  // Don't vectorize when the attribute NoImplicitFloat is used.
4199  if (F.hasFnAttribute(Attribute::NoImplicitFloat))
4200  return false;
4201 
4202  DEBUG(dbgs() << "SLP: Analyzing blocks in " << F.getName() << ".\n");
4203 
4204  // Use the bottom up slp vectorizer to construct chains that start with
4205  // store instructions.
4206  BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC, DB, DL, ORE_);
4207 
4208  // A general note: the vectorizer must use BoUpSLP::eraseInstruction() to
4209  // delete instructions.
4210 
4211  // Scan the blocks in the function in post order.
4212  for (auto BB : post_order(&F.getEntryBlock())) {
4213  collectSeedInstructions(BB);
4214 
4215  // Vectorize trees that end at stores.
4216  if (!Stores.empty()) {
4217  DEBUG(dbgs() << "SLP: Found stores for " << Stores.size()
4218  << " underlying objects.\n");
4219  Changed |= vectorizeStoreChains(R);
4220  }
4221 
4222  // Vectorize trees that end at reductions.
4223  Changed |= vectorizeChainsInBlock(BB, R);
4224 
4225  // Vectorize the index computations of getelementptr instructions. This
4226  // is primarily intended to catch gather-like idioms ending at
4227  // non-consecutive loads.
4228  if (!GEPs.empty()) {
4229  DEBUG(dbgs() << "SLP: Found GEPs for " << GEPs.size()
4230  << " underlying objects.\n");
4231  Changed |= vectorizeGEPIndices(BB, R);
4232  }
4233  }
4234 
4235  if (Changed) {
4236  R.optimizeGatherSequence();
4237  DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
4238  DEBUG(verifyFunction(F));
4239  }
4240  return Changed;
4241 }
4242 
4243 /// \brief Check that the Values in the slice in VL array are still existent in
4244 /// the WeakTrackingVH array.
4245 /// Vectorization of part of the VL array may cause later values in the VL array
4246 /// to become invalid. We track when this has happened in the WeakTrackingVH
4247 /// array.
4249  ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
4250  unsigned SliceSize) {
4251  VL = VL.slice(SliceBegin, SliceSize);
4252  VH = VH.slice(SliceBegin, SliceSize);
4253  return !std::equal(VL.begin(), VL.end(), VH.begin());
4254 }
4255 
4256 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
4257  unsigned VecRegSize) {
4258  unsigned ChainLen = Chain.size();
4259  DEBUG(dbgs() << "SLP: Analyzing a store chain of length " << ChainLen
4260  << "\n");
4261  unsigned Sz = R.getVectorElementSize(Chain[0]);
4262  unsigned VF = VecRegSize / Sz;
4263 
4264  if (!isPowerOf2_32(Sz) || VF < 2)
4265  return false;
4266 
4267  // Keep track of values that were deleted by vectorizing in the loop below.
4268  SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
4269 
4270  bool Changed = false;
4271  // Look for profitable vectorizable trees at all offsets, starting at zero.
4272  for (unsigned i = 0, e = ChainLen; i < e; ++i) {
4273  if (i + VF > e)
4274  break;
4275 
4276  // Check that a previous iteration of this loop did not delete the Value.
4277  if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
4278  continue;
4279 
4280  DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
4281  << "\n");
4282  ArrayRef<Value *> Operands = Chain.slice(i, VF);
4283 
4284  R.buildTree(Operands);
4285  if (R.isTreeTinyAndNotFullyVectorizable())
4286  continue;
4287 
4288  R.computeMinimumValueSizes();
4289 
4290  int Cost = R.getTreeCost();
4291 
4292  DEBUG(dbgs() << "SLP: Found cost=" << Cost << " for VF=" << VF << "\n");
4293  if (Cost < -SLPCostThreshold) {
4294  DEBUG(dbgs() << "SLP: Decided to vectorize cost=" << Cost << "\n");
4295 
4296  using namespace ore;
4297 
4298  R.getORE()->emit(OptimizationRemark(SV_NAME, "StoresVectorized",
4299  cast<StoreInst>(Chain[i]))
4300  << "Stores SLP vectorized with cost " << NV("Cost", Cost)
4301  << " and with tree size "
4302  << NV("TreeSize", R.getTreeSize()));
4303 
4304  R.vectorizeTree();
4305 
4306  // Move to the next bundle.
4307  i += VF - 1;
4308  Changed = true;
4309  }
4310  }
4311 
4312  return Changed;
4313 }
4314 
4315 bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
4316  BoUpSLP &R) {
4317  SetVector<StoreInst *> Heads, Tails;
4318  SmallDenseMap<StoreInst *, StoreInst *> ConsecutiveChain;
4319 
4320  // We may run into multiple chains that merge into a single chain. We mark the
4321  // stores that we vectorized so that we don't visit the same store twice.
4322  BoUpSLP::ValueSet VectorizedStores;
4323  bool Changed = false;
4324 
4325  // Do a quadratic search on all of the given stores and find
4326  // all of the pairs of stores that follow each other.
4327  SmallVector<unsigned, 16> IndexQueue;
4328  for (unsigned i = 0, e = Stores.size(); i < e; ++i) {
4329  IndexQueue.clear();
4330  // If a store has multiple consecutive store candidates, search Stores
4331  // array according to the sequence: from i+1 to e, then from i-1 to 0.
4332  // This is because usually pairing with immediate succeeding or preceding
4333  // candidate create the best chance to find slp vectorization opportunity.
4334  unsigned j = 0;
4335  for (j = i + 1; j < e; ++j)
4336  IndexQueue.push_back(j);
4337  for (j = i; j > 0; --j)
4338  IndexQueue.push_back(j - 1);
4339 
4340  for (auto &k : IndexQueue) {
4341  if (isConsecutiveAccess(Stores[i], Stores[k], *DL, *SE)) {
4342  Tails.insert(Stores[k]);
4343  Heads.insert(Stores[i]);
4344  ConsecutiveChain[Stores[i]] = Stores[k];
4345  break;
4346  }
4347  }
4348  }
4349 
4350  // For stores that start but don't end a link in the chain:
4351  for (SetVector<StoreInst *>::iterator it = Heads.begin(), e = Heads.end();
4352  it != e; ++it) {
4353  if (Tails.count(*it))
4354  continue;
4355 
4356  // We found a store instr that starts a chain. Now follow the chain and try
4357  // to vectorize it.
4358  BoUpSLP::ValueList Operands;
4359  StoreInst *I = *it;
4360  // Collect the chain into a list.
4361  while (Tails.count(I) || Heads.count(I)) {
4362  if (VectorizedStores.count(I))
4363  break;
4364  Operands.push_back(I);
4365  // Move to the next value in the chain.
4366  I = ConsecutiveChain[I];
4367  }
4368 
4369  // FIXME: Is division-by-2 the correct step? Should we assert that the
4370  // register size is a power-of-2?
4371  for (unsigned Size = R.getMaxVecRegSize(); Size >= R.getMinVecRegSize();
4372  Size /= 2) {
4373  if (vectorizeStoreChain(Operands, R, Size)) {
4374  // Mark the vectorized stores so that we don't vectorize them again.
4375  VectorizedStores.insert(Operands.begin(), Operands.end());
4376  Changed = true;
4377  break;
4378  }
4379  }
4380  }
4381 
4382  return Changed;
4383 }
4384 
4385 void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
4386  // Initialize the collections. We will make a single pass over the block.
4387  Stores.clear();
4388  GEPs.clear();
4389 
4390  // Visit the store and getelementptr instructions in BB and organize them in
4391  // Stores and GEPs according to the underlying objects of their pointer
4392  // operands.
4393  for (Instruction &I : *BB) {
4394  // Ignore store instructions that are volatile or have a pointer operand
4395  // that doesn't point to a scalar type.
4396  if (auto *SI = dyn_cast<StoreInst>(&I)) {
4397  if (!SI->isSimple())
4398  continue;
4399  if (!isValidElementType(SI->getValueOperand()->getType()))
4400  continue;
4401  Stores[GetUnderlyingObject(SI->getPointerOperand(), *DL)].push_back(SI);
4402  }
4403 
4404  // Ignore getelementptr instructions that have more than one index, a
4405  // constant index, or a pointer operand that doesn't point to a scalar
4406  // type.
4407  else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
4408  auto Idx = GEP->idx_begin()->get();
4409  if (GEP->getNumIndices() > 1 || isa<Constant>(Idx))
4410  continue;
4411  if (!isValidElementType(Idx->getType()))
4412  continue;
4413  if (GEP->getType()->isVectorTy())
4414  continue;
4415  GEPs[GetUnderlyingObject(GEP->getPointerOperand(), *DL)].push_back(GEP);
4416  }
4417  }
4418 }
4419 
4420 bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
4421  if (!A || !B)
4422  return false;
4423  Value *VL[] = { A, B };
4424  return tryToVectorizeList(VL, R, None, true);
4425 }
4426 
4427 bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
4428  ArrayRef<Value *> BuildVector,
4429  bool AllowReorder) {
4430  if (VL.size() < 2)
4431  return false;
4432 
4433  DEBUG(dbgs() << "SLP: Trying to vectorize a list of length = " << VL.size()
4434  << ".\n");
4435 
4436  // Check that all of the parts are scalar instructions of the same type.
4437  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
4438  if (!I0)
4439  return false;
4440 
4441  unsigned Opcode0 = I0->getOpcode();
4442 
4443  unsigned Sz = R.getVectorElementSize(I0);
4444  unsigned MinVF = std::max(2U, R.getMinVecRegSize() / Sz);
4445  unsigned MaxVF = std::max<unsigned>(PowerOf2Floor(VL.size()), MinVF);
4446  if (MaxVF < 2)
4447  return false;
4448 
4449  for (Value *V : VL) {
4450  Type *Ty = V->getType();
4451  if (!isValidElementType(Ty))
4452  return false;
4453  Instruction *Inst = dyn_cast<Instruction>(V);
4454  if (!Inst || Inst->getOpcode() != Opcode0)
4455  return false;
4456  }
4457 
4458  bool Changed = false;
4459 
4460  // Keep track of values that were deleted by vectorizing in the loop below.
4461  SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
4462 
4463  unsigned NextInst = 0, MaxInst = VL.size();
4464  for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
4465  VF /= 2) {
4466  // No actual vectorization should happen, if number of parts is the same as
4467  // provided vectorization factor (i.e. the scalar type is used for vector
4468  // code during codegen).
4469  auto *VecTy = VectorType::get(VL[0]->getType(), VF);
4470  if (TTI->getNumberOfParts(VecTy) == VF)
4471  continue;
4472  for (unsigned I = NextInst; I < MaxInst; ++I) {
4473  unsigned OpsWidth = 0;
4474 
4475  if (I + VF > MaxInst)
4476  OpsWidth = MaxInst - I;
4477  else
4478  OpsWidth = VF;
4479 
4480  if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
4481  break;
4482 
4483  // Check that a previous iteration of this loop did not delete the Value.
4484  if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
4485  continue;
4486 
4487  DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
4488  << "\n");
4489  ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
4490 
4491  ArrayRef<Value *> BuildVectorSlice;
4492  if (!BuildVector.empty())
4493  BuildVectorSlice = BuildVector.slice(I, OpsWidth);
4494 
4495  R.buildTree(Ops, BuildVectorSlice);
4496  // TODO: check if we can allow reordering for more cases.
4497  if (AllowReorder && R.shouldReorder()) {
4498  // Conceptually, there is nothing actually preventing us from trying to
4499  // reorder a larger list. In fact, we do exactly this when vectorizing
4500  // reductions. However, at this point, we only expect to get here when
4501  // there are exactly two operations.
4502  assert(Ops.size() == 2);
4503  assert(BuildVectorSlice.empty());
4504  Value *ReorderedOps[] = {Ops[1], Ops[0]};
4505  R.buildTree(ReorderedOps, None);
4506  }
4507  if (R.isTreeTinyAndNotFullyVectorizable())
4508  continue;
4509 
4510  R.computeMinimumValueSizes();
4511  int Cost = R.getTreeCost();
4512 
4513  if (Cost < -SLPCostThreshold) {
4514  DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
4515  R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
4516  cast<Instruction>(Ops[0]))
4517  << "SLP vectorized with cost " << ore::NV("Cost", Cost)
4518  << " and with tree size "
4519  << ore::NV("TreeSize", R.getTreeSize()));
4520 
4521  Value *VectorizedRoot = R.vectorizeTree();
4522 
4523  // Reconstruct the build vector by extracting the vectorized root. This
4524  // way we handle the case where some elements of the vector are
4525  // undefined.
4526  // (return (inserelt <4 xi32> (insertelt undef (opd0) 0) (opd1) 2))
4527  if (!BuildVectorSlice.empty()) {
4528  // The insert point is the last build vector instruction. The
4529  // vectorized root will precede it. This guarantees that we get an
4530  // instruction. The vectorized tree could have been constant folded.
4531  Instruction *InsertAfter = cast<Instruction>(BuildVectorSlice.back());
4532  unsigned VecIdx = 0;
4533  for (auto &V : BuildVectorSlice) {
4534  IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
4535  ++BasicBlock::iterator(InsertAfter));
4536  Instruction *I = cast<Instruction>(V);
4537  assert(isa<InsertElementInst>(I) || isa<InsertValueInst>(I));
4538  Instruction *Extract =
4539  cast<Instruction>(Builder.CreateExtractElement(
4540  VectorizedRoot, Builder.getInt32(VecIdx++)));
4541  I->setOperand(1, Extract);
4542  I->moveAfter(Extract);
4543  InsertAfter = I;
4544  }
4545  }
4546  // Move to the next bundle.
4547  I += VF - 1;
4548  NextInst = I + 1;
4549  Changed = true;
4550  }
4551  }
4552  }
4553 
4554  return Changed;
4555 }
4556 
4557 bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
4558  if (!I)
4559  return false;
4560 
4561  if (!isa<BinaryOperator>(I) && !isa<CmpInst>(I))
4562  return false;
4563 
4564  Value *P = I->getParent();
4565 
4566  // Vectorize in current basic block only.
4567  auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
4568  auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
4569  if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
4570  return false;
4571 
4572  // Try to vectorize V.
4573  if (tryToVectorizePair(Op0, Op1, R))
4574  return true;
4575 
4576  auto *A = dyn_cast<BinaryOperator>(Op0);
4577  auto *B = dyn_cast<BinaryOperator>(Op1);
4578  // Try to skip B.
4579  if (B && B->hasOneUse()) {
4580  auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
4581  auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
4582  if (B0 && B0->getParent() == P && tryToVectorizePair(A, B0, R))
4583  return true;
4584  if (B1 && B1->getParent() == P && tryToVectorizePair(A, B1, R))
4585  return true;
4586  }
4587 
4588  // Try to skip A.
4589  if (A && A->hasOneUse()) {
4590  auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
4591  auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
4592  if (A0 && A0->getParent() == P && tryToVectorizePair(A0, B, R))
4593  return true;
4594  if (A1 && A1->getParent() == P && tryToVectorizePair(A1, B, R))
4595  return true;
4596  }
4597  return false;
4598 }
4599 
4600 /// \brief Generate a shuffle mask to be used in a reduction tree.
4601 ///
4602 /// \param VecLen The length of the vector to be reduced.
4603 /// \param NumEltsToRdx The number of elements that should be reduced in the
4604 /// vector.
4605 /// \param IsPairwise Whether the reduction is a pairwise or splitting
4606 /// reduction. A pairwise reduction will generate a mask of
4607 /// <0,2,...> or <1,3,..> while a splitting reduction will generate
4608 /// <2,3, undef,undef> for a vector of 4 and NumElts = 2.
4609 /// \param IsLeft True will generate a mask of even elements, odd otherwise.
4610 static Value *createRdxShuffleMask(unsigned VecLen, unsigned NumEltsToRdx,
4611  bool IsPairwise, bool IsLeft,
4612  IRBuilder<> &Builder) {
4613  assert((IsPairwise || !IsLeft) && "Don't support a <0,1,undef,...> mask");
4614 
4615  SmallVector<Constant *, 32> ShuffleMask(
4616  VecLen, UndefValue::get(Builder.getInt32Ty()));
4617 
4618  if (IsPairwise)
4619  // Build a mask of 0, 2, ... (left) or 1, 3, ... (right).
4620  for (unsigned i = 0; i != NumEltsToRdx; ++i)
4621  ShuffleMask[i] = Builder.getInt32(2 * i + !IsLeft);
4622  else
4623  // Move the upper half of the vector to the lower half.
4624  for (unsigned i = 0; i != NumEltsToRdx; ++i)
4625  ShuffleMask[i] = Builder.getInt32(NumEltsToRdx + i);
4626 
4627  return ConstantVector::get(ShuffleMask);
4628 }
4629 
4630 namespace {
4631 
4632 /// Model horizontal reductions.
4633 ///
4634 /// A horizontal reduction is a tree of reduction operations (currently add and
4635 /// fadd) that has operations that can be put into a vector as its leaf.
4636 /// For example, this tree:
4637 ///
4638 /// mul mul mul mul
4639 /// \ / \ /
4640 /// + +
4641 /// \ /
4642 /// +
4643 /// This tree has "mul" as its reduced values and "+" as its reduction
4644 /// operations. A reduction might be feeding into a store or a binary operation
4645 /// feeding a phi.
4646 /// ...
4647 /// \ /
4648 /// +
4649 /// |
4650 /// phi +=
4651 ///
4652 /// Or:
4653 /// ...
4654 /// \ /
4655 /// +
4656 /// |
4657 /// *p =
4658 ///
4659 class HorizontalReduction {
4660  using ReductionOpsType = SmallVector<Value *, 16>;
4661  using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
4662  ReductionOpsListType ReductionOps;
4663  SmallVector<Value *, 32> ReducedVals;
4664  // Use map vector to make stable output.
4666 
4667  /// Kind of the reduction data.
4668  enum ReductionKind {
4669  RK_None, /// Not a reduction.
4670  RK_Arithmetic, /// Binary reduction data.
4671  RK_Min, /// Minimum reduction data.
4672  RK_UMin, /// Unsigned minimum reduction data.
4673  RK_Max, /// Maximum reduction data.
4674  RK_UMax, /// Unsigned maximum reduction data.
4675  };
4676 
4677  /// Contains info about operation, like its opcode, left and right operands.
4678  class OperationData {
4679  /// Opcode of the instruction.
4680  unsigned Opcode = 0;
4681 
4682  /// Left operand of the reduction operation.
4683  Value *LHS = nullptr;
4684 
4685  /// Right operand of the reduction operation.
4686  Value *RHS = nullptr;
4687 
4688  /// Kind of the reduction operation.
4689  ReductionKind Kind = RK_None;
4690 
4691  /// True if float point min/max reduction has no NaNs.
4692  bool NoNaN = false;
4693 
4694  /// Checks if the reduction operation can be vectorized.
4695  bool isVectorizable() const {
4696  return LHS && RHS &&
4697  // We currently only support adds && min/max reductions.
4698  ((Kind == RK_Arithmetic &&
4699  (Opcode == Instruction::Add || Opcode == Instruction::FAdd)) ||
4700  ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
4701  (Kind == RK_Min || Kind == RK_Max)) ||
4702  (Opcode == Instruction::ICmp &&
4703  (Kind == RK_UMin || Kind == RK_UMax)));
4704  }
4705 
4706  /// Creates reduction operation with the current opcode.
4707  Value *createOp(IRBuilder<> &Builder, const Twine &Name) const {
4708  assert(isVectorizable() &&
4709  "Expected add|fadd or min/max reduction operation.");
4710  Value *Cmp;
4711  switch (Kind) {
4712  case RK_Arithmetic:
4713  return Builder.CreateBinOp((Instruction::BinaryOps)Opcode, LHS, RHS,
4714  Name);
4715  case RK_Min:
4716  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
4717  : Builder.CreateFCmpOLT(LHS, RHS);
4718  break;
4719  case RK_Max:
4720  Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
4721  : Builder.CreateFCmpOGT(LHS, RHS);
4722  break;
4723  case RK_UMin:
4724  assert(Opcode == Instruction::ICmp && "Expected integer types.");
4725  Cmp = Builder.CreateICmpULT(LHS, RHS);
4726  break;
4727  case RK_UMax:
4728  assert(Opcode == Instruction::ICmp && "Expected integer types.");
4729  Cmp = Builder.CreateICmpUGT(LHS, RHS);
4730  break;
4731  case RK_None:
4732  llvm_unreachable("Unknown reduction operation.");
4733  }
4734  return Builder.CreateSelect(Cmp, LHS, RHS, Name);
4735  }
4736 
4737  public:
4738  explicit OperationData() = default;
4739 
4740  /// Construction for reduced values. They are identified by opcode only and
4741  /// don't have associated LHS/RHS values.
4742  explicit OperationData(Value *V) {
4743  if (auto *I = dyn_cast<Instruction>(V))
4744  Opcode = I->getOpcode();
4745  }
4746 
4747  /// Constructor for reduction operations with opcode and its left and
4748  /// right operands.
4749  OperationData(unsigned Opcode, Value *LHS, Value *RHS, ReductionKind Kind,
4750  bool NoNaN = false)
4751  : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind), NoNaN(NoNaN) {
4752  assert(Kind != RK_None && "One of the reduction operations is expected.");
4753  }
4754 
4755  explicit operator bool() const { return Opcode; }
4756 
4757  /// Get the index of the first operand.
4758  unsigned getFirstOperandIndex() const {
4759  assert(!!*this && "The opcode is not set.");
4760  switch (Kind) {
4761  case RK_Min:
4762  case RK_UMin:
4763  case RK_Max:
4764  case RK_UMax:
4765  return 1;
4766  case RK_Arithmetic:
4767  case RK_None:
4768  break;
4769  }
4770  return 0;
4771  }
4772 
4773  /// Total number of operands in the reduction operation.
4774  unsigned getNumberOfOperands() const {
4775  assert(Kind != RK_None && !!*this && LHS && RHS &&
4776  "Expected reduction operation.");
4777  switch (Kind) {
4778  case RK_Arithmetic:
4779  return 2;
4780  case RK_Min:
4781  case RK_UMin:
4782  case RK_Max:
4783  case RK_UMax:
4784  return 3;
4785  case RK_None:
4786  break;
4787  }
4788  llvm_unreachable("Reduction kind is not set");
4789  }
4790 
4791  /// Checks if the operation has the same parent as \p P.
4792  bool hasSameParent(Instruction *I, Value *P, bool IsRedOp) const {
4793  assert(Kind != RK_None && !!*this && LHS && RHS &&
4794  "Expected reduction operation.");
4795  if (!IsRedOp)
4796  return I->getParent() == P;
4797  switch (Kind) {
4798  case RK_Arithmetic:
4799  // Arithmetic reduction operation must be used once only.
4800  return I->getParent() == P;
4801  case RK_Min:
4802  case RK_UMin:
4803  case RK_Max:
4804  case RK_UMax: {
4805  // SelectInst must be used twice while the condition op must have single
4806  // use only.
4807  auto *Cmp = cast<Instruction>(cast<SelectInst>(I)->getCondition());
4808  return I->getParent() == P && Cmp && Cmp->getParent() == P;
4809  }
4810  case RK_None:
4811  break;
4812  }
4813  llvm_unreachable("Reduction kind is not set");
4814  }
4815  /// Expected number of uses for reduction operations/reduced values.
4816  bool hasRequiredNumberOfUses(Instruction *I, bool IsReductionOp) const {
4817  assert(Kind != RK_None && !!*this && LHS && RHS &&
4818  "Expected reduction operation.");
4819  switch (Kind) {
4820  case RK_Arithmetic:
4821  return I->hasOneUse();
4822  case RK_Min:
4823  case RK_UMin:
4824  case RK_Max:
4825  case RK_UMax:
4826  return I->hasNUses(2) &&
4827  (!IsReductionOp ||
4828  cast<SelectInst>(I)->getCondition()->hasOneUse());
4829  case RK_None:
4830  break;
4831  }
4832  llvm_unreachable("Reduction kind is not set");
4833  }
4834 
4835  /// Initializes the list of reduction operations.
4836  void initReductionOps(ReductionOpsListType &ReductionOps) {
4837  assert(Kind != RK_None && !!*this && LHS && RHS &&
4838  "Expected reduction operation.");
4839  switch (Kind) {
4840  case RK_Arithmetic:
4841  ReductionOps.assign(1, ReductionOpsType());
4842  break;
4843  case RK_Min:
4844  case RK_UMin:
4845  case RK_Max:
4846  case RK_UMax:
4847  ReductionOps.assign(2, ReductionOpsType());
4848  break;
4849  case RK_None:
4850  llvm_unreachable("Reduction kind is not set");
4851  }
4852  }
4853  /// Add all reduction operations for the reduction instruction \p I.
4854  void addReductionOps(Instruction *I, ReductionOpsListType &ReductionOps) {
4855  assert(Kind != RK_None && !!*this && LHS && RHS &&
4856  "Expected reduction operation.");
4857  switch (Kind) {
4858  case RK_Arithmetic:
4859  ReductionOps[0].emplace_back(I);
4860  break;
4861  case RK_Min:
4862  case RK_UMin:
4863  case RK_Max:
4864  case RK_UMax:
4865  ReductionOps[0].emplace_back(cast<SelectInst>(I)->getCondition());
4866  ReductionOps[1].emplace_back(I);
4867  break;
4868  case RK_None:
4869  llvm_unreachable("Reduction kind is not set");
4870  }
4871  }
4872 
4873  /// Checks if instruction is associative and can be vectorized.
4874  bool isAssociative(Instruction *I) const {
4875  assert(Kind != RK_None && *this && LHS && RHS &&
4876  "Expected reduction operation.");
4877  switch (Kind) {
4878  case RK_Arithmetic:
4879  return I->isAssociative();
4880  case RK_Min:
4881  case RK_Max:
4882  return Opcode == Instruction::ICmp ||
4883  cast<Instruction>(I->getOperand(0))->hasUnsafeAlgebra();
4884  case RK_UMin:
4885  case RK_UMax:
4886  assert(Opcode == Instruction::ICmp &&
4887  "Only integer compare operation is expected.");
4888  return true;
4889  case RK_None:
4890  break;
4891  }
4892  llvm_unreachable("Reduction kind is not set");
4893  }
4894 
4895  /// Checks if the reduction operation can be vectorized.
4896  bool isVectorizable(Instruction *I) const {
4897  return isVectorizable() && isAssociative(I);
4898  }
4899 
4900  /// Checks if two operation data are both a reduction op or both a reduced
4901  /// value.
4902  bool operator==(const OperationData &OD) {
4903  assert(((Kind != OD.Kind) || ((!LHS == !OD.LHS) && (!RHS == !OD.RHS))) &&
4904  "One of the comparing operations is incorrect.");
4905  return this == &OD || (Kind == OD.Kind && Opcode == OD.Opcode);
4906  }
4907  bool operator!=(const OperationData &OD) { return !(*this == OD); }
4908  void clear() {
4909  Opcode = 0;
4910  LHS = nullptr;
4911  RHS = nullptr;
4912  Kind = RK_None;
4913  NoNaN = false;
4914  }
4915 
4916  /// Get the opcode of the reduction operation.
4917  unsigned getOpcode() const {
4918  assert(isVectorizable() && "Expected vectorizable operation.");
4919  return Opcode;
4920  }
4921 
4922  /// Get kind of reduction data.
4923  ReductionKind getKind() const { return Kind; }
4924  Value *getLHS() const { return LHS; }
4925  Value *getRHS() const { return RHS; }
4926  Type *getConditionType() const {
4927  switch (Kind) {
4928  case RK_Arithmetic:
4929  return nullptr;
4930  case RK_Min:
4931  case RK_Max:
4932  case RK_UMin:
4933  case RK_UMax:
4934  return CmpInst::makeCmpResultType(LHS->getType());
4935  case RK_None:
4936  break;
4937  }
4938  llvm_unreachable("Reduction kind is not set");
4939  }
4940 
4941  /// Creates reduction operation with the current opcode with the IR flags
4942  /// from \p ReductionOps.
4943  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
4944  const ReductionOpsListType &ReductionOps) const {
4945  assert(isVectorizable() &&
4946  "Expected add|fadd or min/max reduction operation.");
4947  auto *Op = createOp(Builder, Name);
4948  switch (Kind) {
4949  case RK_Arithmetic:
4950  propagateIRFlags(Op, ReductionOps[0]);
4951  return Op;
4952  case RK_Min:
4953  case RK_Max:
4954  case RK_UMin:
4955  case RK_UMax:
4956  if (auto *SI = dyn_cast<SelectInst>(Op))
4957  propagateIRFlags(SI->getCondition(), ReductionOps[0]);
4958  propagateIRFlags(Op, ReductionOps[1]);
4959  return Op;
4960  case RK_None:
4961  break;
4962  }
4963  llvm_unreachable("Unknown reduction operation.");
4964  }
4965  /// Creates reduction operation with the current opcode with the IR flags
4966  /// from \p I.
4967  Value *createOp(IRBuilder<> &Builder, const Twine &Name,
4968  Instruction *I) const {
4969  assert(isVectorizable() &&
4970  "Expected add|fadd or min/max reduction operation.");
4971  auto *Op = createOp(Builder, Name);
4972  switch (Kind) {
4973  case RK_Arithmetic:
4974  propagateIRFlags(Op, I);
4975  return Op;
4976  case RK_Min:
4977  case RK_Max:
4978  case RK_UMin:
4979  case RK_UMax:
4980  if (auto *SI = dyn_cast<SelectInst>(Op)) {
4981  propagateIRFlags(SI->getCondition(),
4982  cast<SelectInst>(I)->getCondition());
4983  }
4984  propagateIRFlags(Op, I);
4985  return Op;
4986  case RK_None:
4987  break;
4988  }
4989  llvm_unreachable("Unknown reduction operation.");
4990  }
4991 
4992  TargetTransformInfo::ReductionFlags getFlags() const {
4994  Flags.NoNaN = NoNaN;
4995  switch (Kind) {
4996  case RK_Arithmetic:
4997  break;
4998  case RK_Min:
4999  Flags.IsSigned = Opcode == Instruction::ICmp;
5000  Flags.IsMaxOp = false;
5001  break;
5002  case RK_Max:
5003  Flags.IsSigned = Opcode == Instruction::ICmp;
5004  Flags.IsMaxOp = true;
5005  break;
5006  case RK_UMin:
5007  Flags.IsSigned = false;
5008  Flags.IsMaxOp = false;
5009  break;
5010  case RK_UMax:
5011  Flags.IsSigned = false;
5012  Flags.IsMaxOp = true;
5013  break;
5014  case RK_None:
5015  llvm_unreachable("Reduction kind is not set");
5016  }
5017  return Flags;
5018  }
5019  };
5020 
5021  Instruction *ReductionRoot = nullptr;
5022 
5023  /// The operation data of the reduction operation.
5024  OperationData ReductionData;
5025 
5026  /// The operation data of the values we perform a reduction on.
5027  OperationData ReducedValueData;
5028 
5029  /// Should we model this reduction as a pairwise reduction tree or a tree that
5030  /// splits the vector in halves and adds those halves.
5031  bool IsPairwiseReduction = false;
5032 
5033  /// Checks if the ParentStackElem.first should be marked as a reduction
5034  /// operation with an extra argument or as extra argument itself.
5035  void markExtraArg(std::pair<Instruction *, unsigned> &ParentStackElem,
5036  Value *ExtraArg) {
5037  if (ExtraArgs.count(ParentStackElem.first)) {
5038  ExtraArgs[ParentStackElem.first] = nullptr;
5039  // We ran into something like:
5040  // ParentStackElem.first = ExtraArgs[ParentStackElem.first] + ExtraArg.
5041  // The whole ParentStackElem.first should be considered as an extra value
5042  // in this case.
5043  // Do not perform analysis of remaining operands of ParentStackElem.first
5044  // instruction, this whole instruction is an extra argument.
5045  ParentStackElem.second = ParentStackElem.first->getNumOperands();
5046  } else {
5047  // We ran into something like:
5048  // ParentStackElem.first += ... + ExtraArg + ...
5049  ExtraArgs[ParentStackElem.first] = ExtraArg;
5050  }
5051  }
5052 
5053  static OperationData getOperationData(Value *V) {
5054  if (!V)
5055  return OperationData();
5056 
5057  Value *LHS;
5058  Value *RHS;
5059  if (m_BinOp(m_Value(LHS), m_Value(RHS)).match(V)) {
5060  return OperationData(cast<BinaryOperator>(V)->getOpcode(), LHS, RHS,
5061  RK_Arithmetic);
5062  }
5063  if (auto *Select = dyn_cast<SelectInst>(V)) {
5064  // Look for a min/max pattern.
5065  if (m_UMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5066  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMin);
5067  } else if (m_SMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5068  return OperationData(Instruction::ICmp, LHS, RHS, RK_Min);
5069  } else if (m_OrdFMin(m_Value(LHS), m_Value(RHS)).match(Select) ||
5070  m_UnordFMin(m_Value(LHS), m_Value(RHS)).match(Select)) {
5071  return OperationData(
5072  Instruction::FCmp, LHS, RHS, RK_Min,
5073  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5074  } else if (m_UMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5075  return OperationData(Instruction::ICmp, LHS, RHS, RK_UMax);
5076  } else if (m_SMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5077  return OperationData(Instruction::ICmp, LHS, RHS, RK_Max);
5078  } else if (m_OrdFMax(m_Value(LHS), m_Value(RHS)).match(Select) ||
5079  m_UnordFMax(m_Value(LHS), m_Value(RHS)).match(Select)) {
5080  return OperationData(
5081  Instruction::FCmp, LHS, RHS, RK_Max,
5082  cast<Instruction>(Select->getCondition())->hasNoNaNs());
5083  }
5084  }
5085  return OperationData(V);
5086  }
5087 
5088 public:
5089  HorizontalReduction() = default;
5090 
5091  /// \brief Try to find a reduction tree.
5092  bool matchAssociativeReduction(PHINode *Phi, Instruction *B) {
5093  assert((!Phi || is_contained(Phi->operands(), B)) &&
5094  "Thi phi needs to use the binary operator");
5095 
5096  ReductionData = getOperationData(B);
5097 
5098  // We could have a initial reductions that is not an add.
5099  // r *= v1 + v2 + v3 + v4
5100  // In such a case start looking for a tree rooted in the first '+'.
5101  if (Phi) {
5102  if (ReductionData.getLHS() == Phi) {
5103  Phi = nullptr;
5104  B = dyn_cast<Instruction>(ReductionData.getRHS());
5105  ReductionData = getOperationData(B);
5106  } else if (ReductionData.getRHS() == Phi) {
5107  Phi = nullptr;
5108  B = dyn_cast<Instruction>(ReductionData.getLHS());
5109  ReductionData = getOperationData(B);
5110  }
5111  }
5112 
5113  if (!ReductionData.isVectorizable(B))
5114  return false;
5115 
5116  Type *Ty = B->getType();
5117  if (!isValidElementType(Ty))
5118  return false;
5119 
5120  ReducedValueData.clear();
5121  ReductionRoot = B;
5122 
5123  // Post order traverse the reduction tree starting at B. We only handle true
5124  // trees containing only binary operators.
5126  Stack.push_back(std::make_pair(B, ReductionData.getFirstOperandIndex()));
5127  ReductionData.initReductionOps(ReductionOps);
5128  while (!Stack.empty()) {
5129  Instruction *TreeN = Stack.back().first;
5130  unsigned EdgeToVist = Stack.back().second++;
5131  OperationData OpData = getOperationData(TreeN);
5132  bool IsReducedValue = OpData != ReductionData;
5133 
5134  // Postorder vist.
5135  if (IsReducedValue || EdgeToVist == OpData.getNumberOfOperands()) {
5136  if (IsReducedValue)
5137  ReducedVals.push_back(TreeN);
5138  else {
5139  auto I = ExtraArgs.find(TreeN);
5140  if (I != ExtraArgs.end() && !I->second) {
5141  // Check if TreeN is an extra argument of its parent operation.
5142  if (Stack.size() <= 1) {
5143  // TreeN can't be an extra argument as it is a root reduction
5144  // operation.
5145  return false;
5146  }
5147  // Yes, TreeN is an extra argument, do not add it to a list of
5148  // reduction operations.
5149  // Stack[Stack.size() - 2] always points to the parent operation.
5150  markExtraArg(Stack[Stack.size() - 2], TreeN);
5151  ExtraArgs.erase(TreeN);
5152  } else
5153  ReductionData.addReductionOps(TreeN, ReductionOps);
5154  }
5155  // Retract.
5156  Stack.pop_back();
5157  continue;
5158  }
5159 
5160  // Visit left or right.
5161  Value *NextV = TreeN->getOperand(EdgeToVist);
5162  if (NextV != Phi) {
5163  auto *I = dyn_cast<Instruction>(NextV);
5164  OpData = getOperationData(I);
5165  // Continue analysis if the next operand is a reduction operation or
5166  // (possibly) a reduced value. If the reduced value opcode is not set,
5167  // the first met operation != reduction operation is considered as the
5168  // reduced value class.
5169  if (I && (!ReducedValueData || OpData == ReducedValueData ||
5170  OpData == ReductionData)) {
5171  const bool IsReductionOperation = OpData == ReductionData;
5172  // Only handle trees in the current basic block.
5173  if (!ReductionData.hasSameParent(I, B->getParent(),
5174  IsReductionOperation)) {
5175  // I is an extra argument for TreeN (its parent operation).
5176  markExtraArg(Stack.back(), I);
5177  continue;
5178  }
5179 
5180  // Each tree node needs to have minimal number of users except for the
5181  // ultimate reduction.
5182  if (!ReductionData.hasRequiredNumberOfUses(I,
5183  OpData == ReductionData) &&
5184  I != B) {
5185  // I is an extra argument for TreeN (its parent operation).
5186  markExtraArg(Stack.back(), I);
5187  continue;
5188  }
5189 
5190  if (IsReductionOperation) {
5191  // We need to be able to reassociate the reduction operations.
5192  if (!OpData.isAssociative(I)) {
5193  // I is an extra argument for TreeN (its parent operation).
5194  markExtraArg(Stack.back(), I);
5195  continue;
5196  }
5197  } else if (ReducedValueData &&
5198  ReducedValueData != OpData) {
5199  // Make sure that the opcodes of the operations that we are going to
5200  // reduce match.
5201  // I is an extra argument for TreeN (its parent operation).
5202  markExtraArg(Stack.back(), I);
5203  continue;
5204  } else if (!ReducedValueData)
5205  ReducedValueData = OpData;
5206 
5207  Stack.push_back(std::make_pair(I, OpData.getFirstOperandIndex()));
5208  continue;
5209  }
5210  }
5211  // NextV is an extra argument for TreeN (its parent operation).
5212  markExtraArg(Stack.back(), NextV);
5213  }
5214  return true;
5215  }
5216 
5217  /// \brief Attempt to vectorize the tree found by
5218  /// matchAssociativeReduction.
5219  bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
5220  if (ReducedVals.empty())
5221  return false;
5222 
5223  // If there is a sufficient number of reduction values, reduce
5224  // to a nearby power-of-2. Can safely generate oversized
5225  // vectors and rely on the backend to split them to legal sizes.
5226  unsigned NumReducedVals = ReducedVals.size();
5227  if (NumReducedVals < 4)
5228  return false;
5229 
5230  unsigned ReduxWidth = PowerOf2Floor(NumReducedVals);
5231 
5232  Value *VectorizedTree = nullptr;
5233  IRBuilder<> Builder(ReductionRoot);
5234  FastMathFlags Unsafe;
5235  Unsafe.setUnsafeAlgebra();
5236  Builder.setFastMathFlags(Unsafe);
5237  unsigned i = 0;
5238 
5239  BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues;
5240  // The same extra argument may be used several time, so log each attempt
5241  // to use it.
5242  for (auto &Pair : ExtraArgs)
5243  ExternallyUsedValues[Pair.second].push_back(Pair.first);
5244  SmallVector<Value *, 16> IgnoreList;
5245  for (auto &V : ReductionOps)
5246  IgnoreList.append(V.begin(), V.end());
5247  while (i < NumReducedVals - ReduxWidth + 1 && ReduxWidth > 2) {
5248  auto VL = makeArrayRef(&ReducedVals[i], ReduxWidth);
5249  V.buildTree(VL, ExternallyUsedValues, IgnoreList);
5250  if (V.shouldReorder()) {
5251  SmallVector<Value *, 8> Reversed(VL.rbegin(), VL.rend());
5252  V.buildTree(Reversed, ExternallyUsedValues, IgnoreList);
5253  }
5254  if (V.isTreeTinyAndNotFullyVectorizable())
5255  break;
5256 
5257  V.computeMinimumValueSizes();
5258 
5259  // Estimate cost.
5260  int Cost =
5261  V.getTreeCost() + getReductionCost(TTI, ReducedVals[i], ReduxWidth);
5262  if (Cost >= -SLPCostThreshold)
5263  break;
5264 
5265  DEBUG(dbgs() << "SLP: Vectorizing horizontal reduction at cost:" << Cost
5266  << ". (HorRdx)\n");
5267  auto *I0 = cast<Instruction>(VL[0]);
5268  V.getORE()->emit(
5269  OptimizationRemark(SV_NAME, "VectorizedHorizontalReduction", I0)
5270  << "Vectorized horizontal reduction with cost "
5271  << ore::NV("Cost", Cost) << " and with tree size "
5272  << ore::NV("TreeSize", V.getTreeSize()));
5273 
5274  // Vectorize a tree.
5275  DebugLoc Loc = cast<Instruction>(ReducedVals[i])->getDebugLoc();
5276  Value *VectorizedRoot = V.vectorizeTree(ExternallyUsedValues);
5277 
5278  // Emit a reduction.
5279  Value *ReducedSubTree =
5280  emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
5281  if (VectorizedTree) {
5282  Builder.SetCurrentDebugLocation(Loc);
5283  OperationData VectReductionData(ReductionData.getOpcode(),
5284  VectorizedTree, ReducedSubTree,
5285  ReductionData.getKind());
5286  VectorizedTree =
5287  VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5288  } else
5289  VectorizedTree = ReducedSubTree;
5290  i += ReduxWidth;
5291  ReduxWidth = PowerOf2Floor(NumReducedVals - i);
5292  }
5293 
5294  if (VectorizedTree) {
5295  // Finish the reduction.
5296  for (; i < NumReducedVals; ++i) {
5297  auto *I = cast<Instruction>(ReducedVals[i]);
5298  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5299  OperationData VectReductionData(ReductionData.getOpcode(),
5300  VectorizedTree, I,
5301  ReductionData.getKind());
5302  VectorizedTree = VectReductionData.createOp(Builder, "", ReductionOps);
5303  }
5304  for (auto &Pair : ExternallyUsedValues) {
5305  assert(!Pair.second.empty() &&
5306  "At least one DebugLoc must be inserted");
5307  // Add each externally used value to the final reduction.
5308  for (auto *I : Pair.second) {
5309  Builder.SetCurrentDebugLocation(I->getDebugLoc());
5310  OperationData VectReductionData(ReductionData.getOpcode(),
5311  VectorizedTree, Pair.first,
5312  ReductionData.getKind());
5313  VectorizedTree = VectReductionData.createOp(Builder, "op.extra", I);
5314  }
5315  }
5316  // Update users.
5317  ReductionRoot->replaceAllUsesWith(VectorizedTree);
5318  }
5319  return VectorizedTree != nullptr;
5320  }
5321 
5322  unsigned numReductionValues() const {
5323  return ReducedVals.size();
5324  }
5325 
5326 private:
5327  /// \brief Calculate the cost of a reduction.
5328  int getReductionCost(TargetTransformInfo *TTI, Value *FirstReducedVal,
5329  unsigned ReduxWidth) {
5330  Type *ScalarTy = FirstReducedVal->getType();
5331  Type *VecTy = VectorType::get(ScalarTy, ReduxWidth);
5332 
5333  int PairwiseRdxCost;
5334  int SplittingRdxCost;
5335  switch (ReductionData.getKind()) {
5336  case RK_Arithmetic:
5337  PairwiseRdxCost =
5338  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5339  /*IsPairwiseForm=*/true);
5340  SplittingRdxCost =
5341  TTI->getArithmeticReductionCost(ReductionData.getOpcode(), VecTy,
5342  /*IsPairwiseForm=*/false);
5343  break;
5344  case RK_Min:
5345  case RK_Max:
5346  case RK_UMin:
5347  case RK_UMax: {
5348  Type *VecCondTy = CmpInst::makeCmpResultType(VecTy);
5349  bool IsUnsigned = ReductionData.getKind() == RK_UMin ||
5350  ReductionData.getKind() == RK_UMax;
5351  PairwiseRdxCost =
5352  TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5353  /*IsPairwiseForm=*/true, IsUnsigned);
5354  SplittingRdxCost =
5355  TTI->getMinMaxReductionCost(VecTy, VecCondTy,
5356  /*IsPairwiseForm=*/false, IsUnsigned);
5357  break;
5358  }
5359  case RK_None:
5360  llvm_unreachable("Expected arithmetic or min/max reduction operation");
5361  }
5362 
5363  IsPairwiseReduction = PairwiseRdxCost < SplittingRdxCost;
5364  int VecReduxCost = IsPairwiseReduction ? PairwiseRdxCost : SplittingRdxCost;
5365 
5366  int ScalarReduxCost;
5367  switch (ReductionData.getKind()) {
5368  case RK_Arithmetic:
5369  ScalarReduxCost =
5370  TTI->getArithmeticInstrCost(ReductionData.getOpcode(), ScalarTy);
5371  break;
5372  case RK_Min:
5373  case RK_Max:
5374  case RK_UMin:
5375  case RK_UMax:
5376  ScalarReduxCost =
5377  TTI->getCmpSelInstrCost(ReductionData.getOpcode(), ScalarTy) +
5378  TTI->getCmpSelInstrCost(Instruction::Select, ScalarTy,
5379  CmpInst::makeCmpResultType(ScalarTy));
5380  break;
5381  case RK_None:
5382  llvm_unreachable("Expected arithmetic or min/max reduction operation");
5383  }
5384  ScalarReduxCost *= (ReduxWidth - 1);
5385 
5386  DEBUG(dbgs() << "SLP: Adding cost " << VecReduxCost - ScalarReduxCost
5387  << " for reduction that starts with " << *FirstReducedVal
5388  << " (It is a "
5389  << (IsPairwiseReduction ? "pairwise" : "splitting")
5390  << " reduction)\n");
5391 
5392  return VecReduxCost - ScalarReduxCost;
5393  }
5394 
5395  /// \brief Emit a horizontal reduction of the vectorized value.
5396  Value *emitReduction(Value *VectorizedValue, IRBuilder<> &Builder,
5397  unsigned ReduxWidth, const TargetTransformInfo *TTI) {
5398  assert(VectorizedValue && "Need to have a vectorized tree node");
5399  assert(isPowerOf2_32(ReduxWidth) &&
5400  "We only handle power-of-two reductions for now");
5401 
5402  if (!IsPairwiseReduction)
5404  Builder, TTI, ReductionData.getOpcode(), VectorizedValue,
5405  ReductionData.getFlags(), ReductionOps.back());
5406 
5407  Value *TmpVec = VectorizedValue;
5408  for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) {
5409  Value *LeftMask =
5410  createRdxShuffleMask(ReduxWidth, i, true, true, Builder);
5411  Value *RightMask =
5412  createRdxShuffleMask(ReduxWidth, i, true, false, Builder);
5413 
5414  Value *LeftShuf = Builder.CreateShuffleVector(
5415  TmpVec, UndefValue::get(TmpVec->getType()), LeftMask, "rdx.shuf.l");
5416  Value *RightShuf = Builder.CreateShuffleVector(
5417  TmpVec, UndefValue::get(TmpVec->getType()), (RightMask),
5418  "rdx.shuf.r");
5419  OperationData VectReductionData(ReductionData.getOpcode(), LeftShuf,
5420  RightShuf, ReductionData.getKind());
5421  TmpVec = VectReductionData.createOp(Builder, "op.rdx", ReductionOps);
5422  }
5423 
5424  // The result is in the first element of the vector.
5425  return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));
5426  }
5427 };
5428 
5429 } // end anonymous namespace
5430 
5431 /// \brief Recognize construction of vectors like
5432 /// %ra = insertelement <4 x float> undef, float %s0, i32 0
5433 /// %rb = insertelement <4 x float> %ra, float %s1, i32 1
5434 /// %rc = insertelement <4 x float> %rb, float %s2, i32 2
5435 /// %rd = insertelement <4 x float> %rc, float %s3, i32 3
5436 /// starting from the last insertelement instruction.
5437 ///
5438 /// Returns true if it matches
5439 static bool findBuildVector(InsertElementInst *LastInsertElem,
5440  SmallVectorImpl<Value *> &BuildVector,
5441  SmallVectorImpl<Value *> &BuildVectorOpds) {
5442  Value *V = nullptr;
5443  do {
5444  BuildVector.push_back(LastInsertElem);
5445  BuildVectorOpds.push_back(LastInsertElem->getOperand(1));
5446  V = LastInsertElem->getOperand(0);
5447  if (isa<UndefValue>(V))
5448  break;
5449  LastInsertElem = dyn_cast<InsertElementInst>(V);
5450  if (!LastInsertElem || !LastInsertElem->hasOneUse())
5451  return false;
5452  } while (true);
5453  std::reverse(BuildVector.begin(), BuildVector.end());
5454  std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5455  return true;
5456 }
5457 
5458 /// \brief Like findBuildVector, but looks for construction of aggregate.
5459 ///
5460 /// \return true if it matches.
5462  SmallVectorImpl<Value *> &BuildVector,
5463  SmallVectorImpl<Value *> &BuildVectorOpds) {
5464  Value *V;
5465  do {
5466  BuildVector.push_back(IV);
5467  BuildVectorOpds.push_back(IV->getInsertedValueOperand());
5468  V = IV->getAggregateOperand();
5469  if (isa<UndefValue>(V))
5470  break;
5471  IV = dyn_cast<InsertValueInst>(V);
5472  if (!IV || !IV->hasOneUse())
5473  return false;
5474  } while (true);
5475  std::reverse(BuildVector.begin(), BuildVector.end());
5476  std::reverse(BuildVectorOpds.begin(), BuildVectorOpds.end());
5477  return true;
5478 }
5479 
5480 static bool PhiTypeSorterFunc(Value *V, Value *V2) {
5481  return V->getType() < V2->getType();
5482 }
5483 
5484 /// \brief Try and get a reduction value from a phi node.
5485 ///
5486 /// Given a phi node \p P in a block \p ParentBB, consider possible reductions
5487 /// if they come from either \p ParentBB or a containing loop latch.
5488 ///
5489 /// \returns A candidate reduction value if possible, or \code nullptr \endcode
5490 /// if not possible.
5492  BasicBlock *ParentBB, LoopInfo *LI) {
5493  // There are situations where the reduction value is not dominated by the
5494  // reduction phi. Vectorizing such cases has been reported to cause
5495  // miscompiles. See PR25787.
5496  auto DominatedReduxValue = [&](Value *R) {
5497  return (
5498  dyn_cast<Instruction>(R) &&
5499  DT->dominates(P->getParent(), dyn_cast<Instruction>(R)->getParent()));
5500  };
5501 
5502  Value *Rdx = nullptr;
5503 
5504  // Return the incoming value if it comes from the same BB as the phi node.
5505  if (P->getIncomingBlock(0) == ParentBB) {
5506  Rdx = P->getIncomingValue(0);
5507  } else if (P->getIncomingBlock(1) == ParentBB) {
5508  Rdx = P->getIncomingValue(1);
5509  }
5510 
5511  if (Rdx && DominatedReduxValue(Rdx))
5512  return Rdx;
5513 
5514  // Otherwise, check whether we have a loop latch to look at.
5515  Loop *BBL = LI->getLoopFor(ParentBB);
5516  if (!BBL)
5517  return nullptr;
5518  BasicBlock *BBLatch = BBL->getLoopLatch();
5519  if (!BBLatch)
5520  return nullptr;
5521 
5522  // There is a loop latch, return the incoming value if it comes from
5523  // that. This reduction pattern occasionally turns up.
5524  if (P->getIncomingBlock(0) == BBLatch) {
5525  Rdx = P->getIncomingValue(0);
5526  } else if (P->getIncomingBlock(1) == BBLatch) {
5527  Rdx = P->getIncomingValue(1);
5528  }
5529 
5530  if (Rdx && DominatedReduxValue(Rdx))
5531  return Rdx;
5532 
5533  return nullptr;
5534 }
5535 
5536 /// Attempt to reduce a horizontal reduction.
5537 /// If it is legal to match a horizontal reduction feeding the phi node \a P
5538 /// with reduction operators \a Root (or one of its operands) in a basic block
5539 /// \a BB, then check if it can be done. If horizontal reduction is not found
5540 /// and root instruction is a binary operation, vectorization of the operands is
5541 /// attempted.
5542 /// \returns true if a horizontal reduction was matched and reduced or operands
5543 /// of one of the binary instruction were vectorized.
5544 /// \returns false if a horizontal reduction was not matched (or not possible)
5545 /// or no vectorization of any binary operation feeding \a Root instruction was
5546 /// performed.
5548  PHINode *P, Instruction *Root, BasicBlock *BB, BoUpSLP &R,
5549  TargetTransformInfo *TTI,
5550  const function_ref<bool(Instruction *, BoUpSLP &)> Vectorize) {
5551  if (!ShouldVectorizeHor)
5552  return false;
5553 
5554  if (!Root)
5555  return false;
5556 
5557  if (Root->getParent() != BB || isa<PHINode>(Root))
5558  return false;
5559  // Start analysis starting from Root instruction. If horizontal reduction is
5560  // found, try to vectorize it. If it is not a horizontal reduction or
5561  // vectorization is not possible or not effective, and currently analyzed
5562  // instruction is a binary operation, try to vectorize the operands, using
5563  // pre-order DFS traversal order. If the operands were not vectorized, repeat
5564  // the same procedure considering each operand as a possible root of the
5565  // horizontal reduction.
5566  // Interrupt the process if the Root instruction itself was vectorized or all
5567  // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
5568  SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
5569  SmallSet<Value *, 8> VisitedInstrs;
5570  bool Res = false;
5571  while (!Stack.empty()) {
5572  Value *V;
5573  unsigned Level;
5574  std::tie(V, Level) = Stack.pop_back_val();
5575  if (!V)
5576  continue;
5577  auto *Inst = dyn_cast<Instruction>(V);
5578  if (!Inst)
5579  continue;
5580  auto *BI = dyn_cast<BinaryOperator>(Inst);
5581  auto *SI = dyn_cast<SelectInst>(Inst);
5582  if (BI || SI) {
5583  HorizontalReduction HorRdx;
5584  if (HorRdx.matchAssociativeReduction(P, Inst)) {
5585  if (HorRdx.tryToReduce(R, TTI)) {
5586  Res = true;
5587  // Set P to nullptr to avoid re-analysis of phi node in
5588  // matchAssociativeReduction function unless this is the root node.
5589  P = nullptr;
5590  continue;
5591  }
5592  }
5593  if (P && BI) {
5594  Inst = dyn_cast<Instruction>(BI->getOperand(0));
5595  if (Inst == P)
5596  Inst = dyn_cast<Instruction>(BI->getOperand(1));
5597  if (!Inst) {
5598  // Set P to nullptr to avoid re-analysis of phi node in
5599  // matchAssociativeReduction function unless this is the root node.
5600  P = nullptr;
5601  continue;
5602  }
5603  }
5604  }
5605  // Set P to nullptr to avoid re-analysis of phi node in
5606  // matchAssociativeReduction function unless this is the root node.
5607  P = nullptr;
5608  if (Vectorize(Inst, R)) {
5609  Res = true;
5610  continue;
5611  }
5612 
5613  // Try to vectorize operands.
5614  // Continue analysis for the instruction from the same basic block only to
5615  // save compile time.
5616  if (++Level < RecursionMaxDepth)
5617  for (auto *Op : Inst->operand_values())
5618  if (VisitedInstrs.insert(Op).second)
5619  if (auto *I = dyn_cast<Instruction>(Op))
5620  if (!isa<PHINode>(I) && I->getParent() == BB)
5621  Stack.emplace_back(Op, Level);
5622  }
5623  return Res;
5624 }
5625 
5626 bool SLPVectorizerPass::vectorizeRootInstruction(PHINode *P, Value *V,
5627  BasicBlock *BB, BoUpSLP &R,
5628  TargetTransformInfo *TTI) {
5629  if (!V)
5630  return false;
5631  auto *I = dyn_cast<Instruction>(V);
5632  if (!I)
5633  return false;
5634 
5635  if (!isa<BinaryOperator>(I))
5636  P = nullptr;
5637  // Try to match and vectorize a horizontal reduction.
5638  auto &&ExtraVectorization = [this](Instruction *I, BoUpSLP &R) -> bool {
5639  return tryToVectorize(I, R);
5640  };
5641  return tryToVectorizeHorReductionOrInstOperands(P, I, BB, R, TTI,
5642  ExtraVectorization);
5643 }
5644 
5645 bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
5646  BasicBlock *BB, BoUpSLP &R) {
5647  const DataLayout &DL = BB->getModule()->getDataLayout();
5648  if (!R.canMapToVector(IVI->getType(), DL))
5649  return false;
5650 
5651  SmallVector<Value *, 16> BuildVector;
5652  SmallVector<Value *, 16> BuildVectorOpds;
5653  if (!findBuildAggregate(IVI, BuildVector, BuildVectorOpds))
5654  return false;
5655 
5656  DEBUG(dbgs() << "SLP: array mappable to vector: " << *IVI << "\n");
5657  return tryToVectorizeList(BuildVectorOpds, R, BuildVector, false);
5658 }
5659 
5660 bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
5661  BasicBlock *BB, BoUpSLP &R) {
5662  SmallVector<Value *, 16> BuildVector;
5663  SmallVector<Value *, 16> BuildVectorOpds;
5664  if (!findBuildVector(IEI, BuildVector, BuildVectorOpds))
5665  return false;
5666 
5667  // Vectorize starting with the build vector operands ignoring the BuildVector
5668  // instructions for the purpose of scheduling and user extraction.
5669  return tryToVectorizeList(BuildVectorOpds, R, BuildVector);
5670 }
5671 
5672 bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
5673  BoUpSLP &R) {
5674  if (tryToVectorizePair(CI->getOperand(0), CI->getOperand(1), R))
5675  return true;
5676 
5677  bool OpsChanged = false;
5678  for (int Idx = 0; Idx < 2; ++Idx) {
5679  OpsChanged |=
5680  vectorizeRootInstruction(nullptr, CI->getOperand(Idx), BB, R, TTI);
5681  }
5682  return OpsChanged;
5683 }
5684 
5685 bool SLPVectorizerPass::vectorizeSimpleInstructions(
5686  SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
5687  bool OpsChanged = false;
5688  for (auto &VH : reverse(Instructions)) {
5689  auto *I = dyn_cast_or_null<Instruction>(VH);
5690  if (!I)
5691  continue;
5692  if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
5693  OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
5694  else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I))
5695  OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R);
5696  else if (auto *CI = dyn_cast<CmpInst>(I))
5697  OpsChanged |= vectorizeCmpInst(CI, BB, R);
5698  }
5699  Instructions.clear();
5700  return OpsChanged;
5701 }
5702 
5703 bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
5704  bool Changed = false;
5705  SmallVector<Value *, 4> Incoming;
5706  SmallSet<Value *, 16> VisitedInstrs;
5707 
5708  bool HaveVectorizedPhiNodes = true;
5709  while (HaveVectorizedPhiNodes) {
5710  HaveVectorizedPhiNodes = false;
5711 
5712  // Collect the incoming values from the PHIs.
5713  Incoming.clear();
5714  for (Instruction &I : *BB) {
5715  PHINode *P = dyn_cast<PHINode>(&I);
5716  if (!P)
5717  break;
5718 
5719  if (!VisitedInstrs.count(P))
5720  Incoming.push_back(P);
5721  }
5722 
5723  // Sort by type.
5724  std::stable_sort(Incoming.begin(), Incoming.end(), PhiTypeSorterFunc);
5725 
5726  // Try to vectorize elements base on their type.
5727  for (SmallVector<Value *, 4>::iterator IncIt = Incoming.begin(),
5728  E = Incoming.end();
5729  IncIt != E;) {
5730 
5731  // Look for the next elements with the same type.
5732  SmallVector<Value *, 4>::iterator SameTypeIt = IncIt;
5733  while (SameTypeIt != E &&
5734  (*SameTypeIt)->getType() == (*IncIt)->getType()) {
5735  VisitedInstrs.insert(*SameTypeIt);
5736  ++SameTypeIt;
5737  }
5738 
5739  // Try to vectorize them.
5740  unsigned NumElts = (SameTypeIt - IncIt);
5741  DEBUG(errs() << "SLP: Trying to vectorize starting at PHIs (" << NumElts << ")\n");
5742  // The order in which the phi nodes appear in the program does not matter.
5743  // So allow tryToVectorizeList to reorder them if it is beneficial. This
5744  // is done when there are exactly two elements since tryToVectorizeList
5745  // asserts that there are only two values when AllowReorder is true.
5746  bool AllowReorder = NumElts == 2;
5747  if (NumElts > 1 && tryToVectorizeList(makeArrayRef(IncIt, NumElts), R,
5748  None, AllowReorder)) {
5749  // Success start over because instructions might have been changed.
5750  HaveVectorizedPhiNodes = true;
5751  Changed = true;
5752  break;
5753  }
5754 
5755  // Start over at the next instruction of a different type (or the end).
5756  IncIt = SameTypeIt;
5757  }
5758  }
5759 
5760  VisitedInstrs.clear();
5761 
5762  SmallVector<WeakVH, 8> PostProcessInstructions;
5764  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
5765  // We may go through BB multiple times so skip the one we have checked.
5766  if (!VisitedInstrs.insert(&*it).second) {
5767  if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
5768  vectorizeSimpleInstructions(PostProcessInstructions, BB, R)) {
5769  // We would like to start over since some instructions are deleted
5770  // and the iterator may become invalid value.
5771  Changed = true;
5772  it = BB->begin();
5773  e = BB->end();
5774  }
5775  continue;
5776  }
5777 
5778  if (isa<DbgInfoIntrinsic>(it))
5779  continue;
5780 
5781  // Try to vectorize reductions that use PHINodes.
5782  if (PHINode *P = dyn_cast<PHINode>(it)) {
5783  // Check that the PHI is a reduction PHI.
5784  if (P->getNumIncomingValues() != 2)
5785  return Changed;
5786 
5787  // Try to match and vectorize a horizontal reduction.
5788  if (vectorizeRootInstruction(P, getReductionValue(DT, P, BB, LI), BB, R,
5789  TTI)) {
5790  Changed = true;
5791  it = BB->begin();
5792  e = BB->end();
5793  continue;
5794  }
5795  continue;
5796  }
5797 
5798  // Ran into an instruction without users, like terminator, or function call
5799  // with ignored return value, store. Ignore unused instructions (basing on
5800  // instruction type, except for CallInst and InvokeInst).
5801  if (it->use_empty() && (it->getType()->isVoidTy() || isa<CallInst>(it) ||
5802  isa<InvokeInst>(it))) {
5803  KeyNodes.insert(&*it);
5804  bool OpsChanged = false;
5805  if (ShouldStartVectorizeHorAtStore || !isa<StoreInst>(it)) {
5806  for (auto *V : it->operand_values()) {
5807  // Try to match and vectorize a horizontal reduction.
5808  OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
5809  }
5810  }
5811  // Start vectorization of post-process list of instructions from the
5812  // top-tree instructions to try to vectorize as many instructions as
5813  // possible.
5814  OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R);
5815  if (OpsChanged) {
5816  // We would like to start over since some instructions are deleted
5817  // and the iterator may become invalid value.
5818  Changed = true;
5819  it = BB->begin();
5820  e = BB->end();
5821  continue;
5822  }
5823  }
5824 
5825  if (isa<InsertElementInst>(it) || isa<CmpInst>(it) ||
5826  isa<InsertValueInst>(it))
5827  PostProcessInstructions.push_back(&*it);
5828 
5829  }
5830 
5831  return Changed;
5832 }
5833 
5834 bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
5835  auto Changed = false;
5836  for (auto &Entry : GEPs) {
5837  // If the getelementptr list has fewer than two elements, there's nothing
5838  // to do.
5839  if (Entry.second.size() < 2)
5840  continue;
5841 
5842  DEBUG(dbgs() << "SLP: Analyzing a getelementptr list of length "
5843  << Entry.second.size() << ".\n");
5844 
5845  // We process the getelementptr list in chunks of 16 (like we do for
5846  // stores) to minimize compile-time.
5847  for (unsigned BI = 0, BE = Entry.second.size(); BI < BE; BI += 16) {
5848  auto Len = std::min<unsigned>(BE - BI, 16);
5849  auto GEPList = makeArrayRef(&Entry.second[BI], Len);
5850 
5851  // Initialize a set a candidate getelementptrs. Note that we use a
5852  // SetVector here to preserve program order. If the index computations
5853  // are vectorizable and begin with loads, we want to minimize the chance
5854  // of having to reorder them later.
5855  SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
5856 
5857  // Some of the candidates may have already been vectorized after we
5858  // initially collected them. If so, the WeakTrackingVHs will have
5859  // nullified the
5860  // values, so remove them from the set of candidates.
5861  Candidates.remove(nullptr);
5862 
5863  // Remove from the set of candidates all pairs of getelementptrs with
5864  // constant differences. Such getelementptrs are likely not good
5865  // candidates for vectorization in a bottom-up phase since one can be
5866  // computed from the other. We also ensure all candidate getelementptr
5867  // indices are unique.
5868  for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
5869  auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
5870  if (!Candidates.count(GEPI))
5871  continue;
5872  auto *SCEVI = SE->getSCEV(GEPList[I]);
5873  for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
5874  auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
5875  auto *SCEVJ = SE->getSCEV(GEPList[J]);
5876  if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
5877  Candidates.remove(GEPList[I]);
5878  Candidates.remove(GEPList[J]);
5879  } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
5880  Candidates.r